diff --git a/.clang-format b/.clang-format index cf5af1f0a0a..7ddeba81288 100644 --- a/.clang-format +++ b/.clang-format @@ -1,24 +1,21 @@ ---- -BasedOnStyle: WebKit -Language: Cpp -AlignAfterOpenBracket: false +BasedOnStyle: WebKit +Language: Cpp +AlignAfterOpenBracket: AlwaysBreak BreakBeforeBraces: Custom -BraceWrapping: { - AfterClass: 'true' - AfterControlStatement: 'true' - AfterEnum : 'true' - AfterFunction : 'true' - AfterNamespace : 'true' - AfterStruct : 'true' - AfterUnion : 'true' - BeforeCatch : 'true' - BeforeElse : 'true' - IndentBraces : 'false' -} - +BraceWrapping: + AfterClass: true + AfterControlStatement: true + AfterEnum: true + AfterFunction: true + AfterNamespace: true + AfterStruct: true + AfterUnion: true + BeforeCatch: true + BeforeElse: true + IndentBraces: false BreakConstructorInitializersBeforeComma: false Cpp11BracedListStyle: true -ColumnLimit: 140 +ColumnLimit: 140 ConstructorInitializerAllOnOneLineOrOnePerLine: true ExperimentalAutoDetectBinPacking: true UseTab: Never @@ -28,36 +25,35 @@ Standard: Cpp11 PointerAlignment: Middle MaxEmptyLinesToKeep: 2 KeepEmptyLinesAtTheStartOfBlocks: false -#AllowShortFunctionsOnASingleLine: Inline -AllowShortFunctionsOnASingleLine: Empty +AllowShortFunctionsOnASingleLine: InlineOnly AlwaysBreakTemplateDeclarations: true IndentCaseLabels: true -#SpaceAfterTemplateKeyword: true +SpaceAfterTemplateKeyword: true SortIncludes: true IncludeCategories: - - Regex: '^<[a-z_]+>' - Priority: 1 - - Regex: '^<[a-z_]+.h>' - Priority: 2 - - Regex: '^["<](common|ext|mysqlxx|daemon|zkutil)/' - Priority: 90 - - Regex: '^["<](DB)/' - Priority: 100 - - Regex: '^["<](Poco)/' - Priority: 50 - - Regex: '^"' - Priority: 110 - - Regex: '/' - Priority: 30 - - Regex: '.*' - Priority: 40 + - Regex: '^<[a-z_]+>' + Priority: 1 + - Regex: '^<[a-z_]+.h>' + Priority: 2 + - Regex: '^["<](common|ext|mysqlxx|daemon|zkutil)/' + Priority: 90 + - Regex: '^["<](DB)/' + Priority: 100 + - Regex: '^["<](Poco)/' + Priority: 50 + - Regex: '^"' + Priority: 110 + - Regex: '/' + Priority: 30 + - Regex: '.*' + Priority: 40 ReflowComments: false AlignEscapedNewlinesLeft: true # Not changed: AccessModifierOffset: -4 AlignConsecutiveAssignments: false -AlignOperands: false +AlignOperands: false AlignTrailingComments: false AllowAllParametersOfDeclarationOnNextLine: true AllowShortBlocksOnASingleLine: false @@ -70,16 +66,15 @@ BinPackArguments: false BinPackParameters: false BreakBeforeBinaryOperators: All BreakBeforeTernaryOperators: true -CommentPragmas: '^ IWYU pragma:' +CommentPragmas: '^ IWYU pragma:' ConstructorInitializerIndentWidth: 4 ContinuationIndentWidth: 4 DerivePointerAlignment: false -DisableFormat: false -ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] -IndentWidth: 4 +DisableFormat: false +IndentWidth: 4 IndentWrappedFunctionNames: false MacroBlockBegin: '' -MacroBlockEnd: '' +MacroBlockEnd: '' NamespaceIndentation: Inner ObjCBlockIndentWidth: 4 ObjCSpaceAfterProperty: true @@ -99,5 +94,3 @@ SpacesInContainerLiterals: true SpacesInCStyleCastParentheses: false SpacesInParentheses: false SpacesInSquareBrackets: false -... - diff --git a/.gitignore b/.gitignore index ac01f15002f..585a4074767 100644 --- a/.gitignore +++ b/.gitignore @@ -9,10 +9,16 @@ # auto generated files *.logrt -build -/docs/en_single_page/ -/docs/ru_single_page/ -/docs/venv/ +/build +/build_* +/build-* +/docs/build +/docs/edit +/docs/tools/venv/ +/docs/en/development/build/ +/docs/ru/development/build/ +/docs/en/single.md +/docs/ru/single.md # callgrind files callgrind.out.* @@ -43,6 +49,7 @@ cmake-build-* # Python cache *.pyc __pycache__ +*.pytest_cache # ignore generated files *-metrika-yandex @@ -173,7 +180,6 @@ utils/zookeeper-create-entry-to-download-part/zookeeper-create-entry-to-download utils/zookeeper-dump-tree/zookeeper-dump-tree utils/zookeeper-remove-by-list/zookeeper-remove-by-list dbms/src/Storages/tests/remove_symlink_directory -dbms/tests/queries/1_stateful debian/control debian/copyright debian/tmp/ @@ -236,3 +242,9 @@ node_modules public website/docs website/presentations +website/package-lock.json +.DS_Store +*/.DS_Store + +# Ignore files for locally disabled tests +/dbms/tests/queries/**/*.disabled diff --git a/.gitmodules b/.gitmodules index ab5c0c7bb74..a9e50ab8f6d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "contrib/zookeeper"] - path = contrib/zookeeper - url = https://github.com/ClickHouse-Extras/zookeeper.git [submodule "contrib/poco"] path = contrib/poco url = https://github.com/ClickHouse-Extras/poco @@ -18,7 +15,7 @@ url = https://github.com/google/cctz.git [submodule "contrib/zlib-ng"] path = contrib/zlib-ng - url = https://github.com/Dead2/zlib-ng.git + url = https://github.com/ClickHouse-Extras/zlib-ng.git [submodule "contrib/googletest"] path = contrib/googletest url = https://github.com/google/googletest.git @@ -37,3 +34,18 @@ [submodule "contrib/boost"] path = contrib/boost url = https://github.com/ClickHouse-Extras/boost.git +[submodule "contrib/llvm"] + path = contrib/llvm + url = https://github.com/ClickHouse-Extras/llvm +[submodule "contrib/mariadb-connector-c"] + path = contrib/mariadb-connector-c + url = https://github.com/MariaDB/mariadb-connector-c.git +[submodule "contrib/jemalloc"] + path = contrib/jemalloc + url = https://github.com/jemalloc/jemalloc.git +[submodule "contrib/unixodbc"] + path = contrib/unixodbc + url = https://github.com/ClickHouse-Extras/UnixODBC.git +[submodule "contrib/base64"] + path = contrib/base64 + url = https://github.com/aklomp/base64.git diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 7af12080396..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,150 +0,0 @@ -language: generic - -matrix: - fast_finish: true - include: -# - os: linux -# -# cache: -# ccache: true -# timeout: 1000 -# -# addons: -# apt: -# sources: -# - ubuntu-toolchain-r-test -# packages: [ g++-7, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libzookeeper-mt-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo, openssl ] -# -# env: -# - MATRIX_EVAL="export CC=gcc-7 && export CXX=g++-7" -# -# script: -# - env TEST_RUN= utils/travis/normal.sh - - - # We need to have gcc7 headers to compile c++17 code on clang - - os: linux - - cache: - ccache: true - timeout: 1000 - directories: - - /home/travis/.ccache - - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-trusty-5.0 - packages: [ g++-7, clang-5.0, lld-5.0, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libzookeeper-mt-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo, openssl] - - env: - - MATRIX_EVAL="export CC=clang-5.0 && export CXX=clang++-5.0" - - script: - - utils/travis/normal.sh - - -# TODO: fix internal compiler -# - os: linux -# -# sudo: required -# -# cache: -# timeout: 1000 -# directories: -# - /var/cache/pbuilder/ccache -# -# addons: -# apt: -# packages: [ pbuilder, fakeroot, debhelper ] -# -# env: -# - MATRIX_EVAL="export DEB_CC=clang-5.0 && export DEB_CXX=clang++-5.0" -# -# script: -# - utils/travis/pbuilder.sh - - - - os: linux - - sudo: required - - cache: - timeout: 1000 - directories: - - /var/cache/pbuilder/ccache - - addons: - apt: - packages: [ pbuilder, fakeroot, debhelper ] - - script: - - utils/travis/pbuilder.sh - - -# - os: linux -# -# sudo: required -# -# cache: -# timeout: 1000 -# directories: -# - /var/cache/pbuilder/ccache -# -# addons: -# apt: -# packages: [ pbuilder, fakeroot, debhelper ] -# -# env: -# - MATRIX_EVAL="export ARCH=i386" -# -# script: -# - env PBUILDER_TIMEOUT=40m TEST_TRUE=true TEST_RUN= utils/travis/pbuilder.sh - - -# TODO: Can't bootstrap bionic on trusty host -# - os: linux -# -# sudo: required -# -# cache: -# timeout: 1000 -# directories: -# - /var/cache/pbuilder/ccache -# -# addons: -# apt: -# packages: [ pbuilder, fakeroot, debhelper ] -# -# env: -# - MATRIX_EVAL="export DEB_CC=clang-6.0 && export DEB_CXX=clang++-6.0 && export DIST=bionic && export EXTRAPACKAGES='clang-6.0 lld-6.0'" -# -# script: -# - utils/travis/pbuilder.sh - - -# Cant fit to time limit (48min) -# - os: osx -# osx_image: xcode9.2 -# -# cache: -# ccache: true -# timeout: 1000 -# -# before_install: -# - brew install unixodbc gcc ccache libtool gettext zlib readline double-conversion gperftools google-sparsehash lz4 zstd || true -# - brew link --overwrite gcc || true -# -# env: -# - MATRIX_EVAL="export CC=gcc-7 && export CXX=g++-7" -# -# script: -# - env CMAKE_FLAGS="-DUSE_INTERNAL_BOOST_LIBRARY=1" utils/travis/normal.sh - - - allow_failures: - - os: osx - -before_script: - - eval "${MATRIX_EVAL}" diff --git a/.travis.yml.bak b/.travis.yml.bak new file mode 100644 index 00000000000..0aed0a24427 --- /dev/null +++ b/.travis.yml.bak @@ -0,0 +1,50 @@ +language: generic + +matrix: + fast_finish: true + include: + # We need to have gcc7 headers to compile c++17 code on clang +# - os: linux +# +# cache: +# ccache: true +# timeout: 1000 +# directories: +# - /home/travis/.ccache +# +# addons: +# apt: +# update: true +# sources: +# - ubuntu-toolchain-r-test +# - llvm-toolchain-trusty-5.0 +# packages: [ ninja-build, g++-7, clang-5.0, lld-5.0, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo, openssl] +# +# env: +# - MATRIX_EVAL="export CC=clang-5.0 CXX=clang++-5.0" +# +# script: +# - utils/travis/normal.sh + + - os: linux + + sudo: required + + cache: + timeout: 1000 + directories: + - /var/cache/pbuilder/ccache + + addons: + apt: + update: true + packages: [ pbuilder, fakeroot, debhelper ] + + script: + - utils/travis/pbuilder.sh + + allow_failures: + - os: osx + +before_script: + - eval "${MATRIX_EVAL}" diff --git a/CHANGELOG.draft.md b/CHANGELOG.draft.md new file mode 100644 index 00000000000..0674a4193fe --- /dev/null +++ b/CHANGELOG.draft.md @@ -0,0 +1,4 @@ +* Настройка `enable_optimize_predicate_expression` выключена по-умолчанию. + +### Улучшения: +* Файлы *-preprocessed.xml записываются в директорию с данными (/var/lib/clickhouse/preprocessed_configs). Для /etc/clickhouse-server больше не нужен +w для пользователя clickhouse. Для удобства создан симлинк /var/lib/clickhouse/preprocessed_configs -> /etc/clickhouse-server/preprocessed diff --git a/CHANGELOG.md b/CHANGELOG.md index fdb7f5303b1..4e81432edd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,51 +1,670 @@ -# ClickHouse 1.1.54370 Release Candidate, 2018-03-16 +## ClickHouse release 18.14.15, 2018-11-21 -## New features: +### Bug fixes: +* The size of memory chunk was overestimated while deserializing the column of type `Array(String)` that leads to "Memory limit exceeded" errors. The issue appeared in version 18.12.13. [#3589](https://github.com/yandex/ClickHouse/issues/3589) + +## ClickHouse release 18.14.14, 2018-11-20 + +### Bug fixes: +* Fixed `ON CLUSTER` queries when cluster configured as secure (flag ``). [#3599](https://github.com/yandex/ClickHouse/pull/3599) + +### Build changes: +* Fixed problems (llvm-7 from system, macos) [#3582](https://github.com/yandex/ClickHouse/pull/3582) + +## ClickHouse release 18.14.11, 2018-10-29 + +### Bug fixes: + +* Fixed the error `Block structure mismatch in UNION stream: different number of columns` in LIMIT queries. [#2156](https://github.com/yandex/ClickHouse/issues/2156) +* Fixed errors when merging data in tables containing arrays inside Nested structures. [#3397](https://github.com/yandex/ClickHouse/pull/3397) +* Fixed incorrect query results if the `merge_tree_uniform_read_distribution` setting is disabled (it is enabled by default). [#3429](https://github.com/yandex/ClickHouse/pull/3429) +* Fixed an error on inserts to a Distributed table in Native format. [#3411](https://github.com/yandex/ClickHouse/issues/3411) + +## ClickHouse release 18.14.10, 2018-10-23 + +* The `compile_expressions` setting (JIT compilation of expressions) is disabled by default. [#3410](https://github.com/yandex/ClickHouse/pull/3410) +* The `enable_optimize_predicate_expression` setting is disabled by default. + +## ClickHouse release 18.14.9, 2018-10-16 + +### New features: + +* The `WITH CUBE` modifier for `GROUP BY` (the alternative syntax `GROUP BY CUBE(...)` is also available). [#3172](https://github.com/yandex/ClickHouse/pull/3172) +* Added the `formatDateTime` function. [Alexandr Krasheninnikov](https://github.com/yandex/ClickHouse/pull/2770) +* Added the `JDBC` table engine and `jdbc` table function (requires installing clickhouse-jdbc-bridge). [Alexandr Krasheninnikov](https://github.com/yandex/ClickHouse/pull/3210) +* Added functions for working with the ISO week number: `toISOWeek`, `toISOYear`, `toStartOfISOYear`, and `toDayOfYear`. [#3146](https://github.com/yandex/ClickHouse/pull/3146) +* Now you can use `Nullable` columns for `MySQL` and `ODBC` tables. [#3362](https://github.com/yandex/ClickHouse/pull/3362) +* Nested data structures can be read as nested objects in `JSONEachRow` format. Added the `input_format_import_nested_json` setting. [Veloman Yunkan](https://github.com/yandex/ClickHouse/pull/3144) +* Parallel processing is available for many `MATERIALIZED VIEW`s when inserting data. See the `parallel_view_processing` setting. [Marek Vavruša](https://github.com/yandex/ClickHouse/pull/3208) +* Added the `SYSTEM FLUSH LOGS` query (forced log flushes to system tables such as `query_log`) [#3321](https://github.com/yandex/ClickHouse/pull/3321) +* Now you can use pre-defined `database` and `table` macros when declaring `Replicated` tables. [#3251](https://github.com/yandex/ClickHouse/pull/3251) +* Added the ability to read `Decimal` type values in engineering notation (indicating powers of ten). [#3153](https://github.com/yandex/ClickHouse/pull/3153) + +### Experimental features: + +* Optimization of the GROUP BY clause for `LowCardinality data types.` [#3138](https://github.com/yandex/ClickHouse/pull/3138) +* Optimized calculation of expressions for `LowCardinality data types.` [#3200](https://github.com/yandex/ClickHouse/pull/3200) + +### Improvements: + +* Significantly reduced memory consumption for requests with `ORDER BY` and `LIMIT`. See the `max_bytes_before_remerge_sort` setting. [#3205](https://github.com/yandex/ClickHouse/pull/3205) +* In the absence of `JOIN` (`LEFT`, `INNER`, ...), `INNER JOIN` is assumed. [#3147](https://github.com/yandex/ClickHouse/pull/3147) +* Qualified asterisks work correctly in queries with `JOIN`. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/3202) +* The `ODBC` table engine correctly chooses the method for quoting identifiers in the SQL dialect of a remote database. [Alexandr Krasheninnikov](https://github.com/yandex/ClickHouse/pull/3210) +* The `compile_expressions` setting (JIT compilation of expressions) is enabled by default. +* Fixed behavior for simultaneous DROP DATABASE/TABLE IF EXISTS and CREATE DATABASE/TABLE IF NOT EXISTS. Previously, a `CREATE DATABASE ... IF NOT EXISTS` query could return the error message "File ... already exists", and the `CREATE TABLE ... IF NOT EXISTS` and `DROP TABLE IF EXISTS` queries could return `Table ... is creating or attaching right now`. [#3101](https://github.com/yandex/ClickHouse/pull/3101) +* LIKE and IN expressions with a constant right half are passed to the remote server when querying from MySQL or ODBC tables. [#3182](https://github.com/yandex/ClickHouse/pull/3182) +* Comparisons with constant expressions in a WHERE clause are passed to the remote server when querying from MySQL and ODBC tables. Previously, only comparisons with constants were passed. [#3182](https://github.com/yandex/ClickHouse/pull/3182) +* Correct calculation of row width in the terminal for `Pretty` formats, including strings with hieroglyphs. [Amos Bird](https://github.com/yandex/ClickHouse/pull/3257). +* `ON CLUSTER` can be specified for `ALTER UPDATE` queries. +* Improved performance for reading data in `JSONEachRow` format. [#3332](https://github.com/yandex/ClickHouse/pull/3332) +* Added synonyms for the `LENGTH` and `CHARACTER_LENGTH` functions for compatibility. The `CONCAT` function is no longer case-sensitive. [#3306](https://github.com/yandex/ClickHouse/pull/3306) +* Added the `TIMESTAMP` synonym for the `DateTime` type. [#3390](https://github.com/yandex/ClickHouse/pull/3390) +* There is always space reserved for query_id in the server logs, even if the log line is not related to a query. This makes it easier to parse server text logs with third-party tools. +* Memory consumption by a query is logged when it exceeds the next level of an integer number of gigabytes. [#3205](https://github.com/yandex/ClickHouse/pull/3205) +* Added compatibility mode for the case when the client library that uses the Native protocol sends fewer columns by mistake than the server expects for the INSERT query. This scenario was possible when using the clickhouse-cpp library. Previously, this scenario caused the server to crash. [#3171](https://github.com/yandex/ClickHouse/pull/3171) +* In a user-defined WHERE expression in `clickhouse-copier`, you can now use a `partition_key` alias (for additional filtering by source table partition). This is useful if the partitioning scheme changes during copying, but only changes slightly. [#3166](https://github.com/yandex/ClickHouse/pull/3166) +* The workflow of the `Kafka` engine has been moved to a background thread pool in order to automatically reduce the speed of data reading at high loads. [Marek Vavruša](https://github.com/yandex/ClickHouse/pull/3215). +* Support for reading `Tuple` and `Nested` values of structures like `struct` in the `Cap'n'Proto format`. [Marek Vavruša](https://github.com/yandex/ClickHouse/pull/3216) +* The list of top-level domains for the `firstSignificantSubdomain` function now includes the domain `biz`. [decaseal](https://github.com/yandex/ClickHouse/pull/3219) +* In the configuration of external dictionaries, `null_value` is interpreted as the value of the default data type. [#3330](https://github.com/yandex/ClickHouse/pull/3330) +* Support for the `intDiv` and `intDivOrZero` functions for `Decimal`. [b48402e8](https://github.com/yandex/ClickHouse/commit/b48402e8712e2b9b151e0eef8193811d433a1264) +* Support for the `Date`, `DateTime`, `UUID`, and `Decimal` types as a key for the `sumMap` aggregate function. [#3281](https://github.com/yandex/ClickHouse/pull/3281) +* Support for the `Decimal` data type in external dictionaries. [#3324](https://github.com/yandex/ClickHouse/pull/3324) +* Support for the `Decimal` data type in `SummingMergeTree` tables. [#3348](https://github.com/yandex/ClickHouse/pull/3348) +* Added specializations for `UUID` in `if`. [#3366](https://github.com/yandex/ClickHouse/pull/3366) +* Reduced the number of `open` and `close` system calls when reading from a `MergeTree table`. [#3283](https://github.com/yandex/ClickHouse/pull/3283) +* A `TRUNCATE TABLE` query can be executed on any replica (the query is passed to the leader replica). [Kirill Shvakov](https://github.com/yandex/ClickHouse/pull/3375) + +### Bug fixes: + +* Fixed an issue with `Dictionary` tables for `range_hashed` dictionaries. This error occurred in version 18.12.17. [#1702](https://github.com/yandex/ClickHouse/pull/1702) +* Fixed an error when loading `range_hashed` dictionaries (the message `Unsupported type Nullable (...)`). This error occurred in version 18.12.17. [#3362](https://github.com/yandex/ClickHouse/pull/3362) +* Fixed errors in the `pointInPolygon` function due to the accumulation of inaccurate calculations for polygons with a large number of vertices located close to each other. [#3331](https://github.com/yandex/ClickHouse/pull/3331) [#3341](https://github.com/yandex/ClickHouse/pull/3341) +* If after merging data parts, the checksum for the resulting part differs from the result of the same merge in another replica, the result of the merge is deleted and the data part is downloaded from the other replica (this is the correct behavior). But after downloading the data part, it couldn't be added to the working set because of an error that the part already exists (because the data part was deleted with some delay after the merge). This led to cyclical attempts to download the same data. [#3194](https://github.com/yandex/ClickHouse/pull/3194) +* Fixed incorrect calculation of total memory consumption by queries (because of incorrect calculation, the `max_memory_usage_for_all_queries` setting worked incorrectly and the `MemoryTracking` metric had an incorrect value). This error occurred in version 18.12.13. [Marek Vavruša](https://github.com/yandex/ClickHouse/pull/3344) +* Fixed the functionality of `CREATE TABLE ... ON CLUSTER ... AS SELECT ...` This error occurred in version 18.12.13. [#3247](https://github.com/yandex/ClickHouse/pull/3247) +* Fixed unnecessary preparation of data structures for `JOIN`s on the server that initiates the request if the `JOIN` is only performed on remote servers. [#3340](https://github.com/yandex/ClickHouse/pull/3340) +* Fixed bugs in the `Kafka` engine: deadlocks after exceptions when starting to read data, and locks upon completion [Marek Vavruša](https://github.com/yandex/ClickHouse/pull/3215). +* For `Kafka` tables, the optional `schema` parameter was not passed (the schema of the `Cap'n'Proto` format). [Vojtech Splichal](https://github.com/yandex/ClickHouse/pull/3150) +* If the ensemble of ZooKeeper servers has servers that accept the connection but then immediately close it instead of responding to the handshake, ClickHouse chooses to connect another server. Previously, this produced the error `Cannot read all data. Bytes read: 0. Bytes expected: 4.` and the server couldn't start. [8218cf3a](https://github.com/yandex/ClickHouse/commit/8218cf3a5f39a43401953769d6d12a0bb8d29da9) +* If the ensemble of ZooKeeper servers contains servers for which the DNS query returns an error, these servers are ignored. [17b8e209](https://github.com/yandex/ClickHouse/commit/17b8e209221061325ad7ba0539f03c6e65f87f29) +* Fixed type conversion between `Date` and `DateTime` when inserting data in the `VALUES` format (if `input_format_values_interpret_expressions = 1`). Previously, the conversion was performed between the numerical value of the number of days in Unix Epoch time and the Unix timestamp, which led to unexpected results. [#3229](https://github.com/yandex/ClickHouse/pull/3229) +* Corrected type conversion between `Decimal` and integer numbers. [#3211](https://github.com/yandex/ClickHouse/pull/3211) +* Fixed errors in the `enable_optimize_predicate_expression` setting. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/3231) +* Fixed a parsing error in CSV format with floating-point numbers if a non-default CSV separator is used, such as `;` [#3155](https://github.com/yandex/ClickHouse/pull/3155) +* Fixed the `arrayCumSumNonNegative` function (it does not accumulate negative values if the accumulator is less than zero). [Aleksey Studnev](https://github.com/yandex/ClickHouse/pull/3163) +* Fixed how `Merge` tables work on top of `Distributed` tables when using `PREWHERE`. [#3165](https://github.com/yandex/ClickHouse/pull/3165) +* Bug fixes in the `ALTER UPDATE` query. +* Fixed bugs in the `odbc` table function that appeared in version 18.12. [#3197](https://github.com/yandex/ClickHouse/pull/3197) +* Fixed the operation of aggregate functions with `StateArray` combinators. [#3188](https://github.com/yandex/ClickHouse/pull/3188) +* Fixed a crash when dividing a `Decimal` value by zero. [69dd6609](https://github.com/yandex/ClickHouse/commit/69dd6609193beb4e7acd3e6ad216eca0ccfb8179) +* Fixed output of types for operations using `Decimal` and integer arguments. [#3224](https://github.com/yandex/ClickHouse/pull/3224) +* Fixed the segfault during `GROUP BY` on `Decimal128`. [3359ba06](https://github.com/yandex/ClickHouse/commit/3359ba06c39fcd05bfdb87d6c64154819621e13a) +* The `log_query_threads` setting (logging information about each thread of query execution) now takes effect only if the `log_queries` option (logging information about queries) is set to 1. Since the `log_query_threads` option is enabled by default, information about threads was previously logged even if query logging was disabled. [#3241](https://github.com/yandex/ClickHouse/pull/3241) +* Fixed an error in the distributed operation of the quantiles aggregate function (the error message `Not found column quantile...`). [292a8855](https://github.com/yandex/ClickHouse/commit/292a885533b8e3b41ce8993867069d14cbd5a664) +* Fixed the compatibility problem when working on a cluster of version 18.12.17 servers and older servers at the same time. For distributed queries with GROUP BY keys of both fixed and non-fixed length, if there was a large amount of data to aggregate, the returned data was not always fully aggregated (two different rows contained the same aggregation keys). [#3254](https://github.com/yandex/ClickHouse/pull/3254) +* Fixed handling of substitutions in `clickhouse-performance-test`, if the query contains only part of the substitutions declared in the test. [#3263](https://github.com/yandex/ClickHouse/pull/3263) +* Fixed an error when using `FINAL` with `PREWHERE`. [#3298](https://github.com/yandex/ClickHouse/pull/3298) +* Fixed an error when using `PREWHERE` over columns that were added during `ALTER`. [#3298](https://github.com/yandex/ClickHouse/pull/3298) +* Added a check for the absence of `arrayJoin` for `DEFAULT` and `MATERIALIZED` expressions. Previously, `arrayJoin` led to an error when inserting data. [#3337](https://github.com/yandex/ClickHouse/pull/3337) +* Added a check for the absence of `arrayJoin` in a `PREWHERE` clause. Previously, this led to messages like `Size ... doesn't match` or `Unknown compression method` when executing queries. [#3357](https://github.com/yandex/ClickHouse/pull/3357) +* Fixed segfault that could occur in rare cases after optimization that replaced AND chains from equality evaluations with the corresponding IN expression. [liuyimin-bytedance](https://github.com/yandex/ClickHouse/pull/3339) +* Minor corrections to `clickhouse-benchmark`: previously, client information was not sent to the server; now the number of queries executed is calculated more accurately when shutting down and for limiting the number of iterations. [#3351](https://github.com/yandex/ClickHouse/pull/3351) [#3352](https://github.com/yandex/ClickHouse/pull/3352) + +### Backward incompatible changes: + +* Removed the `allow_experimental_decimal_type` option. The `Decimal` data type is available for default use. [#3329](https://github.com/yandex/ClickHouse/pull/3329) + +## ClickHouse release 18.12.17, 2018-09-16 + +### New features: + +* `invalidate_query` (the ability to specify a query to check whether an external dictionary needs to be updated) is implemented for the `clickhouse` source. [#3126](https://github.com/yandex/ClickHouse/pull/3126) +* Added the ability to use `UInt*`, `Int*`, and `DateTime` data types (along with the `Date` type) as a `range_hashed` external dictionary key that defines the boundaries of ranges. Now `NULL` can be used to designate an open range. [Vasily Nemkov](https://github.com/yandex/ClickHouse/pull/3123) +* The `Decimal` type now supports `var*` and `stddev*` aggregate functions. [#3129](https://github.com/yandex/ClickHouse/pull/3129) +* The `Decimal` type now supports mathematical functions (`exp`, `sin` and so on.) [#3129](https://github.com/yandex/ClickHouse/pull/3129) +* The `system.part_log` table now has the `partition_id` column. [#3089](https://github.com/yandex/ClickHouse/pull/3089) + +### Bug fixes: + +* `Merge` now works correctly on `Distributed` tables. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/3159) +* Fixed incompatibility (unnecessary dependency on the `glibc` version) that made it impossible to run ClickHouse on `Ubuntu Precise` and older versions. The incompatibility arose in version 18.12.13. [#3130](https://github.com/yandex/ClickHouse/pull/3130) +* Fixed errors in the `enable_optimize_predicate_expression` setting. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/3107) +* Fixed a minor issue with backwards compatibility that appeared when working with a cluster of replicas on versions earlier than 18.12.13 and simultaneously creating a new replica of a table on a server with a newer version (shown in the message `Can not clone replica, because the ... updated to new ClickHouse version`, which is logical, but shouldn't happen). [#3122](https://github.com/yandex/ClickHouse/pull/3122) + +### Backward incompatible changes: + +* The `enable_optimize_predicate_expression` option is enabled by default (which is rather optimistic). If query analysis errors occur that are related to searching for the column names, set `enable_optimize_predicate_expression` to 0. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/3107) + +## ClickHouse release 18.12.14, 2018-09-13 + +### New features: + +* Added support for `ALTER UPDATE` queries. [#3035](https://github.com/yandex/ClickHouse/pull/3035) +* Added the `allow_ddl` option, which restricts the user's access to DDL queries. [#3104](https://github.com/yandex/ClickHouse/pull/3104) +* Added the `min_merge_bytes_to_use_direct_io` option for `MergeTree` engines, which allows you to set a threshold for the total size of the merge (when above the threshold, data part files will be handled using O_DIRECT). [#3117](https://github.com/yandex/ClickHouse/pull/3117) +* The `system.merges` system table now contains the `partition_id` column. [#3099](https://github.com/yandex/ClickHouse/pull/3099) + +### Improvements + +* If a data part remains unchanged during mutation, it isn't downloaded by replicas. [#3103](https://github.com/yandex/ClickHouse/pull/3103) +* Autocomplete is available for names of settings when working with `clickhouse-client`. [#3106](https://github.com/yandex/ClickHouse/pull/3106) + +### Bug fixes: + +* Added a check for the sizes of arrays that are elements of `Nested` type fields when inserting. [#3118](https://github.com/yandex/ClickHouse/pull/3118) +* Fixed an error updating external dictionaries with the `ODBC` source and `hashed` storage. This error occurred in version 18.12.13. +* Fixed a crash when creating a temporary table from a query with an `IN` condition. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/3098) +* Fixed an error in aggregate functions for arrays that can have `NULL` elements. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/3097) + + +## ClickHouse release 18.12.13, 2018-09-10 + +### New features: + +* Added the `DECIMAL(digits, scale)` data type (`Decimal32(scale)`, `Decimal64(scale)`, `Decimal128(scale)`). To enable it, use the setting `allow_experimental_decimal_type`. [#2846](https://github.com/yandex/ClickHouse/pull/2846) [#2970](https://github.com/yandex/ClickHouse/pull/2970) [#3008](https://github.com/yandex/ClickHouse/pull/3008) [#3047](https://github.com/yandex/ClickHouse/pull/3047) +* New `WITH ROLLUP` modifier for `GROUP BY` (alternative syntax: `GROUP BY ROLLUP(...)`). [#2948](https://github.com/yandex/ClickHouse/pull/2948) +* In requests with JOIN, the star character expands to a list of columns in all tables, in compliance with the SQL standard. You can restore the old behavior by setting `asterisk_left_columns_only` to 1 on the user configuration level. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2787) +* Added support for JOIN with table functions. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2907) +* Autocomplete by pressing Tab in clickhouse-client. [Sergey Shcherbin](https://github.com/yandex/ClickHouse/pull/2447) +* Ctrl+C in clickhouse-client clears a query that was entered. [#2877](https://github.com/yandex/ClickHouse/pull/2877) +* Added the `join_default_strictness` setting (values: `"`, `'any'`, `'all'`). This allows you to not specify `ANY` or `ALL` for `JOIN`. [#2982](https://github.com/yandex/ClickHouse/pull/2982) +* Each line of the server log related to query processing shows the query ID. [#2482](https://github.com/yandex/ClickHouse/pull/2482) +* Now you can get query execution logs in clickhouse-client (use the `send_logs_level` setting). With distributed query processing, logs are cascaded from all the servers. [#2482](https://github.com/yandex/ClickHouse/pull/2482) +* The `system.query_log` and `system.processes` (`SHOW PROCESSLIST`) tables now have information about all changed settings when you run a query (the nested structure of the `Settings` data). Added the `log_query_settings` setting. [#2482](https://github.com/yandex/ClickHouse/pull/2482) +* The `system.query_log` and `system.processes` tables now show information about the number of threads that are participating in query execution (see the `thread_numbers` column). [#2482](https://github.com/yandex/ClickHouse/pull/2482) +* Added `ProfileEvents` counters that measure the time spent on reading and writing over the network and reading and writing to disk, the number of network errors, and the time spent waiting when network bandwidth is limited. [#2482](https://github.com/yandex/ClickHouse/pull/2482) +* Added `ProfileEvents`counters that contain the system metrics from rusage (you can use them to get information about CPU usage in userspace and the kernel, page faults, and context switches), as well as taskstats metrics (use these to obtain information about I/O wait time, CPU wait time, and the amount of data read and recorded, both with and without page cache). [#2482](https://github.com/yandex/ClickHouse/pull/2482) +* The `ProfileEvents` counters are applied globally and for each query, as well as for each query execution thread, which allows you to profile resource consumption by query in detail. [#2482](https://github.com/yandex/ClickHouse/pull/2482) +* Added the `system.query_thread_log` table, which contains information about each query execution thread. Added the `log_query_threads` setting. [#2482](https://github.com/yandex/ClickHouse/pull/2482) +* The `system.metrics` and `system.events` tables now have built-in documentation. [#3016](https://github.com/yandex/ClickHouse/pull/3016) +* Added the `arrayEnumerateDense` function. [Amos Bird](https://github.com/yandex/ClickHouse/pull/2975) +* Added the `arrayCumSumNonNegative` and `arrayDifference` functions. [Aleksey Studnev](https://github.com/yandex/ClickHouse/pull/2942) +* Added the `retention` aggregate function. [Sundy Li](https://github.com/yandex/ClickHouse/pull/2887) +* Now you can add (merge) states of aggregate functions by using the plus operator, and multiply the states of aggregate functions by a nonnegative constant. [#3062](https://github.com/yandex/ClickHouse/pull/3062) [#3034](https://github.com/yandex/ClickHouse/pull/3034) +* Tables in the MergeTree family now have the virtual column `_partition_id`. [#3089](https://github.com/yandex/ClickHouse/pull/3089) + +### Experimental features: + +* Added the `LowCardinality(T)` data type. This data type automatically creates a local dictionary of values and allows data processing without unpacking the dictionary. [#2830](https://github.com/yandex/ClickHouse/pull/2830) +* Added a cache of JIT-compiled functions and a counter for the number of uses before compiling. To JIT compile expressions, enable the `compile_expressions` setting. [#2990](https://github.com/yandex/ClickHouse/pull/2990) [#3077](https://github.com/yandex/ClickHouse/pull/3077) + +### Improvements: + +* Fixed the problem with unlimited accumulation of the replication log when there are abandoned replicas. Added an effective recovery mode for replicas with a long lag. +* Improved performance of `GROUP BY` with multiple aggregation fields when one of them is string and the others are fixed length. +* Improved performance when using `PREWHERE` and with implicit transfer of expressions in `PREWHERE`. +* Improved parsing performance for text formats (`CSV`, `TSV`). [Amos Bird](https://github.com/yandex/ClickHouse/pull/2977) [#2980](https://github.com/yandex/ClickHouse/pull/2980) +* Improved performance of reading strings and arrays in binary formats. [Amos Bird](https://github.com/yandex/ClickHouse/pull/2955) +* Increased performance and reduced memory consumption for queries to `system.tables` and `system.columns` when there is a very large number of tables on a single server. [#2953](https://github.com/yandex/ClickHouse/pull/2953) +* Fixed a performance problem in the case of a large stream of queries that result in an error (the ` _dl_addr` function is visible in `perf top`, but the server isn't using much CPU). [#2938](https://github.com/yandex/ClickHouse/pull/2938) +* Conditions are cast into the View (when `enable_optimize_predicate_expression` is enabled). [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2907) +* Improvements to the functionality for the `UUID` data type. [#3074](https://github.com/yandex/ClickHouse/pull/3074) [#2985](https://github.com/yandex/ClickHouse/pull/2985) +* The `UUID` data type is supported in The-Alchemist dictionaries. [#2822](https://github.com/yandex/ClickHouse/pull/2822) +* The `visitParamExtractRaw` function works correctly with nested structures. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2974) +* When the `input_format_skip_unknown_fields` setting is enabled, object fields in `JSONEachRow` format are skipped correctly. [BlahGeek](https://github.com/yandex/ClickHouse/pull/2958) +* For a `CASE` expression with conditions, you can now omit `ELSE`, which is equivalent to `ELSE NULL`. [#2920](https://github.com/yandex/ClickHouse/pull/2920) +* The operation timeout can now be configured when working with ZooKeeper. [urykhy](https://github.com/yandex/ClickHouse/pull/2971) +* You can specify an offset for `LIMIT n, m` as `LIMIT n OFFSET m`. [#2840](https://github.com/yandex/ClickHouse/pull/2840) +* You can use the `SELECT TOP n` syntax as an alternative for `LIMIT`. [#2840](https://github.com/yandex/ClickHouse/pull/2840) +* Increased the size of the queue to write to system tables, so the `SystemLog parameter queue is full` error doesn't happen as often. +* The `windowFunnel` aggregate function now supports events that meet multiple conditions. [Amos Bird](https://github.com/yandex/ClickHouse/pull/2801) +* Duplicate columns can be used in a `USING` clause for `JOIN`. [#3006](https://github.com/yandex/ClickHouse/pull/3006) +* `Pretty` formats now have a limit on column alignment by width. Use the `output_format_pretty_max_column_pad_width` setting. If a value is wider, it will still be displayed in its entirety, but the other cells in the table will not be too wide. [#3003](https://github.com/yandex/ClickHouse/pull/3003) +* The `odbc` table function now allows you to specify the database/schema name. [Amos Bird](https://github.com/yandex/ClickHouse/pull/2885) +* Added the ability to use a username specified in the `clickhouse-client` config file. [Vladimir Kozbin](https://github.com/yandex/ClickHouse/pull/2909) +* The `ZooKeeperExceptions` counter has been split into three counters: `ZooKeeperUserExceptions`, `ZooKeeperHardwareExceptions`, and `ZooKeeperOtherExceptions`. +* `ALTER DELETE` queries work for materialized views. +* Added randomization when running the cleanup thread periodically for `ReplicatedMergeTree` tables in order to avoid periodic load spikes when there are a very large number of `ReplicatedMergeTree` tables. +* Support for `ATTACH TABLE ... ON CLUSTER` queries. [#3025](https://github.com/yandex/ClickHouse/pull/3025) + +### Bug fixes: + +* Fixed an issue with `Dictionary` tables (throws the `Size of offsets doesn't match size of column` or `Unknown compression method` exception). This bug appeared in version 18.10.3. [#2913](https://github.com/yandex/ClickHouse/issues/2913) +* Fixed a bug when merging `CollapsingMergeTree` tables if one of the data parts is empty (these parts are formed during merge or `ALTER DELETE` if all data was deleted), and the `vertical` algorithm was used for the merge. [#3049](https://github.com/yandex/ClickHouse/pull/3049) +* Fixed a race condition during `DROP` or `TRUNCATE` for `Memory` tables with a simultaneous `SELECT`, which could lead to server crashes. This bug appeared in version 1.1.54388. [#3038](https://github.com/yandex/ClickHouse/pull/3038) +* Fixed the possibility of data loss when inserting in `Replicated` tables if the `Session is expired` error is returned (data loss can be detected by the `ReplicatedDataLoss` metric). This error occurred in version 1.1.54378. [#2939](https://github.com/yandex/ClickHouse/pull/2939) [#2949](https://github.com/yandex/ClickHouse/pull/2949) [#2964](https://github.com/yandex/ClickHouse/pull/2964) +* Fixed a segfault during `JOIN ... ON`. [#3000](https://github.com/yandex/ClickHouse/pull/3000) +* Fixed the error searching column names when the `WHERE` expression consists entirely of a qualified column name, such as `WHERE table.column`. [#2994](https://github.com/yandex/ClickHouse/pull/2994) +* Fixed the "Not found column" error that occurred when executing distributed queries if a single column consisting of an IN expression with a subquery is requested from a remote server. [#3087](https://github.com/yandex/ClickHouse/pull/3087) +* Fixed the `Block structure mismatch in UNION stream: different number of columns` error that occurred for distributed queries if one of the shards is local and the other is not, and optimization of the move to `PREWHERE` is triggered. [#2226](https://github.com/yandex/ClickHouse/pull/2226) [#3037](https://github.com/yandex/ClickHouse/pull/3037) [#3055](https://github.com/yandex/ClickHouse/pull/3055) [#3065](https://github.com/yandex/ClickHouse/pull/3065) [#3073](https://github.com/yandex/ClickHouse/pull/3073) [#3090](https://github.com/yandex/ClickHouse/pull/3090) [#3093](https://github.com/yandex/ClickHouse/pull/3093) +* Fixed the `pointInPolygon` function for certain cases of non-convex polygons. [#2910](https://github.com/yandex/ClickHouse/pull/2910) +* Fixed the incorrect result when comparing `nan` with integers. [#3024](https://github.com/yandex/ClickHouse/pull/3024) +* Fixed an error in the `zlib-ng` library that could lead to segfault in rare cases. [#2854](https://github.com/yandex/ClickHouse/pull/2854) +* Fixed a memory leak when inserting into a table with `AggregateFunction` columns, if the state of the aggregate function is not simple (allocates memory separately), and if a single insertion request results in multiple small blocks. [#3084](https://github.com/yandex/ClickHouse/pull/3084) +* Fixed a race condition when creating and deleting the same `Buffer` or `MergeTree` table simultaneously. +* Fixed the possibility of a segfault when comparing tuples made up of certain non-trivial types, such as tuples. [#2989](https://github.com/yandex/ClickHouse/pull/2989) +* Fixed the possibility of a segfault when running certain `ON CLUSTER` queries. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2960) +* Fixed an error in the `arrayDistinct` function for `Nullable` array elements. [#2845](https://github.com/yandex/ClickHouse/pull/2845) [#2937](https://github.com/yandex/ClickHouse/pull/2937) +* The `enable_optimize_predicate_expression` option now correctly supports cases with `SELECT *`. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2929) +* Fixed the segfault when re-initializing the ZooKeeper session. [#2917](https://github.com/yandex/ClickHouse/pull/2917) +* Fixed potential blocking when working with ZooKeeper. +* Fixed incorrect code for adding nested data structures in a `SummingMergeTree`. +* When allocating memory for states of aggregate functions, alignment is correctly taken into account, which makes it possible to use operations that require alignment when implementing states of aggregate functions. [chenxing-xc](https://github.com/yandex/ClickHouse/pull/2808) + +### Security fix: + +* Safe use of ODBC data sources. Interaction with ODBC drivers uses a separate `clickhouse-odbc-bridge` process. Errors in third-party ODBC drivers no longer cause problems with server stability or vulnerabilities. [#2828](https://github.com/yandex/ClickHouse/pull/2828) [#2879](https://github.com/yandex/ClickHouse/pull/2879) [#2886](https://github.com/yandex/ClickHouse/pull/2886) [#2893](https://github.com/yandex/ClickHouse/pull/2893) [#2921](https://github.com/yandex/ClickHouse/pull/2921) +* Fixed incorrect validation of the file path in the `catBoostPool` table function. [#2894](https://github.com/yandex/ClickHouse/pull/2894) +* The contents of system tables (`tables`, `databases`, `parts`, `columns`, `parts_columns`, `merges`, `mutations`, `replicas`, and `replication_queue`) are filtered according to the user's configured access to databases (`allow_databases`). [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2856) + +### Backward incompatible changes: + +* In requests with JOIN, the star character expands to a list of columns in all tables, in compliance with the SQL standard. You can restore the old behavior by setting `asterisk_left_columns_only` to 1 on the user configuration level. + +### Build changes: + +* Most integration tests can now be run by commit. +* Code style checks can also be run by commit. +* The `memcpy` implementation is chosen correctly when building on CentOS7/Fedora. [Etienne Champetier](https://github.com/yandex/ClickHouse/pull/2912) +* When using clang to build, some warnings from `-Weverything` have been added, in addition to the regular `-Wall-Wextra -Werror`. [#2957](https://github.com/yandex/ClickHouse/pull/2957) +* Debugging the build uses the `jemalloc` debug option. +* The interface of the library for interacting with ZooKeeper is declared abstract. [#2950](https://github.com/yandex/ClickHouse/pull/2950) + +## ClickHouse release 18.10.3, 2018-08-13 + +### New features: + +* HTTPS can be used for replication. [#2760](https://github.com/yandex/ClickHouse/pull/2760) +* Added the functions `murmurHash2_64`, `murmurHash3_32`, `murmurHash3_64`, and `murmurHash3_128` in addition to the existing `murmurHash2_32`. [#2791](https://github.com/yandex/ClickHouse/pull/2791) +* Support for Nullable types in the ClickHouse ODBC driver (`ODBCDriver2` output format). [#2834](https://github.com/yandex/ClickHouse/pull/2834) +* Support for `UUID` in the key columns. + +### Improvements: + +* Clusters can be removed without restarting the server when they are deleted from the config files. [#2777](https://github.com/yandex/ClickHouse/pull/2777) +* External dictionaries can be removed without restarting the server when they are removed from config files. [#2779](https://github.com/yandex/ClickHouse/pull/2779) +* Added `SETTINGS` support for the `Kafka` table engine. [Alexander Marshalov](https://github.com/yandex/ClickHouse/pull/2781) +* Improvements for the `UUID` data type (not yet complete). [#2618](https://github.com/yandex/ClickHouse/pull/2618) +* Support for empty parts after merges in the `SummingMergeTree`, `CollapsingMergeTree` and `VersionedCollapsingMergeTree` engines. [#2815](https://github.com/yandex/ClickHouse/pull/2815) +* Old records of completed mutations are deleted (`ALTER DELETE`). [#2784](https://github.com/yandex/ClickHouse/pull/2784) +* Added the `system.merge_tree_settings` table. [Kirill Shvakov](https://github.com/yandex/ClickHouse/pull/2841) +* The `system.tables` table now has dependency columns: `dependencies_database` and `dependencies_table`. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2851) +* Added the `max_partition_size_to_drop` config option. [#2782](https://github.com/yandex/ClickHouse/pull/2782) +* Added the `output_format_json_escape_forward_slashes` option. [Alexander Bocharov](https://github.com/yandex/ClickHouse/pull/2812) +* Added the `max_fetch_partition_retries_count` setting. [#2831](https://github.com/yandex/ClickHouse/pull/2831) +* Added the `prefer_localhost_replica` setting for disabling the preference for a local replica and going to a local replica without inter-process interaction. [#2832](https://github.com/yandex/ClickHouse/pull/2832) +* The `quantileExact` aggregate function returns `nan` in the case of aggregation on an empty `Float32` or `Float64` set. [Sundy Li](https://github.com/yandex/ClickHouse/pull/2855) + +### Bug fixes: + +* Removed unnecessary escaping of the connection string parameters for ODBC, which made it impossible to establish a connection. This error occurred in version 18.6.0. +* Fixed the logic for processing `REPLACE PARTITION` commands in the replication queue. If there are two `REPLACE` commands for the same partition, the incorrect logic could cause one of them to remain in the replication queue and not be executed. [#2814](https://github.com/yandex/ClickHouse/pull/2814) +* Fixed a merge bug when all data parts were empty (parts that were formed from a merge or from `ALTER DELETE` if all data was deleted). This bug appeared in version 18.1.0. [#2930](https://github.com/yandex/ClickHouse/pull/2930) +* Fixed an error for concurrent `Set` or `Join`. [Amos Bird](https://github.com/yandex/ClickHouse/pull/2823) +* Fixed the `Block structure mismatch in UNION stream: different number of columns` error that occurred for `UNION ALL` queries inside a sub-query if one of the `SELECT` queries contains duplicate column names. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2094) +* Fixed a memory leak if an exception occurred when connecting to a MySQL server. +* Fixed incorrect clickhouse-client response code in case of a request error. +* Fixed incorrect behavior of materialized views containing DISTINCT. [#2795](https://github.com/yandex/ClickHouse/issues/2795) + +### Backward incompatible changes + +* Removed support for CHECK TABLE queries for Distributed tables. + +### Build changes: + +* The allocator has been replaced: `jemalloc` is now used instead of `tcmalloc`. In some scenarios, this increases speed up to 20%. However, there are queries that have slowed by up to 20%. Memory consumption has been reduced by approximately 10% in some scenarios, with improved stability. With highly competitive loads, CPU usage in userspace and in system shows just a slight increase. [#2773](https://github.com/yandex/ClickHouse/pull/2773) +* Use of libressl from a submodule. [#1983](https://github.com/yandex/ClickHouse/pull/1983) [#2807](https://github.com/yandex/ClickHouse/pull/2807) +* Use of unixodbc from a submodule. [#2789](https://github.com/yandex/ClickHouse/pull/2789) +* Use of mariadb-connector-c from a submodule. [#2785](https://github.com/yandex/ClickHouse/pull/2785) +* Added functional test files to the repository that depend on the availability of test data (for the time being, without the test data itself). + +## ClickHouse release 18.6.0, 2018-08-02 + +### New features: + +* Added support for ON expressions for the JOIN ON syntax: +`JOIN ON Expr([table.]column ...) = Expr([table.]column, ...) [AND Expr([table.]column, ...) = Expr([table.]column, ...) ...]` +The expression must be a chain of equalities joined by the AND operator. Each side of the equality can be an arbitrary expression over the columns of one of the tables. The use of fully qualified column names is supported (`table.name`, `database.table.name`, `table_alias.name`, `subquery_alias.name`) for the right table. [#2742](https://github.com/yandex/ClickHouse/pull/2742) +* HTTPS can be enabled for replication. [#2760](https://github.com/yandex/ClickHouse/pull/2760) + +### Improvements: + +* The server passes the patch component of its version to the client. Data about the patch version component is in `system.processes` and `query_log`. [#2646](https://github.com/yandex/ClickHouse/pull/2646) + +## ClickHouse release 18.5.1, 2018-07-31 + +### New features: + +* Added the hash function `murmurHash2_32` [#2756](https://github.com/yandex/ClickHouse/pull/2756). + +### Improvements: + +* Now you can use the `from_env` [#2741](https://github.com/yandex/ClickHouse/pull/2741) attribute to set values in config files from environment variables. +* Added case-insensitive versions of the `coalesce`, `ifNull`, and `nullIf functions` [#2752](https://github.com/yandex/ClickHouse/pull/2752). + +### Bug fixes: + +* Fixed a possible bug when starting a replica [#2759](https://github.com/yandex/ClickHouse/pull/2759). + +## ClickHouse release 18.4.0, 2018-07-28 + +### New features: + +* Added system tables: `formats`, `data_type_families`, `aggregate_function_combinators`, `table_functions`, `table_engines`, `collations` [#2721](https://github.com/yandex/ClickHouse/pull/2721). +* Added the ability to use a table function instead of a table as an argument of a `remote` or `cluster table function` [#2708](https://github.com/yandex/ClickHouse/pull/2708). +* Support for `HTTP Basic` authentication in the replication protocol [#2727](https://github.com/yandex/ClickHouse/pull/2727). +* The `has` function now allows searching for a numeric value in an array of `Enum` values [Maxim Khrisanfov](https://github.com/yandex/ClickHouse/pull/2699). +* Support for adding arbitrary message separators when reading from `Kafka` [Amos Bird](https://github.com/yandex/ClickHouse/pull/2701). + +### Improvements: + +* The `ALTER TABLE t DELETE WHERE` query does not rewrite data parts that were not affected by the WHERE condition [#2694](https://github.com/yandex/ClickHouse/pull/2694). +* The `use_minimalistic_checksums_in_zookeeper` option for `ReplicatedMergeTree` tables is enabled by default. This setting was added in version 1.1.54378, 2018-04-16. Versions that are older than 1.1.54378 can no longer be installed. +* Support for running `KILL` and `OPTIMIZE` queries that specify `ON CLUSTER` [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2689). + +### Bug fixes: + +* Fixed the error `Column ... is not under an aggregate function and not in GROUP BY` for aggregation with an IN expression. This bug appeared in version 18.1.0. ([bbdd780b](https://github.com/yandex/ClickHouse/commit/bbdd780be0be06a0f336775941cdd536878dd2c2)) +* Fixed a bug in the `windowFunnel aggregate function` [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2735). +* Fixed a bug in the `anyHeavy` aggregate function ([a2101df2](https://github.com/yandex/ClickHouse/commit/a2101df25a6a0fba99aa71f8793d762af2b801ee)) +* Fixed server crash when using the `countArray()` aggregate function. + +### Backward incompatible changes: + +* Parameters for `Kafka` engine was changed from `Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format[, kafka_schema, kafka_num_consumers])` to `Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format[, kafka_row_delimiter, kafka_schema, kafka_num_consumers])`. If your tables use `kafka_schema` or `kafka_num_consumers` parameters, you have to manually edit the metadata files `path/metadata/database/table.sql` and add `kafka_row_delimiter` parameter with `''` value. + +## ClickHouse release 18.1.0, 2018-07-23 + +### New features: + +* Support for the `ALTER TABLE t DELETE WHERE` query for non-replicated MergeTree tables ([#2634](https://github.com/yandex/ClickHouse/pull/2634)). +* Support for arbitrary types for the `uniq*` family of aggregate functions ([#2010](https://github.com/yandex/ClickHouse/issues/2010)). +* Support for arbitrary types in comparison operators ([#2026](https://github.com/yandex/ClickHouse/issues/2026)). +* The `users.xml` file allows setting a subnet mask in the format `10.0.0.1/255.255.255.0`. This is necessary for using masks for IPv6 networks with zeros in the middle ([#2637](https://github.com/yandex/ClickHouse/pull/2637)). +* Added the `arrayDistinct` function ([#2670](https://github.com/yandex/ClickHouse/pull/2670)). +* The SummingMergeTree engine can now work with AggregateFunction type columns ([Constantin S. Pan](https://github.com/yandex/ClickHouse/pull/2566)). + +### Improvements: + +* Changed the numbering scheme for release versions. Now the first part contains the year of release (A.D., Moscow timezone, minus 2000), the second part contains the number for major changes (increases for most releases), and the third part is the patch version. Releases are still backwards compatible, unless otherwise stated in the changelog. +* Faster conversions of floating-point numbers to a string ([Amos Bird](https://github.com/yandex/ClickHouse/pull/2664)). +* If some rows were skipped during an insert due to parsing errors (this is possible with the `input_allow_errors_num` and `input_allow_errors_ratio` settings enabled), the number of skipped rows is now written to the server log ([Leonardo Cecchi](https://github.com/yandex/ClickHouse/pull/2669)). + +### Bug fixes: + +* Fixed the TRUNCATE command for temporary tables ([Amos Bird](https://github.com/yandex/ClickHouse/pull/2624)). +* Fixed a rare deadlock in the ZooKeeper client library that occurred when there was a network error while reading the response ([c315200](https://github.com/yandex/ClickHouse/commit/c315200e64b87e44bdf740707fc857d1fdf7e947)). +* Fixed an error during a CAST to Nullable types ([#1322](https://github.com/yandex/ClickHouse/issues/1322)). +* Fixed the incorrect result of the `maxIntersection()` function when the boundaries of intervals coincided ([Michael Furmur](https://github.com/yandex/ClickHouse/pull/2657)). +* Fixed incorrect transformation of the OR expression chain in a function argument ([chenxing-xc](https://github.com/yandex/ClickHouse/pull/2663)). +* Fixed performance degradation for queries containing `IN (subquery)` expressions inside another subquery ([#2571](https://github.com/yandex/ClickHouse/issues/2571)). +* Fixed incompatibility between servers with different versions in distributed queries that use a `CAST` function that isn't in uppercase letters ([fe8c4d6](https://github.com/yandex/ClickHouse/commit/fe8c4d64e434cacd4ceef34faa9005129f2190a5)). +* Added missing quoting of identifiers for queries to an external DBMS ([#2635](https://github.com/yandex/ClickHouse/issues/2635)). + +### Backward incompatible changes: + +* Converting a string containing the number zero to DateTime does not work. Example: `SELECT toDateTime('0')`. This is also the reason that `DateTime DEFAULT '0'` does not work in tables, as well as `0` in dictionaries. Solution: replace `0` with `0000-00-00 00:00:00`. + +## ClickHouse release 1.1.54394, 2018-07-12 + +### New features: + +* Added the `histogram` aggregate function ([Mikhail Surin](https://github.com/yandex/ClickHouse/pull/2521)). +* Now `OPTIMIZE TABLE ... FINAL` can be used without specifying partitions for `ReplicatedMergeTree` ([Amos Bird](https://github.com/yandex/ClickHouse/pull/2600)). + +### Bug fixes: + +* Fixed a problem with a very small timeout for sockets (one second) for reading and writing when sending and downloading replicated data, which made it impossible to download larger parts if there is a load on the network or disk (it resulted in cyclical attempts to download parts). This error occurred in version 1.1.54388. +* Fixed issues when using chroot in ZooKeeper if you inserted duplicate data blocks in the table. +* The `has` function now works correctly for an array with Nullable elements ([#2115](https://github.com/yandex/ClickHouse/issues/2115)). +* The `system.tables` table now works correctly when used in distributed queries. The `metadata_modification_time` and `engine_full` columns are now non-virtual. Fixed an error that occurred if only these columns were requested from the table. +* Fixed how an empty `TinyLog` table works after inserting an empty data block ([#2563](https://github.com/yandex/ClickHouse/issues/2563)). +* The `system.zookeeper` table works if the value of the node in ZooKeeper is NULL. + +## ClickHouse release 1.1.54390, 2018-07-06 + +### New features: + +* Queries can be sent in `multipart/form-data` format (in the `query` field), which is useful if external data is also sent for query processing ([Olga Hvostikova](https://github.com/yandex/ClickHouse/pull/2490)). +* Added the ability to enable or disable processing single or double quotes when reading data in CSV format. You can configure this in the `format_csv_allow_single_quotes` and `format_csv_allow_double_quotes` settings ([Amos Bird](https://github.com/yandex/ClickHouse/pull/2574)). +* Now `OPTIMIZE TABLE ... FINAL` can be used without specifying the partition for non-replicated variants of `MergeTree` ([Amos Bird](https://github.com/yandex/ClickHouse/pull/2599)). + +### Improvements: + +* Improved performance, reduced memory consumption, and correct memory consumption tracking with use of the IN operator when a table index could be used ([#2584](https://github.com/yandex/ClickHouse/pull/2584)). +* Removed redundant checking of checksums when adding a data part. This is important when there are a large number of replicas, because in these cases the total number of checks was equal to N^2. +* Added support for `Array(Tuple(...))` arguments for the `arrayEnumerateUniq` function ([#2573](https://github.com/yandex/ClickHouse/pull/2573)). +* Added `Nullable` support for the `runningDifference` function ([#2594](https://github.com/yandex/ClickHouse/pull/2594)). +* Improved query analysis performance when there is a very large number of expressions ([#2572](https://github.com/yandex/ClickHouse/pull/2572)). +* Faster selection of data parts for merging in `ReplicatedMergeTree` tables. Faster recovery of the ZooKeeper session ([#2597](https://github.com/yandex/ClickHouse/pull/2597)). +* The `format_version.txt` file for `MergeTree` tables is re-created if it is missing, which makes sense if ClickHouse is launched after copying the directory structure without files ([Ciprian Hacman](https://github.com/yandex/ClickHouse/pull/2593)). + +### Bug fixes: + +* Fixed a bug when working with ZooKeeper that could make it impossible to recover the session and readonly states of tables before restarting the server. +* Fixed a bug when working with ZooKeeper that could result in old nodes not being deleted if the session is interrupted. +* Fixed an error in the `quantileTDigest` function for Float arguments (this bug was introduced in version 1.1.54388) ([Mikhail Surin](https://github.com/yandex/ClickHouse/pull/2553)). +* Fixed a bug in the index for MergeTree tables if the primary key column is located inside the function for converting types between signed and unsigned integers of the same size ([#2603](https://github.com/yandex/ClickHouse/pull/2603)). +* Fixed segfault if `macros` are used but they aren't in the config file ([#2570](https://github.com/yandex/ClickHouse/pull/2570)). +* Fixed switching to the default database when reconnecting the client ([#2583](https://github.com/yandex/ClickHouse/pull/2583)). +* Fixed a bug that occurred when the `use_index_for_in_with_subqueries` setting was disabled. + +### Security fix: + +* Sending files is no longer possible when connected to MySQL (`LOAD DATA LOCAL INFILE`). + +## ClickHouse release 1.1.54388, 2018-06-28 + +### New features: + +* Support for the `ALTER TABLE t DELETE WHERE` query for replicated tables. Added the `system.mutations` table to track progress of this type of queries. +* Support for the `ALTER TABLE t [REPLACE|ATTACH] PARTITION` query for \*MergeTree tables. +* Support for the `TRUNCATE TABLE` query ([Winter Zhang](https://github.com/yandex/ClickHouse/pull/2260)) +* Several new `SYSTEM` queries for replicated tables (`RESTART REPLICAS`, `SYNC REPLICA`, `[STOP|START] [MERGES|FETCHES|SENDS REPLICATED|REPLICATION QUEUES]`). +* Added the ability to write to a table with the MySQL engine and the corresponding table function ([sundy-li](https://github.com/yandex/ClickHouse/pull/2294)). +* Added the `url()` table function and the `URL` table engine ([Alexander Sapin](https://github.com/yandex/ClickHouse/pull/2501)). +* Added the `windowFunnel` aggregate function ([sundy-li](https://github.com/yandex/ClickHouse/pull/2352)). +* New `startsWith` and `endsWith` functions for strings ([Vadim Plakhtinsky](https://github.com/yandex/ClickHouse/pull/2429)). +* The `numbers()` table function now allows you to specify the offset ([Winter Zhang](https://github.com/yandex/ClickHouse/pull/2535)). +* The password to `clickhouse-client` can be entered interactively. +* Server logs can now be sent to syslog ([Alexander Krasheninnikov](https://github.com/yandex/ClickHouse/pull/2459)). +* Support for logging in dictionaries with a shared library source ([Alexander Sapin](https://github.com/yandex/ClickHouse/pull/2472)). +* Support for custom CSV delimiters ([Ivan Zhukov](https://github.com/yandex/ClickHouse/pull/2263)) +* Added the `date_time_input_format` setting. If you switch this setting to `'best_effort'`, DateTime values will be read in a wide range of formats. +* Added the `clickhouse-obfuscator` utility for data obfuscation. Usage example: publishing data used in performance tests. + +### Experimental features: + +* Added the ability to calculate `and` arguments only where they are needed ([Anastasia Tsarkova](https://github.com/yandex/ClickHouse/pull/2272)) +* JIT compilation to native code is now available for some expressions ([pyos](https://github.com/yandex/ClickHouse/pull/2277)). + +### Bug fixes: + +* Duplicates no longer appear for a query with `DISTINCT` and `ORDER BY`. +* Queries with `ARRAY JOIN` and `arrayFilter` no longer return an incorrect result. +* Fixed an error when reading an array column from a Nested structure ([#2066](https://github.com/yandex/ClickHouse/issues/2066)). +* Fixed an error when analyzing queries with a HAVING clause like `HAVING tuple IN (...)`. +* Fixed an error when analyzing queries with recursive aliases. +* Fixed an error when reading from ReplacingMergeTree with a condition in PREWHERE that filters all rows ([#2525](https://github.com/yandex/ClickHouse/issues/2525)). +* User profile settings were not applied when using sessions in the HTTP interface. +* Fixed how settings are applied from the command line parameters in clickhouse-local. +* The ZooKeeper client library now uses the session timeout received from the server. +* Fixed a bug in the ZooKeeper client library when the client waited for the server response longer than the timeout. +* Fixed pruning of parts for queries with conditions on partition key columns ([#2342](https://github.com/yandex/ClickHouse/issues/2342)). +* Merges are now possible after `CLEAR COLUMN IN PARTITION` ([#2315](https://github.com/yandex/ClickHouse/issues/2315)). +* Type mapping in the ODBC table function has been fixed ([sundy-li](https://github.com/yandex/ClickHouse/pull/2268)). +* Type comparisons have been fixed for `DateTime` with and without the time zone ([Alexander Bocharov](https://github.com/yandex/ClickHouse/pull/2400)). +* Fixed syntactic parsing and formatting of the `CAST` operator. +* Fixed insertion into a materialized view for the Distributed table engine ([Babacar Diassé](https://github.com/yandex/ClickHouse/pull/2411)). +* Fixed a race condition when writing data from the `Kafka` engine to materialized views ([Yangkuan Liu](https://github.com/yandex/ClickHouse/pull/2448)). +* Fixed SSRF in the remote() table function. +* Fixed exit behavior of `clickhouse-client` in multiline mode ([#2510](https://github.com/yandex/ClickHouse/issues/2510)). + +### Improvements: + +* Background tasks in replicated tables are now performed in a thread pool instead of in separate threads ([Silviu Caragea](https://github.com/yandex/ClickHouse/pull/1722)). +* Improved LZ4 compression performance. +* Faster analysis for queries with a large number of JOINs and sub-queries. +* The DNS cache is now updated automatically when there are too many network errors. +* Table inserts no longer occur if the insert into one of the materialized views is not possible because it has too many parts. +* Corrected the discrepancy in the event counters `Query`, `SelectQuery`, and `InsertQuery`. +* Expressions like `tuple IN (SELECT tuple)` are allowed if the tuple types match. +* A server with replicated tables can start even if you haven't configured ZooKeeper. +* When calculating the number of available CPU cores, limits on cgroups are now taken into account ([Atri Sharma](https://github.com/yandex/ClickHouse/pull/2325)). +* Added chown for config directories in the systemd config file ([Mikhail Shiryaev](https://github.com/yandex/ClickHouse/pull/2421)). + +### Build changes: + +* The gcc8 compiler can be used for builds. +* Added the ability to build llvm from submodule. +* The version of the librdkafka library has been updated to v0.11.4. +* Added the ability to use the system libcpuid library. The library version has been updated to 0.4.0. +* Fixed the build using the vectorclass library ([Babacar Diassé](https://github.com/yandex/ClickHouse/pull/2274)). +* Cmake now generates files for ninja by default (like when using `-G Ninja`). +* Added the ability to use the libtinfo library instead of libtermcap ([Georgy Kondratiev](https://github.com/yandex/ClickHouse/pull/2519)). +* Fixed a header file conflict in Fedora Rawhide ([#2520](https://github.com/yandex/ClickHouse/issues/2520)). + +### Backward incompatible changes: + +* Removed escaping in `Vertical` and `Pretty*` formats and deleted the `VerticalRaw` format. +* If servers with version 1.1.54388 (or newer) and servers with an older version are used simultaneously in a distributed query and the query has the `cast(x, 'Type')` expression without the `AS` keyword and doesn't have the word `cast` in uppercase, an exception will be thrown with a message like `Not found column cast(0, 'UInt8') in block`. Solution: Update the server on the entire cluster. + +## ClickHouse release 1.1.54385, 2018-06-01 + +### Bug fixes: + +* Fixed an error that in some cases caused ZooKeeper operations to block. + +## ClickHouse release 1.1.54383, 2018-05-22 + +### Bug fixes: + +* Fixed a slowdown of replication queue if a table has many replicas. + +## ClickHouse release 1.1.54381, 2018-05-14 + +### Bug fixes: + +* Fixed a nodes leak in ZooKeeper when ClickHouse loses connection to ZooKeeper server. + +## ClickHouse release 1.1.54380, 2018-04-21 + +### New features: + +* Added the table function `file(path, format, structure)`. An example reading bytes from `/dev/urandom`: `ln -s /dev/urandom /var/lib/clickhouse/user_files/random``clickhouse-client -q "SELECT * FROM file('random', 'RowBinary', 'd UInt8') LIMIT 10"`. + +### Improvements: + +* Subqueries can be wrapped in `()` brackets to enhance query readability. For example: `(SELECT 1) UNION ALL (SELECT 1)`. +* Simple `SELECT` queries from the `system.processes` table are not included in the `max_concurrent_queries` limit. + +### Bug fixes: + +* Fixed incorrect behavior of the `IN` operator when select from `MATERIALIZED VIEW`. +* Fixed incorrect filtering by partition index in expressions like `partition_key_column IN (...)`. +* Fixed inability to execute `OPTIMIZE` query on non-leader replica if `REANAME` was performed on the table. +* Fixed the authorization error when executing `OPTIMIZE` or `ALTER` queries on a non-leader replica. +* Fixed freezing of `KILL QUERY`. +* Fixed an error in ZooKeeper client library which led to loss of watches, freezing of distributed DDL queue, and slowdowns in the replication queue if a non-empty `chroot` prefix is used in the ZooKeeper configuration. + +### Backward incompatible changes: + +* Removed support for expressions like `(a, b) IN (SELECT (a, b))` (you can use the equivalent expression `(a, b) IN (SELECT a, b)`). In previous releases, these expressions led to undetermined `WHERE` filtering or caused errors. + +## ClickHouse release 1.1.54378, 2018-04-16 + +### New features: + +* Logging level can be changed without restarting the server. +* Added the `SHOW CREATE DATABASE` query. +* The `query_id` can be passed to `clickhouse-client` (elBroom). +* New setting: `max_network_bandwidth_for_all_users`. +* Added support for `ALTER TABLE ... PARTITION ... ` for `MATERIALIZED VIEW`. +* Added information about the size of data parts in uncompressed form in the system table. +* Server-to-server encryption support for distributed tables (`1` in the replica config in ``). +* Configuration of the table level for the `ReplicatedMergeTree` family in order to minimize the amount of data stored in Zookeeper: : `use_minimalistic_checksums_in_zookeeper = 1` +* Configuration of the `clickhouse-client` prompt. By default, server names are now output to the prompt. The server's display name can be changed. It's also sent in the `X-ClickHouse-Display-Name` HTTP header (Kirill Shvakov). +* Multiple comma-separated `topics` can be specified for the `Kafka` engine (Tobias Adamson) +* When a query is stopped by `KILL QUERY` or `replace_running_query`, the client receives the `Query was cancelled` exception instead of an incomplete result. + +### Improvements: + +* `ALTER TABLE ... DROP/DETACH PARTITION` queries are run at the front of the replication queue. +* `SELECT ... FINAL` and `OPTIMIZE ... FINAL` can be used even when the table has a single data part. +* A `query_log` table is recreated on the fly if it was deleted manually (Kirill Shvakov). +* The `lengthUTF8` function runs faster (zhang2014). +* Improved performance of synchronous inserts in `Distributed` tables (`insert_distributed_sync = 1`) when there is a very large number of shards. +* The server accepts the `send_timeout` and `receive_timeout` settings from the client and applies them when connecting to the client (they are applied in reverse order: the server socket's `send_timeout` is set to the `receive_timeout` value received from the client, and vice versa). +* More robust crash recovery for asynchronous insertion into `Distributed` tables. +* The return type of the `countEqual` function changed from `UInt32` to `UInt64` (谢磊). + +### Bug fixes: + +* Fixed an error with `IN` when the left side of the expression is `Nullable`. +* Correct results are now returned when using tuples with `IN` when some of the tuple components are in the table index. +* The `max_execution_time` limit now works correctly with distributed queries. +* Fixed errors when calculating the size of composite columns in the `system.columns` table. +* Fixed an error when creating a temporary table `CREATE TEMPORARY TABLE IF NOT EXISTS.` +* Fixed errors in `StorageKafka` (##2075) +* Fixed server crashes from invalid arguments of certain aggregate functions. +* Fixed the error that prevented the `DETACH DATABASE` query from stopping background tasks for `ReplicatedMergeTree` tables. +* `Too many parts` state is less likely to happen when inserting into aggregated materialized views (##2084). +* Corrected recursive handling of substitutions in the config if a substitution must be followed by another substitution on the same level. +* Corrected the syntax in the metadata file when creating a `VIEW` that uses a query with `UNION ALL`. +* `SummingMergeTree` now works correctly for summation of nested data structures with a composite key. +* Fixed the possibility of a race condition when choosing the leader for `ReplicatedMergeTree` tables. + +### Build changes: + +* The build supports `ninja` instead of `make` and uses `ninja` by default for building releases. +* Renamed packages: `clickhouse-server-base` in `clickhouse-common-static`; `clickhouse-server-common` in `clickhouse-server`; `clickhouse-common-dbg` in `clickhouse-common-static-dbg`. To install, use `clickhouse-server clickhouse-client`. Packages with the old names will still load in the repositories for backward compatibility. + +### Backward incompatible changes: + +* Removed the special interpretation of an IN expression if an array is specified on the left side. Previously, the expression `arr IN (set)` was interpreted as "at least one `arr` element belongs to the `set`". To get the same behavior in the new version, write `arrayExists(x -> x IN (set), arr)`. +* Disabled the incorrect use of the socket option `SO_REUSEPORT`, which was incorrectly enabled by default in the Poco library. Note that on Linux there is no longer any reason to simultaneously specify the addresses `::` and `0.0.0.0` for listen – use just `::`, which allows listening to the connection both over IPv4 and IPv6 (with the default kernel config settings). You can also revert to the behavior from previous versions by specifying `1` in the config. + +## ClickHouse release 1.1.54370, 2018-03-16 + +### New features: * Added the `system.macros` table and auto updating of macros when the config file is changed. * Added the `SYSTEM RELOAD CONFIG` query. * Added the `maxIntersections(left_col, right_col)` aggregate function, which returns the maximum number of simultaneously intersecting intervals `[left; right]`. The `maxIntersectionsPosition(left, right)` function returns the beginning of the "maximum" interval. ([Michael Furmur](https://github.com/yandex/ClickHouse/pull/2012)). -## Improvements: +### Improvements: * When inserting data in a `Replicated` table, fewer requests are made to `ZooKeeper` (and most of the user-level errors have disappeared from the `ZooKeeper` log). -* Added the ability to create aliases for sets. Example: `WITH (1, 2, 3) AS set SELECT number IN set FROM system.numbers LIMIT 10`. +* Added the ability to create aliases for data sets. Example: `WITH (1, 2, 3) AS set SELECT number IN set FROM system.numbers LIMIT 10`. -## Bug fixes: +### Bug fixes: -* Fixed the `Illegal PREWHERE` error when reading from `Merge` tables over `Distributed` tables. -* Added fixes that allow you to run `clickhouse-server` in IPv4-only Docker containers. -* Fixed a race condition when reading from system `system.parts_columns` tables. +* Fixed the `Illegal PREWHERE` error when reading from Merge tables for `Distributed`tables. +* Added fixes that allow you to start clickhouse-server in IPv4-only Docker containers. +* Fixed a race condition when reading from system `system.parts_columns tables.` * Removed double buffering during a synchronous insert to a `Distributed` table, which could have caused the connection to timeout. * Fixed a bug that caused excessively long waits for an unavailable replica before beginning a `SELECT` query. * Fixed incorrect dates in the `system.parts` table. * Fixed a bug that made it impossible to insert data in a `Replicated` table if `chroot` was non-empty in the configuration of the `ZooKeeper` cluster. * Fixed the vertical merging algorithm for an empty `ORDER BY` table. * Restored the ability to use dictionaries in queries to remote tables, even if these dictionaries are not present on the requestor server. This functionality was lost in release 1.1.54362. -* Restored the behavior for queries like `SELECT * FROM remote('server2', default.table) WHERE col IN (SELECT col2 FROM default.table)` when the right side argument of the `IN` should use a remote `default.table` instead of a local one. This behavior was broken in version 1.1.54358. +* Restored the behavior for queries like `SELECT * FROM remote('server2', default.table) WHERE col IN (SELECT col2 FROM default.table)` when the right side of the `IN` should use a remote `default.table` instead of a local one. This behavior was broken in version 1.1.54358. * Removed extraneous error-level logging of `Not found column ... in block`. -# ClickHouse release 1.1.54356, 2018-03-06 +## Clickhouse Release 1.1.54362, 2018-03-11 -## New features: +### New features: * Aggregation without `GROUP BY` for an empty set (such as `SELECT count(*) FROM table WHERE 0`) now returns a result with one row with null values for aggregate functions, in compliance with the SQL standard. To restore the old behavior (return an empty result), set `empty_result_for_aggregation_by_empty_set` to 1. * Added type conversion for `UNION ALL`. Different alias names are allowed in `SELECT` positions in `UNION ALL`, in compliance with the SQL standard. -* Arbitrary expressions are supported in `LIMIT BY` sections. Previously, it was only possible to use columns resulting from `SELECT`. +* Arbitrary expressions are supported in `LIMIT BY` clauses. Previously, it was only possible to use columns resulting from `SELECT`. * An index of `MergeTree` tables is used when `IN` is applied to a tuple of expressions from the columns of the primary key. Example: `WHERE (UserID, EventDate) IN ((123, '2000-01-01'), ...)` (Anastasiya Tsarkova). * Added the `clickhouse-copier` tool for copying between clusters and resharding data (beta). * Added consistent hashing functions: `yandexConsistentHash`, `jumpConsistentHash`, `sumburConsistentHash`. They can be used as a sharding key in order to reduce the amount of network traffic during subsequent reshardings. * Added functions: `arrayAny`, `arrayAll`, `hasAny`, `hasAll`, `arrayIntersect`, `arrayResize`. * Added the `arrayCumSum` function (Javi Santana). -* Added the `parseDateTimeBestEffort`, `parseDateTimeBestEffortOrZero`, and `parseDateTimeBestEffortOrNull`functions to read the DateTime from a string containing text in a wide variety of possible formats. -* It is now possible to change the logging settings without restarting the server. +* Added the `parseDateTimeBestEffort`, `parseDateTimeBestEffortOrZero`, and `parseDateTimeBestEffortOrNull` functions to read the DateTime from a string containing text in a wide variety of possible formats. +* Data can be partially reloaded from external dictionaries during updating (load just the records in which the value of the specified field greater than in the previous download) (Arsen Hakobyan). * Added the `cluster` table function. Example: `cluster(cluster_name, db, table)`. The `remote` table function can accept the cluster name as the first argument, if it is specified as an identifier. +* The `remote` and `cluster` table functions can be used in `INSERT` requests. * Added the `create_table_query` and `engine_full` virtual columns to the `system.tables`table . The `metadata_modification_time` column is virtual. -* Added the `data_path` and `metadata_path` columns to `system.tables` and` system.databases` tables, and added the `path` column to the `system.parts` and `system.parts_columns` tables. +* Added the `data_path` and `metadata_path` columns to `system.tables`and` system.databases` tables, and added the `path` column to the `system.parts` and `system.parts_columns` tables. * Added additional information about merges in the `system.part_log` table. * An arbitrary partitioning key can be used for the `system.query_log` table (Kirill Shvakov). * The `SHOW TABLES` query now also shows temporary tables. Added temporary tables and the `is_temporary` column to `system.tables` (zhang2014). -* Added the `DROP TEMPORARY TABLE` query (zhang2014). +* Added `DROP TEMPORARY TABLE` and `EXISTS TEMPORARY TABLE` queries (zhang2014). * Support for `SHOW CREATE TABLE` for temporary tables (zhang2014). * Added the `system_profile` configuration parameter for the settings used by internal processes. * Support for loading `object_id` as an attribute in `MongoDB` dictionaries (Pavel Litvinenko). @@ -60,9 +679,11 @@ * `MergeTree` tables can be used without a primary key (you need to specify `ORDER BY tuple()`). * A `Nullable` type can be `CAST` to a non-`Nullable` type if the argument is not `NULL`. * `RENAME TABLE` can be performed for `VIEW`. +* Added the `throwIf` function. * Added the `odbc_default_field_size` option, which allows you to extend the maximum size of the value loaded from an ODBC source (by default, it is 1024). +* The `system.processes` table and `SHOW PROCESSLIST` now have the `is_cancelled` and `peak_memory_usage` columns. -## Improvements: +### Improvements: * Limits and quotas on the result are no longer applied to intermediate data for `INSERT SELECT` queries or for `SELECT` subqueries. * Fewer false triggers of `force_restore_data` when checking the status of `Replicated` tables when the server starts. @@ -78,12 +699,13 @@ * `Enum` values can be used in `min`, `max`, `sum` and some other functions. In these cases, it uses the corresponding numeric values. This feature was previously available but was lost in the release 1.1.54337. * Added `max_expanded_ast_elements` to restrict the size of the AST after recursively expanding aliases. -## Bug fixes: +### Bug fixes: * Fixed cases when unnecessary columns were removed from subqueries in error, or not removed from subqueries containing `UNION ALL`. * Fixed a bug in merges for `ReplacingMergeTree` tables. * Fixed synchronous insertions in `Distributed` tables (`insert_distributed_sync = 1`). * Fixed segfault for certain uses of `FULL` and `RIGHT JOIN` with duplicate columns in subqueries. +* Fixed segfault for certain uses of `replace_running_query` and `KILL QUERY`. * Fixed the order of the `source` and `last_exception` columns in the `system.dictionaries` table. * Fixed a bug when the `DROP DATABASE` query did not delete the file with metadata. * Fixed the `DROP DATABASE` query for `Dictionary` databases. @@ -103,278 +725,297 @@ * Fixed a race condition in the query execution pipeline that occurred in very rare cases when using `Merge` tables with a large number of tables, and when using `GLOBAL` subqueries. * Fixed a crash when passing arrays of different sizes to an `arrayReduce` function when using aggregate functions from multiple arguments. * Prohibited the use of queries with `UNION ALL` in a `MATERIALIZED VIEW`. +* Fixed an error during initialization of the `part_log` system table when the server starts (by default, `part_log` is disabled). -## Backward incompatible changes: +### Backward incompatible changes: * Removed the `distributed_ddl_allow_replicated_alter` option. This behavior is enabled by default. +* Removed the `strict_insert_defaults` setting. If you were using this functionality, write to `clickhouse-feedback@yandex-team.com`. * Removed the `UnsortedMergeTree` engine. -# ClickHouse release 1.1.54343, 2018-02-05 +## Clickhouse Release 1.1.54343, 2018-02-05 * Added macros support for defining cluster names in distributed DDL queries and constructors of Distributed tables: `CREATE TABLE distr ON CLUSTER '{cluster}' (...) ENGINE = Distributed('{cluster}', 'db', 'table')`. -* Now the table index is used for conditions like `expr IN (subquery)`. +* Now queries like `SELECT ... FROM table WHERE expr IN (subquery)` are processed using the `table` index. * Improved processing of duplicates when inserting to Replicated tables, so they no longer slow down execution of the replication queue. -# ClickHouse release 1.1.54342, 2018-01-22 +## Clickhouse Release 1.1.54342, 2018-01-22 This release contains bug fixes for the previous release 1.1.54337: -* Fixed a regression in 1.1.54337: if the default user has readonly access, then the server refuses to start up with the message `Cannot create database in readonly mode`. -* Fixed a regression in 1.1.54337: on systems with `systemd`, logs are always written to syslog regardless of the configuration; the watchdog script still uses `init.d`. + +* Fixed a regression in 1.1.54337: if the default user has readonly access, then the server refuses to start up with the message `Cannot create database in readonly mode`. +* Fixed a regression in 1.1.54337: on systems with systemd, logs are always written to syslog regardless of the configuration; the watchdog script still uses init.d. * Fixed a regression in 1.1.54337: wrong default configuration in the Docker image. -* Fixed nondeterministic behaviour of GraphiteMergeTree (you can notice it in log messages `Data after merge is not byte-identical to data on another replicas`). -* Fixed a bug that may lead to inconsistent merges after OPTIMIZE query to Replicated tables (you may notice it in log messages `Part ... intersects previous part`). +* Fixed nondeterministic behavior of GraphiteMergeTree (you can see it in log messages `Data after merge is not byte-identical to the data on another replicas`). +* Fixed a bug that may lead to inconsistent merges after OPTIMIZE query to Replicated tables (you may see it in log messages `Part ... intersects the previous part`). * Buffer tables now work correctly when MATERIALIZED columns are present in the destination table (by zhang2014). * Fixed a bug in implementation of NULL. -# ClickHouse release 1.1.54337, 2018-01-18 +## Clickhouse Release 1.1.54337, 2018-01-18 -## New features: +### New features: -* Added support for storage of multidimensional arrays and tuples (`Tuple` data type) in tables. -* Added support for table functions in `DESCRIBE` and `INSERT` queries. Added support for subqueries in `DESCRIBE`. Examples: `DESC TABLE remote('host', default.hits)`; `DESC TABLE (SELECT 1)`; `INSERT INTO TABLE FUNCTION remote('host', default.hits)`. Support for `INSERT INTO TABLE` syntax in addition to `INSERT INTO`. -* Improved support for timezones. The `DateTime` data type can be annotated with the timezone that is used for parsing and formatting in text formats. Example: `DateTime('Europe/Moscow')`. When timezones are specified in functions for DateTime arguments, the return type will track the timezone, and the value will be displayed as expected. -* Added the functions `toTimeZone`, `timeDiff`, `toQuarter`, `toRelativeQuarterNum`. The `toRelativeHour`/`Minute`/`Second` functions can take a value of type `Date` as an argument. The name of the `now` function has been made case-insensitive. +* Added support for storage of multi-dimensional arrays and tuples (`Tuple` data type) in tables. +* Support for table functions for `DESCRIBE` and `INSERT` queries. Added support for subqueries in `DESCRIBE`. Examples: `DESC TABLE remote('host', default.hits)`; `DESC TABLE (SELECT 1)`; `INSERT INTO TABLE FUNCTION remote('host', default.hits)`. Support for `INSERT INTO TABLE` in addition to `INSERT INTO`. +* Improved support for time zones. The `DateTime` data type can be annotated with the timezone that is used for parsing and formatting in text formats. Example: `DateTime('Europe/Moscow')`. When timezones are specified in functions for `DateTime` arguments, the return type will track the timezone, and the value will be displayed as expected. +* Added the functions `toTimeZone`, `timeDiff`, `toQuarter`, `toRelativeQuarterNum`. The `toRelativeHour`/`Minute`/`Second` functions can take a value of type `Date` as an argument. The `now` function name is case-sensitive. * Added the `toStartOfFifteenMinutes` function (Kirill Shvakov). * Added the `clickhouse format` tool for formatting queries. -* Added the `format_schema_path` configuration parameter (Marek Vavruša). It is used for specifying a schema in `Cap'n'Proto` format. Schema files can be located only in the specified directory. +* Added the `format_schema_path` configuration parameter (Marek Vavruşa). It is used for specifying a schema in `Cap'n Proto` format. Schema files can be located only in the specified directory. * Added support for config substitutions (`incl` and `conf.d`) for configuration of external dictionaries and models (Pavel Yakunin). -* Added a column with documentation for the `system.settings` table (Kirill Shvakov). +* Added a column with documentation for the `system.settings` table (Kirill Shvakov). * Added the `system.parts_columns` table with information about column sizes in each data part of `MergeTree` tables. * Added the `system.models` table with information about loaded `CatBoost` machine learning models. -* Added the `mysql` and `odbc` table functions along with the corresponding `MySQL` and `ODBC` table engines for working with foreign databases. This feature is in the beta stage. -* Added the possibility to pass an argument of type `AggregateFunction` for the `groupArray` aggregate function (so you can create an array of states of some aggregate function). -* Removed restrictions on various combinations of aggregate function combinators. For example, you can use `avgForEachIf` as well as `avgIfForEach` aggregate functions, which have different behaviors. -* The `-ForEach` aggregate function combinator is extended for the case of aggregate functions of multiple arguments. -* Added support for aggregate functions of `Nullable` arguments even for cases when the function returns a non-`Nullable` result (added with the contribution of Silviu Caragea). Examples: `groupArray`, `groupUniqArray`, `topK`. -* Added the `max_client_network_bandwidth` command line parameter for `clickhouse-client` (Kirill Shvakov). -* Users with the `readonly = 2` setting are allowed to work with TEMPORARY tables (CREATE, DROP, INSERT...) (Kirill Shvakov). -* Added support for using multiple consumers with the `Kafka` engine. Extended configuration options for `Kafka` (Marek Vavruša). -* Added the `intExp2` and `intExp10` functions. -* Added the `sumKahan` aggregate function (computationally stable summation of floating point numbers). -* Added to*Number*OrNull functions, where *Number* is a numeric type. -* Added support for the `WITH` clause for an `INSERT SELECT` query (by zhang2014). -* Added the settings `http_connection_timeout`, `http_send_timeout`, and `http_receive_timeout`. In particular, these settings are used for downloading data parts for replication. Changing these settings allows for faster failover if the network is overloaded. -* Added support for the `ALTER` query for tables of type `Null` (Anastasiya Tsarkova). Tables of type `Null` are often used with materialized views. -* The `reinterpretAsString` function is extended for all data types that are stored contiguously in memory. -* Added the `--silent` option for the `clickhouse-local` tool. It suppresses printing query execution info in stderr. -* Added support for reading values of type `Date` from text in a format where the month and/or day of the month is specified using a single digit instead of two digits (Amos Bird). +* Added the `mysql` and `odbc` table function and corresponding `MySQL` and `ODBC` table engines for accessing remote databases. This functionality is in the beta stage. +* Added the possibility to pass an argument of type `AggregateFunction` for the `groupArray` aggregate function (so you can create an array of states of some aggregate function). +* Removed restrictions on various combinations of aggregate function combinators. For example, you can use `avgForEachIf` as well as `avgIfForEach` aggregate functions, which have different behaviors. +* The `-ForEach` aggregate function combinator is extended for the case of aggregate functions of multiple arguments. +* Added support for aggregate functions of `Nullable` arguments even for cases when the function returns a non-`Nullable` result (added with the contribution of Silviu Caragea). Example: `groupArray`, `groupUniqArray`, `topK`. +* Added the `max_client_network_bandwidth` for `clickhouse-client` (Kirill Shvakov). +* Users with the ` readonly = 2` setting are allowed to work with TEMPORARY tables (CREATE, DROP, INSERT...) (Kirill Shvakov). +* Added support for using multiple consumers with the `Kafka` engine. Extended configuration options for `Kafka` (Marek Vavruša). +* Added the `intExp3` and `intExp4` functions. +* Added the `sumKahan` aggregate function. +* Added the to * Number* OrNull functions, where * Number* is a numeric type. +* Added support for `WITH` clauses for an `INSERT SELECT` query (author: zhang2014). +* Added settings: `http_connection_timeout`, `http_send_timeout`, `http_receive_timeout`. In particular, these settings are used for downloading data parts for replication. Changing these settings allows for faster failover if the network is overloaded. +* Added support for `ALTER` for tables of type `Null` (Anastasiya Tsarkova). +* The `reinterpretAsString` function is extended for all data types that are stored contiguously in memory. +* Added the `--silent` option for the `clickhouse-local` tool. It suppresses printing query execution info in stderr. +* Added support for reading values of type `Date` from text in a format where the month and/or day of the month is specified using a single digit instead of two digits (Amos Bird). -## Performance optimizations: +### Performance optimizations: -* Improved performance of `min`, `max`, `any`, `anyLast`, `anyHeavy`, `argMin`, `argMax` aggregate functions for String arguments. -* Improved performance of `isInfinite`, `isFinite`, `isNaN`, `roundToExp2` functions. -* Improved performance of parsing and formatting values of type `Date` and `DateTime` in text formats. +* Improved performance of aggregate functions `min`, `max`, `any`, `anyLast`, `anyHeavy`, `argMin`, `argMax` from string arguments. +* Improved performance of the functions `isInfinite`, `isFinite`, `isNaN`, `roundToExp2`. +* Improved performance of parsing and formatting `Date` and `DateTime` type values in text format. * Improved performance and precision of parsing floating point numbers. -* Lowered memory usage for `JOIN` in the case when the left and right parts have columns with identical names that are not contained in `USING`. -* Improved performance of `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, and `corr` aggregate functions by reducing computational stability. The old functions are available under the names: `varSampStable`, `varPopStable`, `stddevSampStable`, `stddevPopStable`, `covarSampStable`, `covarPopStable`, `corrStable`. +* Lowered memory usage for `JOIN` in the case when the left and right parts have columns with identical names that are not contained in `USING` . +* Improved performance of aggregate functions `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr` by reducing computational stability. The old functions are available under the names `varSampStable`, `varPopStable`, `stddevSampStable`, `stddevPopStable`, `covarSampStable`, `covarPopStable`, `corrStable`. -## Bug fixes: +### Bug fixes: -* Fixed data deduplication after running a `DROP PARTITION` query. In the previous version, dropping a partition and INSERTing the same data again was not working because INSERTed blocks were considered duplicates. -* Fixed a bug that could lead to incorrect interpretation of the `WHERE` clause for `CREATE MATERIALIZED VIEW` queries with `POPULATE`. +* Fixed data deduplication after running a `DROP` or `DETACH PARTITION` query. In the previous version, dropping a partition and inserting the same data again was not working because inserted blocks were considered duplicates. +* Fixed a bug that could lead to incorrect interpretation of the `WHERE` clause for ` CREATE MATERIALIZED VIEW` queries with `POPULATE` . * Fixed a bug in using the `root_path` parameter in the `zookeeper_servers` configuration. -* Fixed unexpected results of passing the `Date` argument to `toStartOfDay`. -* Fixed the `addMonths` and `subtractMonths` functions and the arithmetic for `INTERVAL n MONTH` in cases when the result has the previous year. -* Added missing support for the `UUID` data type for `DISTINCT`, `JOIN`, and `uniq` aggregate functions and external dictionaries (Evgeniy Ivanov). Support for `UUID` is still incomplete. -* Fixed `SummingMergeTree` behavior in cases when the rows summed to zero. -* Various fixes for the `Kafka` engine (Marek Vavruša). -* Fixed incorrect behavior of the `Join` table engine (Amos Bird). +* Fixed unexpected results of passing the `Date` argument to `toStartOfDay` . +* Fixed the `addMonths` and `subtractMonths` functions and the arithmetic for ` INTERVAL n MONTH` in cases when the result has the previous year. +* Added missing support for the `UUID` data type for `DISTINCT` , `JOIN` , and `uniq` aggregate functions and external dictionaries (Evgeniy Ivanov). Support for `UUID` is still incomplete. +* Fixed `SummingMergeTree` behavior in cases when the rows summed to zero. +* Various fixes for the `Kafka` engine (Marek Vavruša). +* Fixed incorrect behavior of the `Join` table engine (Amos Bird). * Fixed incorrect allocator behavior under FreeBSD and OS X. -* The `extractAll` function now supports empty matches. -* Fixed an error that blocked usage of `libressl` instead of `openssl`. -* Fixed the `CREATE TABLE AS SELECT` query from temporary tables. +* The `extractAll` function now supports empty matches. +* Fixed an error that blocked usage of `libressl` instead of `openssl` . +* Fixed the ` CREATE TABLE AS SELECT` query from temporary tables. * Fixed non-atomicity of updating the replication queue. This could lead to replicas being out of sync until the server restarts. -* Fixed possible overflow in `gcd`, `lcm` and `modulo` (`%` operator) (Maks Skorokhod). -* `-preprocessed` files are now created after changing `umask` (`umask` can be changed in the config). -* Fixed a bug in the background check of parts (`MergeTreePartChecker`) when using a custom partition key. -* Fixed parsing of tuples (values of the `Tuple` data type) in text formats. -* Improved error messages about incompatible types passed to `multiIf`, `array` and some other functions. -* Support for `Nullable` types is completely reworked. Fixed bugs that may lead to a server crash. Fixed almost all other bugs related to NULL support: incorrect type conversions in INSERT SELECT, insufficient support for Nullable in HAVING and PREWHERE, `join_use_nulls` mode, Nullable types as arguments of OR operator, etc. -* Fixed various bugs related to internal semantics of data types. Examples: unnecessary summing of `Enum` type fields in `SummingMergeTree`; alignment of `Enum` types in Pretty formats, etc. -* Stricter checks for allowed combinations of composite columns. Fixed several bugs that could lead to a server crash. -* Fixed the overflow when specifying a very large parameter for the `FixedString` data type. -* Fixed a bug in the `topK` aggregate function in a generic case. -* Added the missing check for equality of array sizes in arguments of n-ary variants of aggregate functions with an `-Array` combinator. -* Fixed the `--pager` option for `clickhouse-client` (by ks1322). -* Fixed the precision of the `exp10` function. -* Fixed the behavior of the `visitParamExtract` function for better compliance with documentation. +* Fixed possible overflow in `gcd` , `lcm` and `modulo` (`%` operator) (Maks Skorokhod). +* `-preprocessed` files are now created after changing `umask` (`umask` can be changed in the config). +* Fixed a bug in the background check of parts (`MergeTreePartChecker` ) when using a custom partition key. +* Fixed parsing of tuples (values of the `Tuple` data type) in text formats. +* Improved error messages about incompatible types passed to `multiIf` , `array` and some other functions. +* Redesigned support for `Nullable` types. Fixed bugs that may lead to a server crash. Fixed almost all other bugs related to ` NULL` support: incorrect type conversions in INSERT SELECT, insufficient support for Nullable in HAVING and PREWHERE, `join_use_nulls` mode, Nullable types as arguments of `OR` operator, etc. +* Fixed various bugs related to internal semantics of data types. Examples: unnecessary summing of `Enum` type fields in `SummingMergeTree` ; alignment of `Enum` types in `Pretty` formats, etc. +* Stricter checks for allowed combinations of composite columns. +* Fixed the overflow when specifying a very large parameter for the `FixedString` data type. +* Fixed a bug in the `topK` aggregate function in a generic case. +* Added the missing check for equality of array sizes in arguments of n-ary variants of aggregate functions with an `-Array` combinator. +* Fixed a bug in `--pager` for `clickhouse-client` (author: ks1322). +* Fixed the precision of the `exp10` function. +* Fixed the behavior of the `visitParamExtract` function for better compliance with documentation. * Fixed the crash when incorrect data types are specified. -* Fixed the behavior of `DISTINCT` in the case when all columns are constants. -* Fixed query formatting in the case of using the `tupleElement` function with a complex constant expression as the tuple element index. -* Fixed the `Dictionary` table engine for dictionaries of type `range_hashed`. -* Fixed a bug that leads to excessive rows in the result of `FULL` and `RIGHT JOIN` (Amos Bird). -* Fixed a server crash when creating and removing temporary files in `config.d` directories during config reload. -* Fixed the `SYSTEM DROP DNS CACHE` query: the cache was flushed but addresses of cluster nodes were not updated. -* Fixed the behavior of `MATERIALIZED VIEW` after executing `DETACH TABLE` for the table under the view (Marek Vavruša). +* Fixed the behavior of `DISTINCT` in the case when all columns are constants. +* Fixed query formatting in the case of using the `tupleElement` function with a complex constant expression as the tuple element index. +* Fixed a bug in `Dictionary` tables for `range_hashed` dictionaries. +* Fixed a bug that leads to excessive rows in the result of `FULL` and ` RIGHT JOIN` (Amos Bird). +* Fixed a server crash when creating and removing temporary files in `config.d` directories during config reload. +* Fixed the ` SYSTEM DROP DNS CACHE` query: the cache was flushed but addresses of cluster nodes were not updated. +* Fixed the behavior of ` MATERIALIZED VIEW` after executing ` DETACH TABLE` for the table under the view (Marek Vavruša). -## Build improvements: +### Build improvements: -* Builds use `pbuilder`. The build process is almost completely independent of the build host environment. +* The `pbuilder` tool is used for builds. The build process is almost completely independent of the build host environment. * A single build is used for different OS versions. Packages and binaries have been made compatible with a wide range of Linux systems. -* Added the `clickhouse-test` package. It can be used to run functional tests. +* Added the `clickhouse-test` package. It can be used to run functional tests. * The source tarball can now be published to the repository. It can be used to reproduce the build without using GitHub. * Added limited integration with Travis CI. Due to limits on build time in Travis, only the debug build is tested and a limited subset of tests are run. -* Added support for `Cap'n'Proto` in the default build. -* Changed the format of documentation sources from `Restructured Text` to `Markdown`. +* Added support for `Cap'n'Proto` in the default build. +* Changed the format of documentation sources from `Restricted Text` to `Markdown`. * Added support for `systemd` (Vladimir Smirnov). It is disabled by default due to incompatibility with some OS images and can be enabled manually. -* For dynamic code generation, `clang` and `lld` are embedded into the `clickhouse` binary. They can also be invoked as `clickhouse clang` and `clickhouse lld`. -* Removed usage of GNU extensions from the code. Enabled the `-Wextra` option. When building with `clang`, `libc++` is used instead of `libstdc++`. -* Extracted `clickhouse_parsers` and `clickhouse_common_io` libraries to speed up builds of various tools. +* For dynamic code generation, `clang` and `lld` are embedded into the `clickhouse` binary. They can also be invoked as ` clickhouse clang` and ` clickhouse lld` . +* Removed usage of GNU extensions from the code. Enabled the `-Wextra` option. When building with `clang` the default is `libc++` instead of `libstdc++`. +* Extracted `clickhouse_parsers` and `clickhouse_common_io` libraries to speed up builds of various tools. -## Backward incompatible changes: +### Backward incompatible changes: -* The format for marks in `Log` type tables that contain `Nullable` columns was changed in a backward incompatible way. If you have these tables, you should convert them to the `TinyLog` type before starting up the new server version. To do this, replace `ENGINE = Log` with `ENGINE = TinyLog` in the corresponding `.sql` file in the `metadata` directory. If your table doesn't have `Nullable` columns or if the type of your table is not `Log`, then you don't need to do anything. +* The format for marks in `Log` type tables that contain `Nullable` columns was changed in a backward incompatible way. If you have these tables, you should convert them to the `TinyLog` type before starting up the new server version. To do this, replace `ENGINE = Log` with `ENGINE = TinyLog` in the corresponding `.sql` file in the `metadata` directory. If your table doesn't have `Nullable` columns or if the type of your table is not `Log`, then you don't need to do anything. * Removed the `experimental_allow_extended_storage_definition_syntax` setting. Now this feature is enabled by default. -* To avoid confusion, the `runningIncome` function has been renamed to `runningDifferenceStartingWithFirstValue`. -* Removed the `FROM ARRAY JOIN arr` syntax when ARRAY JOIN is specified directly after FROM with no table (Amos Bird). -* Removed the `BlockTabSeparated` format that was used solely for demonstration purposes. -* Changed the serialization format of intermediate states of the aggregate functions `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, and `corr`. If you have stored states of these aggregate functions in tables (using the AggregateFunction data type or materialized views with corresponding states), please write to clickhouse-feedback@yandex-team.com. +* The `runningIncome` function was renamed to `runningDifferenceStartingWithFirstvalue` to avoid confusion. +* Removed the ` FROM ARRAY JOIN arr` syntax when ARRAY JOIN is specified directly after FROM with no table (Amos Bird). +* Removed the `BlockTabSeparated` format that was used solely for demonstration purposes. +* Changed the state format for aggregate functions `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr`. If you have stored states of these aggregate functions in tables (using the `AggregateFunction` data type or materialized views with corresponding states), please write to clickhouse-feedback@yandex-team.com. * In previous server versions there was an undocumented feature: if an aggregate function depends on parameters, you can still specify it without parameters in the AggregateFunction data type. Example: `AggregateFunction(quantiles, UInt64)` instead of `AggregateFunction(quantiles(0.5, 0.9), UInt64)`. This feature was lost. Although it was undocumented, we plan to support it again in future releases. -* Enum data types cannot be used in min/max aggregate functions. The possibility will be returned back in future release. +* Enum data types cannot be used in min/max aggregate functions. This ability will be returned in the next release. -## Please note when upgrading: -* When doing a rolling update on a cluster, at the point when some of the replicas are running the old version of ClickHouse and some are running the new version, replication is temporarily stopped and the message `unknown parameter 'shard'` appears in the log. Replication will continue after all replicas of the cluster are updated. -* If you have different ClickHouse versions on the cluster, you can get incorrect results for distributed queries with the aggregate functions `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, and `corr`. You should update all cluster nodes. +### Please note when upgrading: -# ClickHouse release 1.1.54327, 2017-12-21 +* When doing a rolling update on a cluster, at the point when some of the replicas are running the old version of ClickHouse and some are running the new version, replication is temporarily stopped and the message ` unknown parameter 'shard'` appears in the log. Replication will continue after all replicas of the cluster are updated. +* If different versions of ClickHouse are running on the cluster servers, it is possible that distributed queries using the following functions will have incorrect results: `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr`. You should update all cluster nodes. + +## ClickHouse release 1.1.54327, 2017-12-21 This release contains bug fixes for the previous release 1.1.54318: -* Fixed bug with possible race condition in replication that could lead to data loss. This issue affects versions 1.1.54310 and 1.1.54318. If you use one of these versions with Replicated tables, the update is strongly recommended. This issue shows in logs in Warning messages like `Part ... from own log doesn't exist.` The issue is relevant even if you don't see these messages in logs. -# ClickHouse release 1.1.54318, 2017-11-30 +* Fixed bug with possible race condition in replication that could lead to data loss. This issue affects versions 1.1.54310 and 1.1.54318. If you use one of these versions with Replicated tables, the update is strongly recommended. This issue shows in logs in Warning messages like ` Part ... from own log doesn't exist.` The issue is relevant even if you don't see these messages in logs. + +## ClickHouse release 1.1.54318, 2017-11-30 This release contains bug fixes for the previous release 1.1.54310: + * Fixed incorrect row deletions during merges in the SummingMergeTree engine * Fixed a memory leak in unreplicated MergeTree engines * Fixed performance degradation with frequent inserts in MergeTree engines * Fixed an issue that was causing the replication queue to stop running * Fixed rotation and archiving of server logs -# ClickHouse release 1.1.54310, 2017-11-01 +## ClickHouse release 1.1.54310, 2017-11-01 + +### New features: -## New features: * Custom partitioning key for the MergeTree family of table engines. -* [Kafka](https://clickhouse.yandex/docs/en/single/index.html#document-table_engines/kafka) table engine. -* Added support for loading [CatBoost](https://catboost.yandex/) models and applying them to data stored in ClickHouse. +* [ Kafka](https://clickhouse.yandex/docs/en/single/index.html#document-table_engines/kafka) table engine. +* Added support for loading [CatBoost](https://catboost.yandex/) models and applying them to data stored in ClickHouse. * Added support for time zones with non-integer offsets from UTC. * Added support for arithmetic operations with time intervals. * The range of values for the Date and DateTime types is extended to the year 2105. -* Added the `CREATE MATERIALIZED VIEW x TO y` query (specifies an existing table for storing the data of a materialized view). +* Added the ` CREATE MATERIALIZED VIEW x TO y` query (specifies an existing table for storing the data of a materialized view). * Added the `ATTACH TABLE` query without arguments. * The processing logic for Nested columns with names ending in -Map in a SummingMergeTree table was extracted to the sumMap aggregate function. You can now specify such columns explicitly. * Max size of the IP trie dictionary is increased to 128M entries. -* Added the `getSizeOfEnumType` function. -* Added the `sumWithOverflow` aggregate function. +* Added the getSizeOfEnumType function. +* Added the sumWithOverflow aggregate function. * Added support for the Cap'n Proto input format. * You can now customize compression level when using the zstd algorithm. -## Backward incompatible changes: -* Creation of temporary tables with an engine other than Memory is forbidden. -* Explicit creation of tables with the View or MaterializedView engine is forbidden. +### Backward incompatible changes: + +* Creation of temporary tables with an engine other than Memory is not allowed. +* Explicit creation of tables with the View or MaterializedView engine is not allowed. * During table creation, a new check verifies that the sampling key expression is included in the primary key. -## Bug fixes: +### Bug fixes: + * Fixed hangups when synchronously inserting into a Distributed table. * Fixed nonatomic adding and removing of parts in Replicated tables. * Data inserted into a materialized view is not subjected to unnecessary deduplication. * Executing a query to a Distributed table for which the local replica is lagging and remote replicas are unavailable does not result in an error anymore. -* Users don't need access permissions to the `default` database to create temporary tables anymore. +* Users don't need access permissions to the `default` database to create temporary tables anymore. * Fixed crashing when specifying the Array type without arguments. * Fixed hangups when the disk volume containing server logs is full. -* Fixed an overflow in the `toRelativeWeekNum` function for the first week of the Unix epoch. +* Fixed an overflow in the toRelativeWeekNum function for the first week of the Unix epoch. + +### Build improvements: -## Build improvements: * Several third-party libraries (notably Poco) were updated and converted to git submodules. -# ClickHouse release 1.1.54304, 2017-10-19 +## ClickHouse release 1.1.54304, 2017-10-19 -## New features: -* TLS support in the native protocol (to enable, set `tcp_ssl_port` in `config.xml`) +### New features: -## Bug fixes: -* `ALTER` for replicated tables now tries to start running as soon as possible -* Fixed crashing when reading data with the setting `preferred_block_size_bytes=0` -* Fixed crashes of `clickhouse-client` when `Page Down` is pressed +* TLS support in the native protocol (to enable, set `tcp_ssl_port` in `config.xml` ). + +### Bug fixes: + +* `ALTER` for replicated tables now tries to start running as soon as possible. +* Fixed crashing when reading data with the setting `preferred_block_size_bytes=0.` +* Fixed crashes of `clickhouse-client` when pressing ` Page Down` * Correct interpretation of certain complex queries with `GLOBAL IN` and `UNION ALL` -* `FREEZE PARTITION` always works atomically now -* Empty POST requests now return a response with code 411 -* Fixed interpretation errors for expressions like `CAST(1 AS Nullable(UInt8))` -* Fixed an error when reading columns like `Array(Nullable(String))` from `MergeTree` tables +* `FREEZE PARTITION` always works atomically now. +* Empty POST requests now return a response with code 411. +* Fixed interpretation errors for expressions like `CAST(1 AS Nullable(UInt8)).` +* Fixed an error when reading `Array(Nullable(String))` columns from `MergeTree` tables. * Fixed crashing when parsing queries like `SELECT dummy AS dummy, dummy AS b` -* Users are updated correctly when `users.xml` is invalid -* Correct handling when an executable dictionary returns a non-zero response code +* Users are updated correctly with invalid `users.xml` +* Correct handling when an executable dictionary returns a non-zero response code. -# ClickHouse release 1.1.54292, 2017-09-20 +## ClickHouse release 1.1.54292, 2017-09-20 -## New features: -* Added the `pointInPolygon` function for working with coordinates on a coordinate plane. -* Added the `sumMap` aggregate function for calculating the sum of arrays, similar to `SummingMergeTree`. -* Added the `trunc` function. Improved performance of the rounding functions (`round`, `floor`, `ceil`, `roundToExp2`) and corrected the logic of how they work. Changed the logic of the `roundToExp2` function for fractions and negative numbers. -* The ClickHouse executable file is now less dependent on the libc version. The same ClickHouse executable file can run on a wide variety of Linux systems. Note: There is still a dependency when using compiled queries (with the setting `compile = 1`, which is not used by default). +### New features: + +* Added the `pointInPolygon` function for working with coordinates on a coordinate plane. +* Added the `sumMap` aggregate function for calculating the sum of arrays, similar to `SummingMergeTree`. +* Added the `trunc` function. Improved performance of the rounding functions (`round`, `floor`, `ceil`, `roundToExp2`) and corrected the logic of how they work. Changed the logic of the `roundToExp2` function for fractions and negative numbers. +* The ClickHouse executable file is now less dependent on the libc version. The same ClickHouse executable file can run on a wide variety of Linux systems. There is still a dependency when using compiled queries (with the setting ` compile = 1` , which is not used by default). * Reduced the time needed for dynamic compilation of queries. -## Bug fixes: -* Fixed an error that sometimes produced `part ... intersects previous part` messages and weakened replica consistency. +### Bug fixes: + +* Fixed an error that sometimes produced ` part ... intersects previous part` messages and weakened replica consistency. * Fixed an error that caused the server to lock up if ZooKeeper was unavailable during shutdown. * Removed excessive logging when restoring replicas. * Fixed an error in the UNION ALL implementation. * Fixed an error in the concat function that occurred if the first column in a block has the Array type. * Progress is now displayed correctly in the system.merges table. -# ClickHouse release 1.1.54289, 2017-09-13 +## ClickHouse release 1.1.54289, 2017-09-13 + +### New features: -## New features: * `SYSTEM` queries for server administration: `SYSTEM RELOAD DICTIONARY`, `SYSTEM RELOAD DICTIONARIES`, `SYSTEM DROP DNS CACHE`, `SYSTEM SHUTDOWN`, `SYSTEM KILL`. * Added functions for working with arrays: `concat`, `arraySlice`, `arrayPushBack`, `arrayPushFront`, `arrayPopBack`, `arrayPopFront`. -* Added the `root` and `identity` parameters for the ZooKeeper configuration. This allows you to isolate individual users on the same ZooKeeper cluster. -* Added the aggregate functions `groupBitAnd`, `groupBitOr`, and `groupBitXor` (for compatibility, they can also be accessed with the names `BIT_AND`, `BIT_OR`, and `BIT_XOR`). +* Added `root` and `identity` parameters for the ZooKeeper configuration. This allows you to isolate individual users on the same ZooKeeper cluster. +* Added aggregate functions `groupBitAnd`, `groupBitOr`, and `groupBitXor` (for compatibility, they are also available under the names `BIT_AND`, `BIT_OR`, and `BIT_XOR`). * External dictionaries can be loaded from MySQL by specifying a socket in the filesystem. -* External dictionaries can be loaded from MySQL over SSL (the `ssl_cert`, `ssl_key`, and `ssl_ca` parameters). +* External dictionaries can be loaded from MySQL over SSL (`ssl_cert`, `ssl_key`, `ssl_ca` parameters). * Added the `max_network_bandwidth_for_user` setting to restrict the overall bandwidth use for queries per user. * Support for `DROP TABLE` for temporary tables. -* Support for reading `DateTime` values in Unix timestamp format from the `CSV` and `JSONEachRow` formats. +* Support for reading `DateTime` values in Unix timestamp format from the `CSV` and `JSONEachRow` formats. * Lagging replicas in distributed queries are now excluded by default (the default threshold is 5 minutes). * FIFO locking is used during ALTER: an ALTER query isn't blocked indefinitely for continuously running queries. -* Option to set `umask` in the config file. -* Improved performance for queries with `DISTINCT`. +* Option to set `umask` in the config file. +* Improved performance for queries with `DISTINCT` . + +### Bug fixes: -## Bug fixes: * Improved the process for deleting old nodes in ZooKeeper. Previously, old nodes sometimes didn't get deleted if there were very frequent inserts, which caused the server to be slow to shut down, among other things. * Fixed randomization when choosing hosts for the connection to ZooKeeper. * Fixed the exclusion of lagging replicas in distributed queries if the replica is localhost. -* Fixed an error where a data part in a `ReplicatedMergeTree` table could be broken after running `ALTER MODIFY` on an element in a `Nested` structure. +* Fixed an error where a data part in a `ReplicatedMergeTree` table could be broken after running ` ALTER MODIFY` on an element in a `Nested` structure. * Fixed an error that could cause SELECT queries to "hang". * Improvements to distributed DDL queries. * Fixed the query `CREATE TABLE ... AS `. -* Resolved the deadlock in the `ALTER ... CLEAR COLUMN IN PARTITION` query for `Buffer` tables. -* Fixed the invalid default value for `Enum`s (0 instead of the minimum) when using the `JSONEachRow` and `TSKV` formats. -* Resolved the appearance of zombie processes when using a dictionary with an `executable` source. +* Resolved the deadlock in the ` ALTER ... CLEAR COLUMN IN PARTITION` query for `Buffer` tables. +* Fixed the invalid default value for `Enum` s (0 instead of the minimum) when using the `JSONEachRow` and `TSKV` formats. +* Resolved the appearance of zombie processes when using a dictionary with an `executable` source. * Fixed segfault for the HEAD query. -## Improvements to development workflow and ClickHouse build: -* You can use `pbuilder` to build ClickHouse. -* You can use `libc++` instead of `libstdc++` for builds on Linux. -* Added instructions for using static code analysis tools: `Coverity`, `clang-tidy`, and `cppcheck`. +### Improved workflow for developing and assembling ClickHouse: -## Please note when upgrading: -* There is now a higher default value for the MergeTree setting `max_bytes_to_merge_at_max_space_in_pool` (the maximum total size of data parts to merge, in bytes): it has increased from 100 GiB to 150 GiB. This might result in large merges running after the server upgrade, which could cause an increased load on the disk subsystem. If the free space available on the server is less than twice the total amount of the merges that are running, this will cause all other merges to stop running, including merges of small data parts. As a result, INSERT requests will fail with the message "Merges are processing significantly slower than inserts." Use the `SELECT * FROM system.merges` request to monitor the situation. You can also check the `DiskSpaceReservedForMerge` metric in the `system.metrics` table, or in Graphite. You don't need to do anything to fix this, since the issue will resolve itself once the large merges finish. If you find this unacceptable, you can restore the previous value for the `max_bytes_to_merge_at_max_space_in_pool` setting (to do this, go to the `` section in config.xml, set `107374182400` and restart the server). +* You can use `pbuilder` to build ClickHouse. +* You can use `libc++` instead of `libstdc++` for builds on Linux. +* Added instructions for using static code analysis tools: `Coverage`, `clang-tidy`, `cppcheck`. -# ClickHouse release 1.1.54284, 2017-08-29 +### Please note when upgrading: -* This is bugfix release for previous 1.1.54282 release. It fixes ZooKeeper nodes leak in `parts/` directory. +* There is now a higher default value for the MergeTree setting `max_bytes_to_merge_at_max_space_in_pool` (the maximum total size of data parts to merge, in bytes): it has increased from 100 GiB to 150 GiB. This might result in large merges running after the server upgrade, which could cause an increased load on the disk subsystem. If the free space available on the server is less than twice the total amount of the merges that are running, this will cause all other merges to stop running, including merges of small data parts. As a result, INSERT requests will fail with the message "Merges are processing significantly slower than inserts." Use the ` SELECT * FROM system.merges` request to monitor the situation. You can also check the `DiskSpaceReservedForMerge` metric in the `system.metrics` table, or in Graphite. You don't need to do anything to fix this, since the issue will resolve itself once the large merges finish. If you find this unacceptable, you can restore the previous value for the `max_bytes_to_merge_at_max_space_in_pool` setting. To do this, go to the section in config.xml, set ```107374182400` and restart the server. -# ClickHouse release 1.1.54282, 2017-08-23 +## ClickHouse release 1.1.54284, 2017-08-29 -This is a bugfix release. The following bugs were fixed: -* `DB::Exception: Assertion violation: !_path.empty()` error when inserting into a Distributed table. -* Error when parsing inserted data in RowBinary format if the data begins with ';' character. +* This is a bugfix release for the previous 1.1.54282 release. It fixes leaks in the parts directory in ZooKeeper. + +## ClickHouse release 1.1.54282, 2017-08-23 + +This release contains bug fixes for the previous release 1.1.54276: + +* Fixed `DB::Exception: Assertion violation: !_path.empty()` when inserting into a Distributed table. +* Fixed parsing when inserting in RowBinary format if input data starts with';'. * Errors during runtime compilation of certain aggregate functions (e.g. `groupArray()`). -# ClickHouse release 1.1.54276, 2017-08-16 +## Clickhouse Release 1.1.54276, 2017-08-16 -## New features: +### New features: -* You can use an optional WITH clause in a SELECT query. Example query: `WITH 1+1 AS a SELECT a, a*a` +* Added an optional WITH section for a SELECT query. Example query: `WITH 1+1 AS a SELECT a, a*a` * INSERT can be performed synchronously in a Distributed table: OK is returned only after all the data is saved on all the shards. This is activated by the setting insert_distributed_sync=1. * Added the UUID data type for working with 16-byte identifiers. * Added aliases of CHAR, FLOAT and other types for compatibility with the Tableau. @@ -383,40 +1024,40 @@ This is a bugfix release. The following bugs were fixed: * Added support for non-constant arguments and negative offsets in the function `substring(str, pos, len).` * Added the max_size parameter for the `groupArray(max_size)(column)` aggregate function, and optimized its performance. -## Major changes: +### Main changes: -* Improved security: all server files are created with 0640 permissions (can be changed via config parameter). +* Security improvements: all server files are created with 0640 permissions (can be changed via config parameter). * Improved error messages for queries with invalid syntax. * Significantly reduced memory consumption and improved performance when merging large sections of MergeTree data. * Significantly increased the performance of data merges for the ReplacingMergeTree engine. -* Improved performance for asynchronous inserts from a Distributed table by batching multiple source inserts. To enable this functionality, use the setting distributed_directory_monitor_batch_inserts=1. +* Improved performance for asynchronous inserts from a Distributed table by combining multiple source inserts. To enable this functionality, use the setting distributed_directory_monitor_batch_inserts=1. -## Backward incompatible changes: +### Backward incompatible changes: * Changed the binary format of aggregate states of `groupArray(array_column)` functions for arrays. -## Complete list of changes: +### Complete list of changes: * Added the `output_format_json_quote_denormals` setting, which enables outputting nan and inf values in JSON format. -* Optimized thread allocation when reading from a Distributed table. -* Settings can be modified in readonly mode if the value doesn't change. -* Added the ability to read fractional granules of the MergeTree engine in order to meet restrictions on the block size specified in the preferred_block_size_bytes setting. The purpose is to reduce the consumption of RAM and increase cache locality when processing queries from tables with large columns. +* Optimized stream allocation when reading from a Distributed table. +* Settings can be configured in readonly mode if the value doesn't change. +* Added the ability to retrieve non-integer granules of the MergeTree engine in order to meet restrictions on the block size specified in the preferred_block_size_bytes setting. The purpose is to reduce the consumption of RAM and increase cache locality when processing queries from tables with large columns. * Efficient use of indexes that contain expressions like `toStartOfHour(x)` for conditions like `toStartOfHour(x) op сonstexpr.` * Added new settings for MergeTree engines (the merge_tree section in config.xml): - - replicated_deduplication_window_seconds sets the size of deduplication window in seconds for Replicated tables. - - cleanup_delay_period sets how often to start cleanup to remove outdated data. - - replicated_can_become_leader can prevent a replica from becoming the leader (and assigning merges). + - replicated_deduplication_window_seconds sets the number of seconds allowed for deduplicating inserts in Replicated tables. + - cleanup_delay_period sets how often to start cleanup to remove outdated data. + - replicated_can_become_leader can prevent a replica from becoming the leader (and assigning merges). * Accelerated cleanup to remove outdated data from ZooKeeper. * Multiple improvements and fixes for clustered DDL queries. Of particular interest is the new setting distributed_ddl_task_timeout, which limits the time to wait for a response from the servers in the cluster. * Improved display of stack traces in the server logs. * Added the "none" value for the compression method. * You can use multiple dictionaries_config sections in config.xml. * It is possible to connect to MySQL through a socket in the file system. -* The `system.parts` table has a new column with information about the size of marks, in bytes. +* The system.parts table has a new column with information about the size of marks, in bytes. -## Bug fixes: +### Bug fixes: -* Distributed tables using a Merge table now work correctly for a SELECT query with a condition on the _table field. +* Distributed tables using a Merge table now work correctly for a SELECT query with a condition on the `_table` field. * Fixed a rare race condition in ReplicatedMergeTree when checking data parts. * Fixed possible freezing on "leader election" when starting a server. * The max_replica_delay_for_distributed_queries setting was ignored when using a local replica of the data source. This has been fixed. @@ -425,30 +1066,30 @@ This is a bugfix release. The following bugs were fixed: * Fixed excessive memory allocations when deserializing Native format. * Fixed incorrect auto-update of Trie dictionaries. * Fixed an exception when running queries with a GROUP BY clause from a Merge table when using SAMPLE. -* Fixed a crash of GROUP BY when using distributed_aggregation_memory_efficient=1. +* Fixed a crash of GROUP BY when using distributed_aggregation_memory_efficient=1. * Now you can specify the database.table in the right side of IN and JOIN. * Too many threads were used for parallel aggregation. This has been fixed. * Fixed how the "if" function works with FixedString arguments. * SELECT worked incorrectly from a Distributed table for shards with a weight of 0. This has been fixed. -* Crashes no longer occur when running `CREATE VIEW IF EXISTS.` +* Running `CREATE VIEW IF EXISTS no longer causes crashes.` * Fixed incorrect behavior when input_format_skip_unknown_fields=1 is set and there are negative numbers. * Fixed an infinite loop in the `dictGetHierarchy()` function if there is some invalid data in the dictionary. * Fixed `Syntax error: unexpected (...)` errors when running distributed queries with subqueries in an IN or JOIN clause and Merge tables. -* Fixed the incorrect interpretation of a SELECT query from Dictionary tables. +* Fixed an incorrect interpretation of a SELECT query from Dictionary tables. * Fixed the "Cannot mremap" error when using arrays in IN and JOIN clauses with more than 2 billion elements. * Fixed the failover for dictionaries with MySQL as the source. -## Improved workflow for developing and assembling ClickHouse: +### Improved workflow for developing and assembling ClickHouse: * Builds can be assembled in Arcadia. * You can use gcc 7 to compile ClickHouse. * Parallel builds using ccache+distcc are faster now. -# ClickHouse release 1.1.54245, 2017-07-04 +## ClickHouse release 1.1.54245, 2017-07-04 -## New features: +### New features: -* Distributed DDL (for example, `CREATE TABLE ON CLUSTER`). +* Distributed DDL (for example, `CREATE TABLE ON CLUSTER`) * The replicated request `ALTER TABLE CLEAR COLUMN IN PARTITION.` * The engine for Dictionary tables (access to dictionary data in the form of a table). * Dictionary database engine (this type of database automatically has Dictionary tables available for all the connected external dictionaries). @@ -458,16 +1099,16 @@ This is a bugfix release. The following bugs were fixed: * Sessions in the HTTP interface. * The OPTIMIZE query for a Replicated table can can run not only on the leader. -## Backward incompatible changes: +### Backward incompatible changes: * Removed SET GLOBAL. -## Minor changes: +### Minor changes: -* If an alert is triggered, the full stack trace is printed into the log. -* Relaxed the verification of the number of damaged or extra data parts at startup (there were too many false positives). +* Now after an alert is triggered, the log prints the full stack trace. +* Relaxed the verification of the number of damaged/extra data parts at startup (there were too many false positives). -## Bug fixes: +### Bug fixes: * Fixed a bad connection "sticking" when inserting into a Distributed table. * GLOBAL IN now works for a query from a Merge table that looks at a Distributed table. @@ -475,7 +1116,7 @@ This is a bugfix release. The following bugs were fixed: * Changes in how an executable source of cached external dictionaries works. * Fixed the comparison of strings containing null characters. * Fixed the comparison of Float32 primary key fields with constants. -* Previously, an incorrect estimate of the size of a field could lead to overly large allocations. This has been fixed. +* Previously, an incorrect estimate of the size of a field could lead to overly large allocations. * Fixed a crash when querying a Nullable column added to a table using ALTER. * Fixed a crash when sorting by a Nullable column, if the number of rows is less than LIMIT. * Fixed an ORDER BY subquery consisting of only constant values. diff --git a/CHANGELOG_RU.md b/CHANGELOG_RU.md index 4e0b481e718..216d80f100f 100644 --- a/CHANGELOG_RU.md +++ b/CHANGELOG_RU.md @@ -1,17 +1,633 @@ -# ClickHouse 1.1.54370 Release Candidate, 2018-03-16 +## ClickHouse release 18.14.15, 2018-11-21 -## Новые возможности: +### Исправления ошибок: +* При чтении столбцов типа `Array(String)`, размер требуемого куска памяти оценивался слишком большим, что приводило к исключению "Memory limit exceeded" при выполнении запроса. Ошибка появилась в версии 18.12.13. [#3589](https://github.com/yandex/ClickHouse/issues/3589) + +## ClickHouse release 18.14.14, 2018-11-20 + +### Исправления ошибок: +* Исправлена работа запросов `ON CLUSTER` в случае, когда в конфигурации кластера включено шифрование (флаг ``). [#3599](https://github.com/yandex/ClickHouse/pull/3599) + +### Улучшения процесса сборки ClickHouse: +* Испрпавлены проблемы сборки (llvm-7 из системы, macos) [#3582](https://github.com/yandex/ClickHouse/pull/3582) + +## ClickHouse release 18.14.13, 2018-11-08 + +### Исправления ошибок: +* Исправлена ошибка `Block structure mismatch in MergingSorted stream`. [#3162](https://github.com/yandex/ClickHouse/issues/3162) +* Исправлена работа запросов `ON CLUSTER` в случае, когда в конфигурации кластера включено шифрование (флаг ``). [#3465](https://github.com/yandex/ClickHouse/pull/3465) +* Исправлена ошибка при использовании `SAMPLE`, `PREWHERE` и столбцов-алиасов. [#3543](https://github.com/yandex/ClickHouse/pull/3543) +* Исправлена редкая ошибка `unknown compression method` при использовании настройки `min_bytes_to_use_direct_io`. [3544](https://github.com/yandex/ClickHouse/pull/3544) + +### Улучшения производительности: +* Исправлена деградация производительности запросов с `GROUP BY` столбцов типа Int16, Date на процессорах AMD EPYC. [Игорь Лапко](https://github.com/yandex/ClickHouse/pull/3512) +* Исправлена деградация производительности при обработке длинных строк. [#3530](https://github.com/yandex/ClickHouse/pull/3530) + +### Улучшения процесса сборки ClickHouse: +* Доработки для упрощения сборки в Arcadia. [#3475](https://github.com/yandex/ClickHouse/pull/3475), [#3535](https://github.com/yandex/ClickHouse/pull/3535) + +## ClickHouse release 18.14.12, 2018-11-02 + +### Исправления ошибок: + +* Исправлена ошибка при join-запросе двух неименованных подзапросов. [#3505](https://github.com/yandex/ClickHouse/pull/3505) +* Исправлена генерация пустой `WHERE`-части при запросах к внешним базам. [hotid](https://github.com/yandex/ClickHouse/pull/3477) +* Исправлена ошибка использования неправильной настройки таймаута в ODBC-словарях. [Marek Vavruša](https://github.com/yandex/ClickHouse/pull/3511) + +## ClickHouse release 18.14.11, 2018-10-29 + +### Исправления ошибок: + +* Исправлена ошибка `Block structure mismatch in UNION stream: different number of columns` в запросах с LIMIT. [#2156](https://github.com/yandex/ClickHouse/issues/2156) +* Исправлены ошибки при слиянии данных в таблицах, содержащих массивы внутри Nested структур. [#3397](https://github.com/yandex/ClickHouse/pull/3397) +* Исправлен неправильный результат запросов при выключенной настройке `merge_tree_uniform_read_distribution` (включена по умолчанию). [#3429](https://github.com/yandex/ClickHouse/pull/3429) +* Исправлена ошибка при вставке в Distributed таблицу в формате Native. [#3411](https://github.com/yandex/ClickHouse/issues/3411) + +## ClickHouse release 18.14.10, 2018-10-23 + +* Настройка `compile_expressions` (JIT компиляция выражений) выключена по умолчанию. [#3410](https://github.com/yandex/ClickHouse/pull/3410) +* Настройка `enable_optimize_predicate_expression` выключена по умолчанию. + +## ClickHouse release 18.14.9, 2018-10-16 + +### Новые возможности: + +* Модификатор `WITH CUBE` для `GROUP BY` (также доступен синтаксис: `GROUP BY CUBE(...)`). [#3172](https://github.com/yandex/ClickHouse/pull/3172) +* Добавлена функция `formatDateTime`. [Alexandr Krasheninnikov](https://github.com/yandex/ClickHouse/pull/2770) +* Добавлен движок таблиц `JDBC` и табличная функция `jdbc` (для работы требуется установка clickhouse-jdbc-bridge). [Alexandr Krasheninnikov](https://github.com/yandex/ClickHouse/pull/3210) +* Добавлены функции для работы с ISO номером недели: `toISOWeek`, `toISOYear`, `toStartOfISOYear`, а также `toDayOfYear`. [#3146](https://github.com/yandex/ClickHouse/pull/3146) +* Добавлена возможность использования столбцов типа `Nullable` для таблиц типа `MySQL`, `ODBC`. [#3362](https://github.com/yandex/ClickHouse/pull/3362) +* Возможность чтения вложенных структур данных как вложенных объектов в формате `JSONEachRow`. Добавлена настройка `input_format_import_nested_json`. [Veloman Yunkan](https://github.com/yandex/ClickHouse/pull/3144) +* Возможность параллельной обработки многих `MATERIALIZED VIEW` при вставке данных. Настройка `parallel_view_processing`. [Marek Vavruša](https://github.com/yandex/ClickHouse/pull/3208) +* Добавлен запрос `SYSTEM FLUSH LOGS` (форсированный сброс логов в системные таблицы, такие как например, `query_log`) [#3321](https://github.com/yandex/ClickHouse/pull/3321) +* Возможность использования предопределённых макросов `database` и `table` в объявлении `Replicated` таблиц. [#3251](https://github.com/yandex/ClickHouse/pull/3251) +* Добавлена возможность чтения значения типа `Decimal` в инженерной нотации (с указанием десятичной экспоненты). [#3153](https://github.com/yandex/ClickHouse/pull/3153) + +### Экспериментальные возможности: + +* Оптимизация GROUP BY для типов данных `LowCardinality` [#3138](https://github.com/yandex/ClickHouse/pull/3138) +* Оптимизации вычисления выражений для типов данных `LowCardinality` [#3200](https://github.com/yandex/ClickHouse/pull/3200) + +### Улучшения: + +* Существенно уменьшено потребление памяти для запросов с `ORDER BY` и `LIMIT`. Настройка `max_bytes_before_remerge_sort`. [#3205](https://github.com/yandex/ClickHouse/pull/3205) +* При отсутствии указания типа `JOIN` (`LEFT`, `INNER`, ...), подразумевается `INNER JOIN`. [#3147](https://github.com/yandex/ClickHouse/pull/3147) +* Корректная работа квалифицированных звёздочек в запросах с `JOIN`. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/3202) +* Движок таблиц `ODBC` корректно выбирает способ квотирования идентификаторов в SQL диалекте удалённой СУБД. [Alexandr Krasheninnikov](https://github.com/yandex/ClickHouse/pull/3210) +* Настройка `compile_expressions` (JIT компиляция выражений) включена по-умолчанию. +* Исправлено поведение при одновременном DROP DATABASE/TABLE IF EXISTS и CREATE DATABASE/TABLE IF NOT EXISTS. Ранее запрос `CREATE DATABASE ... IF NOT EXISTS` мог выдавать сообщение об ошибке вида "File ... already exists", а запросы `CREATE TABLE ... IF NOT EXISTS` и `DROP TABLE IF EXISTS` могли выдавать сообщение `Table ... is creating or attaching right now`. [#3101](https://github.com/yandex/ClickHouse/pull/3101) +* Выражения LIKE и IN с константной правой частью пробрасываются на удалённый сервер при запросах из таблиц типа MySQL и ODBC. [#3182](https://github.com/yandex/ClickHouse/pull/3182) +* Сравнения с константными выражениями в секции WHERE пробрасываются на удалённый сервер при запросах из таблиц типа MySQL и ODBC. Ранее пробрасывались только сравнения с константами. [#3182](https://github.com/yandex/ClickHouse/pull/3182) +* Корректное вычисление ширины строк в терминале для `Pretty` форматов, в том числе для строк с иероглифами. [Amos Bird](https://github.com/yandex/ClickHouse/pull/3257). +* Возможность указания `ON CLUSTER` для запросов `ALTER UPDATE`. +* Увеличена производительность чтения данных в формате `JSONEachRow`. [#3332](https://github.com/yandex/ClickHouse/pull/3332) +* Добавлены синонимы функций `LENGTH`, `CHARACTER_LENGTH` для совместимости. Функция `CONCAT` стала регистронезависимой. [#3306](https://github.com/yandex/ClickHouse/pull/3306) +* Добавлен синоним `TIMESTAMP` для типа `DateTime`. [#3390](https://github.com/yandex/ClickHouse/pull/3390) +* В логах сервера всегда присутствует место для query_id, даже если строчка лога не относится к запросу. Это сделано для более простого парсинга текстовых логов сервера сторонними инструментами. +* Логгирование потребления памяти запросом при превышении очередной отметки целого числа гигабайт. [#3205](https://github.com/yandex/ClickHouse/pull/3205) +* Добавлен режим совместимости для случая, когда клиентская библиотека, работающая по Native протоколу, по ошибке отправляет меньшее количество столбцов, чем сервер ожидает для запроса INSERT. Такой сценарий был возможен при использовании библиотеки clickhouse-cpp. Ранее этот сценарий приводил к падению сервера. [#3171](https://github.com/yandex/ClickHouse/pull/3171) +* В `clickhouse-copier`, в задаваемом пользователем выражении WHERE, появилась возможность использовать алиас `partition_key` (для дополнительной фильтрации по партициям исходной таблицы). Это полезно, если схема партиционирования изменяется при копировании, но изменяется незначительно. [#3166](https://github.com/yandex/ClickHouse/pull/3166) +* Рабочий поток движка `Kafka` перенесён в фоновый пул потоков для того, чтобы автоматически уменьшать скорость чтения данных при большой нагрузке. [Marek Vavruša](https://github.com/yandex/ClickHouse/pull/3215). +* Поддержка чтения значений типа `Tuple` и `Nested` структур как `struct` в формате `Cap'n'Proto` [Marek Vavruša](https://github.com/yandex/ClickHouse/pull/3216). +* В список доменов верхнего уровня для функции `firstSignificantSubdomain` добавлен домен `biz` [decaseal](https://github.com/yandex/ClickHouse/pull/3219). +* В конфигурации внешних словарей, пустое значение `null_value` интерпретируется, как значение типа данных по-умоланию. [#3330](https://github.com/yandex/ClickHouse/pull/3330) +* Поддержка функций `intDiv`, `intDivOrZero` для `Decimal`. [b48402e8](https://github.com/yandex/ClickHouse/commit/b48402e8712e2b9b151e0eef8193811d433a1264) +* Поддержка типов `Date`, `DateTime`, `UUID`, `Decimal` в качестве ключа для агрегатной функции `sumMap`. [#3281](https://github.com/yandex/ClickHouse/pull/3281) +* Поддержка типа данных `Decimal` во внешних словарях. [#3324](https://github.com/yandex/ClickHouse/pull/3324) +* Поддержка типа данных `Decimal` в таблицах типа `SummingMergeTree`. [#3348](https://github.com/yandex/ClickHouse/pull/3348) +* Добавлена специализация для `UUID` в функции `if`. [#3366](https://github.com/yandex/ClickHouse/pull/3366) +* Уменьшено количество системных вызовов `open`, `close` при чтении из таблиц семейства `MergeTree` [#3283](https://github.com/yandex/ClickHouse/pull/3283). +* Возможность выполнения запроса `TRUNCATE TABLE` на любой реплике (запрос пробрасывается на реплику-лидера). [Kirill Shvakov](https://github.com/yandex/ClickHouse/pull/3375) + +### Исправление ошибок: + +* Исправлена ошибка в работе таблиц типа `Dictionary` для словарей типа `range_hashed`. Ошибка возникла в версии 18.12.17. [#1702](https://github.com/yandex/ClickHouse/pull/1702) +* Исправлена ошибка при загрузке словарей типа `range_hashed` (сообщение `Unsupported type Nullable(...)`). Ошибка возникла в версии 18.12.17. [#3362](https://github.com/yandex/ClickHouse/pull/3362) +* Исправлена некорректная работа функции `pointInPolygon` из-за накопления погрешности при вычислениях для полигонов с большим количеством близко расположенных вершин. [#3331](https://github.com/yandex/ClickHouse/pull/3331) [#3341](https://github.com/yandex/ClickHouse/pull/3341) +* Если после слияния кусков данных, у результирующего куска чексумма отличается от результата того же слияния на другой реплике, то результат слияния удаляется, и вместо этого кусок скачивается с другой реплики (это правильное поведение). Но после скачивания куска, он не мог добавиться в рабочий набор из-за ошибки, что кусок уже существует (так как кусок после слияния удалялся не сразу, а с задержкой). Это приводило к циклическим попыткам скачивания одних и тех же данных. [#3194](https://github.com/yandex/ClickHouse/pull/3194) +* Исправлен некорректный учёт общего потребления оперативной памяти запросами (что приводило к неправильной работе настройки `max_memory_usage_for_all_queries` и неправильному значению метрики `MemoryTracking`). Ошибка возникла в версии 18.12.13. [Marek Vavruša](https://github.com/yandex/ClickHouse/pull/3344) +* Исправлена работоспособность запросов `CREATE TABLE ... ON CLUSTER ... AS SELECT ...` Ошибка возникла в версии 18.12.13. [#3247](https://github.com/yandex/ClickHouse/pull/3247) +* Исправлена лишняя подготовка структуры данных для `JOIN` на сервере-инициаторе запроса, если `JOIN` выполняется только на удалённых серверах. [#3340](https://github.com/yandex/ClickHouse/pull/3340) +* Исправлены ошибки в движке `Kafka`: неработоспособность после исключения при начале чтения данных; блокировка при завершении [Marek Vavruša](https://github.com/yandex/ClickHouse/pull/3215). +* Для таблиц `Kafka` не передавался опциональный параметр `schema` (схема формата `Cap'n'Proto`). [Vojtech Splichal](https://github.com/yandex/ClickHouse/pull/3150) +* Если ансамбль серверов ZooKeeper содержит серверы, которые принимают соединение, но сразу же разрывают его вместо ответа на рукопожатие, то ClickHouse выбирает для соединения другой сервер. Ранее в этом случае возникала ошибка `Cannot read all data. Bytes read: 0. Bytes expected: 4.` и сервер не мог стартовать. [8218cf3a](https://github.com/yandex/ClickHouse/commit/8218cf3a5f39a43401953769d6d12a0bb8d29da9) +* Если ансамбль серверов ZooKeeper содержит серверы, для которых DNS запрос возвращает ошибку, то такие серверы пропускаются. [17b8e209](https://github.com/yandex/ClickHouse/commit/17b8e209221061325ad7ba0539f03c6e65f87f29) +* Исправлено преобразование типов между `Date` и `DateTime` при вставке данных в формате `VALUES` (в случае, когда `input_format_values_interpret_expressions = 1`). Ранее преобразование производилось между числовым значением количества дней с начала unix эпохи и unix timestamp, что приводило к неожиданным результатам. [#3229](https://github.com/yandex/ClickHouse/pull/3229) +* Исправление преобразования типов между `Decimal` и целыми числами. [#3211](https://github.com/yandex/ClickHouse/pull/3211) +* Исправлены ошибки в работе настройки `enable_optimize_predicate_expression`. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/3231) +* Исправлена ошибка парсинга формата CSV с числами с плавающей запятой, если используется разделитель CSV не по-умолчанию, такой как например, `;` [#3155](https://github.com/yandex/ClickHouse/pull/3155). +* Исправлена функция `arrayCumSumNonNegative` (она не накапливает отрицательные значения, если аккумулятор становится меньше нуля). [Aleksey Studnev](https://github.com/yandex/ClickHouse/pull/3163) +* Исправлена работа `Merge` таблицы поверх `Distributed` таблиц при использовании `PREWHERE`. [#3165](https://github.com/yandex/ClickHouse/pull/3165) +* Исправления ошибок в запросе `ALTER UPDATE`. +* Исправления ошибок в табличной функции `odbc`, которые возникли в версии 18.12. [#3197](https://github.com/yandex/ClickHouse/pull/3197) +* Исправлена работа агрегатных функций с комбинаторами `StateArray`. [#3188](https://github.com/yandex/ClickHouse/pull/3188) +* Исправлено падение при делении значения типа `Decimal` на ноль. [69dd6609](https://github.com/yandex/ClickHouse/commit/69dd6609193beb4e7acd3e6ad216eca0ccfb8179) +* Исправлен вывод типов для операций с использованием аргументов типа `Decimal` и целых чисел. [#3224](https://github.com/yandex/ClickHouse/pull/3224) +* Исправлен segfault при `GROUP BY` по `Decimal128`. [3359ba06](https://github.com/yandex/ClickHouse/commit/3359ba06c39fcd05bfdb87d6c64154819621e13a) +* Настройка `log_query_threads` (логгирование информации о каждом потоке исполнения запроса) теперь имеет эффект только если настройка `log_queries` (логгирование информации о запросах) выставлена в 1. Так как настройка `log_query_threads` включена по-умолчанию, ранее информация о потоках логгировалась даже если логгирование запросов выключено. [#3241](https://github.com/yandex/ClickHouse/pull/3241) +* Исправлена ошибка в распределённой работе агрегатной функции quantiles (сообщение об ошибке вида `Not found column quantile...`). [292a8855](https://github.com/yandex/ClickHouse/commit/292a885533b8e3b41ce8993867069d14cbd5a664) +* Исправлена проблема совместимости при одновременной работе на кластере серверов версии 18.12.17 и более старых, приводящая к тому, что при распределённых запросах с GROUP BY по ключам одновременно фиксированной и не фиксированной длины, при условии, что количество данных в процессе агрегации большое, могли возвращаться не до конца агрегированные данные (одни и те же ключи агрегации в двух разных строках). [#3254](https://github.com/yandex/ClickHouse/pull/3254) +* Исправлена обработка подстановок в `clickhouse-performance-test`, если запрос содержит только часть из объявленных в тесте подстановок. [#3263](https://github.com/yandex/ClickHouse/pull/3263) +* Исправлена ошибка при использовании `FINAL` совместно с `PREWHERE`. [#3298](https://github.com/yandex/ClickHouse/pull/3298) +* Исправлена ошибка при использовании `PREWHERE` над столбцами, добавленными при `ALTER`. [#3298](https://github.com/yandex/ClickHouse/pull/3298) +* Добавлена проверка отсутствия `arrayJoin` для `DEFAULT`, `MATERIALIZED` выражений. Ранее наличие `arrayJoin` приводило к ошибке при вставке данных. [#3337](https://github.com/yandex/ClickHouse/pull/3337) +* Добавлена проверка отсутствия `arrayJoin` в секции `PREWHERE`. Ранее это приводило к сообщениям вида `Size ... doesn't match` или `Unknown compression method` при выполнении запросов. [#3357](https://github.com/yandex/ClickHouse/pull/3357) +* Исправлен segfault, который мог возникать в редких случаях после оптимизации - замены цепочек AND из равенства выражения константам на соответствующее выражение IN. [liuyimin-bytedance](https://github.com/yandex/ClickHouse/pull/3339). +* Мелкие исправления `clickhouse-benchmark`: ранее информация о клиенте не передавалась на сервер; более корректный подсчёт числа выполненных запросов при завершении работы и для ограничения числа итераций. [#3351](https://github.com/yandex/ClickHouse/pull/3351) [#3352](https://github.com/yandex/ClickHouse/pull/3352) + +### Обратно несовместимые изменения: + +* Удалена настройка `allow_experimental_decimal_type`. Тип данных `Decimal` доступен для использования по-умолчанию. [#3329](https://github.com/yandex/ClickHouse/pull/3329) + + +## ClickHouse release 18.12.17, 2018-09-16 + +### Новые возможности: + +* `invalidate_query` (возможность задать запрос для проверки необходимости обновления внешнего словаря) реализована для источника `clickhouse`. [#3126](https://github.com/yandex/ClickHouse/pull/3126) +* Добавлена возможность использования типов данных `UInt*`, `Int*`, `DateTime` (наравне с типом `Date`) в качестве ключа внешнего словаря типа `range_hashed`, определяющего границы диапазонов. Возможность использования `NULL` в качестве обозначения открытого диапазона. [Vasily Nemkov](https://github.com/yandex/ClickHouse/pull/3123) +* Для типа `Decimal` добавлена поддержка агрегатных функций `var*`, `stddev*`. [#3129](https://github.com/yandex/ClickHouse/pull/3129) +* Для типа `Decimal` добавлена поддержка математических функций (`exp`, `sin` и т. п.) [#3129](https://github.com/yandex/ClickHouse/pull/3129) +* В таблицу `system.part_log` добавлен столбец `partition_id`. [#3089](https://github.com/yandex/ClickHouse/pull/3089) + +### Исправление ошибок: + +* Исправлена работа `Merge` таблицы поверх `Distributed` таблиц. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/3159) +* Исправлена несовместимость (лишняя зависимость от версии `glibc`), приводящая к невозможности запуска ClickHouse на `Ubuntu Precise` и более старых. Несовместимость возникла в версии 18.12.13. [#3130](https://github.com/yandex/ClickHouse/pull/3130) +* Исправлены ошибки в работе настройки `enable_optimize_predicate_expression`. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/3107) +* Исправлено незначительное нарушение обратной совместимости, проявляющееся при одновременной работе на кластере реплик версий до 18.12.13 и создании новой реплики таблицы на сервере более новой версии (выдаётся сообщение `Can not clone replica, because the ... updated to new ClickHouse version`, что полностью логично, но не должно было происходить). [#3122](https://github.com/yandex/ClickHouse/pull/3122) + +### Обратно несовместимые изменения: + +* Настройка `enable_optimize_predicate_expression` включена по-умолчанию, что конечно очень оптимистично. При возникновении ошибок анализа запроса, связанных с поиском имён столбцов, следует выставить `enable_optimize_predicate_expression` в 0. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/3107) + + +## ClickHouse release 18.12.14, 2018-09-13 + +### Новые возможности: + +* Добавлена поддержка запросов `ALTER UPDATE`. [#3035](https://github.com/yandex/ClickHouse/pull/3035) +* Добавлена настройка `allow_ddl`, упраляющая доступом пользователя к DDL-запросам. [#3104](https://github.com/yandex/ClickHouse/pull/3104) +* Добавлена настройка `min_merge_bytes_to_use_direct_io` для движков семейства `MergeTree`, позволяющая задать порог на суммарный размер слияния после которого работа с файлами кусков будет происходить с O_DIRECT. [#3117](https://github.com/yandex/ClickHouse/pull/3117) +* В системную таблицу `system.merges` добавлен столбец `partition_id`. [#3099](https://github.com/yandex/ClickHouse/pull/3099) + +### Улучшения + +* Если в процессе мутации кусок остался неизменённым, он не будет скачан репликами. [#3103](https://github.com/yandex/ClickHouse/pull/3103) +* При работе с `clickhouse-client` добавлено автодополнение для имён настроек. [#3106](https://github.com/yandex/ClickHouse/pull/3106) + +### Исправление ошибок + +* Добавлена проверка размеров массивов, которые являются элементами полей типа `Nested`, при вставке. [#3118](https://github.com/yandex/ClickHouse/pull/3118) +* Исправлена ошибка обновления внешних словарей с источником `ODBC` и форматом хранения `hashed`. Ошибка возникла в версии 18.12.13. +* Исправлено падение при создании временной таблицы таблицы из запроса с условием `IN`. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/3098) +* Исправлена ошибка в работе агрегатных функций для массивов, элементами которых может быть `NULL`. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/3097) + + +## ClickHouse release 18.12.13, 2018-09-10 + +### Новые возможности: + +* Добавлен тип данных `DECIMAL(digits, scale)` (`Decimal32(scale)`, `Decimal64(scale)`, `Decimal128(scale)`). Возможность доступна под настройкой `allow_experimental_decimal_type`. [#2846](https://github.com/yandex/ClickHouse/pull/2846) [#2970](https://github.com/yandex/ClickHouse/pull/2970) [#3008](https://github.com/yandex/ClickHouse/pull/3008) [#3047](https://github.com/yandex/ClickHouse/pull/3047) +* Модификатор `WITH ROLLUP` для `GROUP BY` (также доступен синтаксис: `GROUP BY ROLLUP(...)`). [#2948](https://github.com/yandex/ClickHouse/pull/2948) +* В запросах с JOIN, звёздочка раскрывается в список столбцов всех таблиц, в соответствии со стандартом SQL. Вернуть старое поведение можно, выставив настройку (уровня пользователя) `asterisk_left_columns_only` в значение 1. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2787) +* Добавлена поддержка JOIN с табличной функцией. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2907) +* Автодополнение по нажатию Tab в clickhouse-client. [Sergey Shcherbin](https://github.com/yandex/ClickHouse/pull/2447) +* Нажатие Ctrl+C в clickhouse-client очищает запрос, если он был введён. [#2877](https://github.com/yandex/ClickHouse/pull/2877) +* Добавлена настройка `join_default_strictness` (значения `''`, `'any'`, `'all'`). Её использование позволяет не указывать `ANY` или `ALL` для `JOIN`. [#2982](https://github.com/yandex/ClickHouse/pull/2982) +* В каждой строчке лога сервера, относящейся к обработке запроса, выводится идентификатор запроса. [#2482](https://github.com/yandex/ClickHouse/pull/2482) +* Возможность получения логов выполнения запроса в clickhouse-client (настройка `send_logs_level`). При распределённой обработке запроса, логи отправляются каскадно со всех серверов. [#2482](https://github.com/yandex/ClickHouse/pull/2482) +* В таблицах `system.query_log` и `system.processes` (`SHOW PROCESSLIST`) появилась информация о всех изменённых настройках при выполнении запроса (вложенная структура данных `Settings`). Добавлена настройка `log_query_settings`. [#2482](https://github.com/yandex/ClickHouse/pull/2482) +* В таблицах `system.query_log` и `system.processes` появилась информация о номерах потоков, участвующих в исполнении запроса (столбец `thread_numbers`). [#2482](https://github.com/yandex/ClickHouse/pull/2482) +* Добавлены счётчики `ProfileEvents`, измеряющие время, потраченное на чтение и запись по сети; чтение и запись на диск; количество сетевых ошибок; время потраченное на ожидании при ограничении сетевой полосы. [#2482](https://github.com/yandex/ClickHouse/pull/2482) +* Добавлены счётчики `ProfileEvents`, содержащие системные метрики из rusage (позволяющие получить информацию об использовании CPU в userspace и ядре, page faults, context switches) а также метрики taskstats (позволяющие получить информацию о времени ожидания IO, CPU, а также количество прочитанных и записанных данных с учётом и без учёта page cache). [#2482](https://github.com/yandex/ClickHouse/pull/2482) +* Счётчики `ProfileEvents` учитываются не только глобально, но и на каждый запрос, а также на каждый поток выполнения запроса, что позволяет детально профилировать потребление ресурсов отдельными запросами. [#2482](https://github.com/yandex/ClickHouse/pull/2482) +* Добавлена таблица `system.query_thread_log`, содержащая информацию о каждом потоке выполнения запроса. Добавлена настройка `log_query_threads`. [#2482](https://github.com/yandex/ClickHouse/pull/2482) +* В таблицах `system.metrics` и `system.events` появилась встроенная документация. [#3016](https://github.com/yandex/ClickHouse/pull/3016) +* Добавлена функция `arrayEnumerateDense`. [Amos Bird](https://github.com/yandex/ClickHouse/pull/2975) +* Добавлены функции `arrayCumSumNonNegative` и `arrayDifference`. [Aleksey Studnev](https://github.com/yandex/ClickHouse/pull/2942) +* Добавлена агрегатная функция `retention`. [Sundy Li](https://github.com/yandex/ClickHouse/pull/2887) +* Возможность сложения (слияния) состояний агрегатных функций с помощью оператора плюс, а также умножения состояний агрегатных функций на целую неотрицательную константу. [#3062](https://github.com/yandex/ClickHouse/pull/3062) [#3034](https://github.com/yandex/ClickHouse/pull/3034) +* В таблицах семейства MergeTree добавлен виртуальный столбец `_partition_id`. [#3089](https://github.com/yandex/ClickHouse/pull/3089) + +### Экспериментальные возможности: + +* Добавлен тип данных `LowCardinality(T)`. Тип данных автоматически создаёт локальный словарь значений и позволяет обрабатывать данные без распаковки словаря. [#2830](https://github.com/yandex/ClickHouse/pull/2830) +* Добавлен кэш JIT-скомпилированных функций, а также счётчик числа использований перед компиляцией. Возможность JIT-компиляции выражений включается настройкой `compile_expressions`. [#2990](https://github.com/yandex/ClickHouse/pull/2990) [#3077](https://github.com/yandex/ClickHouse/pull/3077) + +### Улучшения: + +* Исправлена проблема неограниченного накопления лога репликации в случае наличия заброшенных реплик. Добавлен режим эффективного восстановления реплик после длительного отставания. +* Увеличена производительность при выполнении `GROUP BY` в случае, если есть несколько полей агрегации, одно из которых строковое, а другие - фиксированной длины. +* Увеличена производительность при использовании `PREWHERE` и при неявном переносе выражений в `PREWHERE`. +* Увеличена производительность парсинга текстовых форматов (`CSV`, `TSV`). [Amos Bird](https://github.com/yandex/ClickHouse/pull/2977) [#2980](https://github.com/yandex/ClickHouse/pull/2980) +* Увеличена производительность чтения строк и массивов в бинарных форматах. [Amos Bird](https://github.com/yandex/ClickHouse/pull/2955) +* Увеличена производительность и уменьшено потребление памяти в запросах к таблицам `system.tables` и `system.columns` в случае наличия очень большого количества таблиц на одном сервере. [#2953](https://github.com/yandex/ClickHouse/pull/2953) +* Исправлена проблема низкой производительности в случае наличия большого потока запросов, для которых возвращается ошибка (в `perf top` видна функция `_dl_addr`, при этом сервер использует мало CPU). [#2938](https://github.com/yandex/ClickHouse/pull/2938) +* Прокидывание условий внутрь View (при включенной настройке `enable_optimize_predicate_expression`) [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2907) +* Доработки недостающей функциональности для типа данных `UUID`. [#3074](https://github.com/yandex/ClickHouse/pull/3074) [#2985](https://github.com/yandex/ClickHouse/pull/2985) +* Тип данных `UUID` поддержан в словарях The-Alchemist. [#2822](https://github.com/yandex/ClickHouse/pull/2822) +* Функция `visitParamExtractRaw` корректно работает с вложенными структурами. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2974) +* При использовании настройки `input_format_skip_unknown_fields` корректно работает пропуск значений-объектов в формате `JSONEachRow`. [BlahGeek](https://github.com/yandex/ClickHouse/pull/2958) +* Для выражения `CASE` с условиями, появилась возможность не указывать `ELSE`, что эквивалентно `ELSE NULL`. [#2920](https://github.com/yandex/ClickHouse/pull/2920) +* Возможность конфигурирования operation timeout при работе с ZooKeeper. [urykhy](https://github.com/yandex/ClickHouse/pull/2971) +* Возможность указания смещения для `LIMIT n, m` в виде `LIMIT n OFFSET m`. [#2840](https://github.com/yandex/ClickHouse/pull/2840) +* Возможность использования синтаксиса `SELECT TOP n` в качестве альтернативы для `LIMIT`. [#2840](https://github.com/yandex/ClickHouse/pull/2840) +* Увеличен размер очереди записи в системные таблицы, что позволяет уменьшить количество ситуаций `SystemLog queue is full`. +* В агрегатной функции `windowFunnel` добавлена поддержка событий, подходящих под несколько условий. [Amos Bird](https://github.com/yandex/ClickHouse/pull/2801) +* Возможность использования дублирующихся столбцов в секции `USING` для `JOIN`. [#3006](https://github.com/yandex/ClickHouse/pull/3006) +* Для форматов `Pretty` введено ограничение выравнивания столбцов по ширине. Настройка `output_format_pretty_max_column_pad_width`. В случае более широкого значения, оно всё ещё будет выведено целиком, но остальные ячейки таблицы не будут излишне широкими. [#3003](https://github.com/yandex/ClickHouse/pull/3003) +* В табличной функции `odbc` добавлена возможность указания имени базы данных/схемы. [Amos Bird](https://github.com/yandex/ClickHouse/pull/2885) +* Добавлена возможность использования имени пользователя, заданного в конфигурационном файле `clickhouse-client`. [Vladimir Kozbin](https://github.com/yandex/ClickHouse/pull/2909) +* Счётчик `ZooKeeperExceptions` разделён на три счётчика `ZooKeeperUserExceptions`, `ZooKeeperHardwareExceptions`, `ZooKeeperOtherExceptions`. +* Запросы `ALTER DELETE` работают для материализованных представлений. +* Добавлена рандомизация во времени периодического запуска cleanup thread для таблиц типа `ReplicatedMergeTree`, чтобы избежать периодических всплесков нагрузки в случае очень большого количества таблиц типа `ReplicatedMergeTree`. +* Поддержка запроса `ATTACH TABLE ... ON CLUSTER`. [#3025](https://github.com/yandex/ClickHouse/pull/3025) + +### Исправление ошибок: + +* Исправлена ошибка в работе таблиц типа `Dictionary` (кидается исключение `Size of offsets doesn't match size of column` или `Unknown compression method`). Ошибка появилась в версии 18.10.3. [#2913](https://github.com/yandex/ClickHouse/issues/2913) +* Исправлена ошибка при мерже данных таблиц типа `CollapsingMergeTree`, если один из кусков данных пустой (такие куски, в свою очередь, образуются при слиянии или при `ALTER DELETE` в случае удаления всех данных), и для слияния был выбран алгоритм `vertical`. [#3049](https://github.com/yandex/ClickHouse/pull/3049) +* Исправлен race condition при `DROP` или `TRUNCATE` таблиц типа `Memory` при одновременном `SELECT`, который мог приводить к падениям сервера. Ошибка появилась в версии 1.1.54388. [#3038](https://github.com/yandex/ClickHouse/pull/3038) +* Исправлена возможность потери данных при вставке в `Replicated` таблицы в случае получения ошибки `Session expired` (потеря данных может быть обнаружена по метрике `ReplicatedDataLoss`). Ошибка возникла в версии 1.1.54378. [#2939](https://github.com/yandex/ClickHouse/pull/2939) [#2949](https://github.com/yandex/ClickHouse/pull/2949) [#2964](https://github.com/yandex/ClickHouse/pull/2964) +* Исправлен segfault при `JOIN ... ON`. [#3000](https://github.com/yandex/ClickHouse/pull/3000) +* Исправлена ошибка поиска имён столбцов в случае, если выражение `WHERE` состоит целиком из квалифицированного имени столбца, как например `WHERE table.column`. [#2994](https://github.com/yandex/ClickHouse/pull/2994) +* Исправлена ошибка вида "Not found column" при выполнении распределённых запросов в случае, если с удалённого сервера запрашивается единственный столбец, представляющий собой выражение IN с подзапросом. [#3087](https://github.com/yandex/ClickHouse/pull/3087) +* Исправлена ошибка `Block structure mismatch in UNION stream: different number of columns`, возникающая при распределённых запросах, если один из шардов локальный, а другой - нет, и если при этом срабатывает оптимизация переноса в `PREWHERE`. [#2226](https://github.com/yandex/ClickHouse/pull/2226) [#3037](https://github.com/yandex/ClickHouse/pull/3037) [#3055](https://github.com/yandex/ClickHouse/pull/3055) [#3065](https://github.com/yandex/ClickHouse/pull/3065) [#3073](https://github.com/yandex/ClickHouse/pull/3073) [#3090](https://github.com/yandex/ClickHouse/pull/3090) [#3093](https://github.com/yandex/ClickHouse/pull/3093) +* Исправлена работа функции `pointInPolygon` для некоторого случая невыпуклых полигонов. [#2910](https://github.com/yandex/ClickHouse/pull/2910) +* Исправлен некорректный результат при сравнении `nan` с целыми числами. [#3024](https://github.com/yandex/ClickHouse/pull/3024) +* Исправлена ошибка в библиотеке `zlib-ng`, которая могла приводить к segfault в редких случаях. [#2854](https://github.com/yandex/ClickHouse/pull/2854) +* Исправлена утечка памяти при вставке в таблицу со столбцами типа `AggregateFunction`, если состояние агрегатной функции нетривиальное (выделяет память отдельно), и если в одном запросе на вставку получается несколько маленьких блоков. [#3084](https://github.com/yandex/ClickHouse/pull/3084) +* Исправлен race condition при одновременном создании и удалении одной и той же таблицы типа `Buffer` или `MergeTree`. +* Исправлена возможность segfault при сравнении кортежей из некоторых нетривиальных типов, таких как, например, кортежей. [#2989](https://github.com/yandex/ClickHouse/pull/2989) +* Исправлена возможность segfault при выполнении некоторых запросов `ON CLUSTER`. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2960) +* Исправлена ошибка в функции `arrayDistinct` в случае `Nullable` элементов массивов. [#2845](https://github.com/yandex/ClickHouse/pull/2845) [#2937](https://github.com/yandex/ClickHouse/pull/2937) +* Возможность `enable_optimize_predicate_expression` корректно поддерживает случаи с `SELECT *`. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2929) +* Исправлена возможность segfault при переинициализации сессии с ZooKeeper. [#2917](https://github.com/yandex/ClickHouse/pull/2917) +* Исправлена возможность блокировки при взаимодействии с ZooKeeper. +* Исправлен некорректный код суммирования вложенных структур данных в `SummingMergeTree`. +* При выделении памяти для состояний агрегатных функций, корректно учитывается выравнивание, что позволяет использовать при реализации состояний агрегатных функций операции, для которых выравнивание является необходимым. [chenxing-xc](https://github.com/yandex/ClickHouse/pull/2808) + +### Исправления безопасности: + +* Безопасная работа с ODBC источниками данных. Взаимодействие с ODBC драйверами выполняется через отдельный процесс `clickhouse-odbc-bridge`. Ошибки в сторонних ODBC драйверах теперь не приводят к проблемам со стабильностью сервера или уязвимостям. [#2828](https://github.com/yandex/ClickHouse/pull/2828) [#2879](https://github.com/yandex/ClickHouse/pull/2879) [#2886](https://github.com/yandex/ClickHouse/pull/2886) [#2893](https://github.com/yandex/ClickHouse/pull/2893) [#2921](https://github.com/yandex/ClickHouse/pull/2921) +* Исправлена некорректная валидация пути к файлу в табличной функции `catBoostPool`. [#2894](https://github.com/yandex/ClickHouse/pull/2894) +* Содержимое системных таблиц (`tables`, `databases`, `parts`, `columns`, `parts_columns`, `merges`, `mutations`, `replicas`, `replication_queue`) фильтруется согласно конфигурации доступа к базам данных для пользователя (`allow_databases`) [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2856) + +### Обратно несовместимые изменения: + +* В запросах с JOIN, звёздочка раскрывается в список столбцов всех таблиц, в соответствии со стандартом SQL. Вернуть старое поведение можно, выставив настройку (уровня пользователя) `asterisk_left_columns_only` в значение 1. + +### Изменения сборки: + +* Добавлен покоммитный запуск большинства интеграционных тестов. +* Добавлен покоммитный запуск проверки стиля кода. +* Корректный выбор реализации `memcpy` при сборке на CentOS7 / Fedora. [Etienne Champetier](https://github.com/yandex/ClickHouse/pull/2912) +* При сборке с помощью clang добавлены некоторые warnings из `-Weverything` в дополнение к обычным `-Wall -Wextra -Werror`. [#2957](https://github.com/yandex/ClickHouse/pull/2957) +* При debug сборке используется debug вариант `jemalloc`. +* Абстрагирован интерфейс библиотеки для взаимодействия с ZooKeeper. [#2950](https://github.com/yandex/ClickHouse/pull/2950) + + +## ClickHouse release 18.10.3, 2018-08-13 + +### Новые возможности: +* Возможность использования HTTPS для репликации. [#2760](https://github.com/yandex/ClickHouse/pull/2760) +* Добавлены функции `murmurHash2_64`, `murmurHash3_32`, `murmurHash3_64`, `murmurHash3_128` в дополнение к имеющемуся `murmurHash2_32`. [#2791](https://github.com/yandex/ClickHouse/pull/2791) +* Поддержка Nullable типов в ODBC драйвере ClickHouse (формат вывода `ODBCDriver2`) [#2834](https://github.com/yandex/ClickHouse/pull/2834) +* Поддержка `UUID` в ключевых столбцах. + +### Улучшения: +* Удаление кластеров без перезагрузки сервера при их удалении из конфигурационных файлов. [#2777](https://github.com/yandex/ClickHouse/pull/2777) +* Удаление внешних словарей без перезагрузки сервера при их удалении из конфигурационных файлов. [#2779](https://github.com/yandex/ClickHouse/pull/2779) +* Добавлена поддержка `SETTINGS` для движка таблиц `Kafka`. [Alexander Marshalov](https://github.com/yandex/ClickHouse/pull/2781) +* Доработки для типа данных `UUID` (не полностью) Šimon Podlipský. [#2618](https://github.com/yandex/ClickHouse/pull/2618) +* Поддежка пустых кусков после мержей в движках `SummingMergeTree`, `CollapsingMergeTree` and `VersionedCollapsingMergeTree`. [#2815](https://github.com/yandex/ClickHouse/pull/2815) +* Удаление старых записей о полностью выполнившихся мутациях (`ALTER DELETE`) [#2784](https://github.com/yandex/ClickHouse/pull/2784) +* Добавлена таблица `system.merge_tree_settings`. [Kirill Shvakov](https://github.com/yandex/ClickHouse/pull/2841) +* В таблицу `system.tables` добавлены столбцы зависимостей: `dependencies_database` и `dependencies_table`. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2851) +* Добавлена опция конфига `max_partition_size_to_drop`. [#2782](https://github.com/yandex/ClickHouse/pull/2782) +* Добавлена настройка `output_format_json_escape_forward_slashes`. [Alexander Bocharov](https://github.com/yandex/ClickHouse/pull/2812) +* Добавлена настройка `max_fetch_partition_retries_count`. [#2831](https://github.com/yandex/ClickHouse/pull/2831) +* Добавлена настройка `prefer_localhost_replica`, позволяющая отключить предпочтение локальной реплики и хождение на локальную реплику без межпроцессного взаимодействия. [#2832](https://github.com/yandex/ClickHouse/pull/2832) +* Агрегатная функция `quantileExact` возвращает `nan` в случае агрегации по пустому множеству `Float32`/`Float64` типов. [Sundy Li](https://github.com/yandex/ClickHouse/pull/2855) + +### Исправление ошибок: +* Убрано излишнее экранирование параметров connection string для ODBC, котрое приводило к невозможности соединения. Ошибка возникла в версии 18.6.0. +* Исправлена логика обработки команд на `REPLACE PARTITION` в очереди репликации. Неправильная логика могла приводить к тому, что при наличии двух `REPLACE` одной и той же партиции, один из них оставался в очереди репликации и не мог выполниться. [#2814](https://github.com/yandex/ClickHouse/pull/2814) +* Исправлена ошибка при мерже, если все куски были пустыми (такие куски, в свою очередь, образуются при слиянии или при `ALTER DELETE` в случае удаления всех данных). Ошибка появилась в версии 18.1.0. [#2930](https://github.com/yandex/ClickHouse/pull/2930) +* Исправлена ошибка при параллельной записи в таблицы типа `Set` или `Join`. [Amos Bird](https://github.com/yandex/ClickHouse/pull/2823) +* Исправлена ошибка `Block structure mismatch in UNION stream: different number of columns`, возникающая при запросах с `UNION ALL` внутри подзапроса, в случае, если один из `SELECT` запросов содержит дублирующиеся имена столбцов. [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2094) +* Исправлена утечка памяти в случае исключения при соединении с MySQL сервером. +* Исправлен некорректный код возврата clickhouse-client в случае ошибочного запроса +* Исправлен некорректная работа materialized views, содержащих DISTINCT. [#2795](https://github.com/yandex/ClickHouse/issues/2795) + +### Обратно несовместимые изменения +* Убрана поддержка запросов CHECK TABLE для Distributed таблиц. + +### Изменения сборки: +* Заменен аллокатор, теперь используется `jemalloc` вместо `tcmalloc`. На некоторых сценариях ускорение достигает 20%. В то же время, существуют запросы, замедлившиеся до 20%. Потребление памяти на некоторых сценариях примерно на 10% меньше и более стабильно. При высококонкурентной нагрузке, потребление CPU в userspace и в system незначительно вырастает. [#2773](https://github.com/yandex/ClickHouse/pull/2773) +* Использование libressl из submodule. [#1983](https://github.com/yandex/ClickHouse/pull/1983) [#2807](https://github.com/yandex/ClickHouse/pull/2807) +* Использование unixodbc из submodule. [#2789](https://github.com/yandex/ClickHouse/pull/2789) +* Использование mariadb-connector-c из submodule. [#2785](https://github.com/yandex/ClickHouse/pull/2785) +* В репозиторий добавлены файлы функциональных тестов, рассчитывающих на наличие тестовых данных (пока без самих тестовых данных). + + +## ClickHouse release 18.6.0, 2018-08-02 + +### Новые возможности: +* Добавлена поддержка ON выражений для JOIN ON синтаксиса: +`JOIN ON Expr([table.]column, ...) = Expr([table.]column, ...) [AND Expr([table.]column, ...) = Expr([table.]column, ...) ...]` +Выражение должно представлять из себя цепочку равенств, объединенных оператором AND. Каждая часть равенства может являться произвольным выражением над столбцами одной из таблиц. Поддержана возможность использования fully qualified имен столбцов (`table.name`, `database.table.name`, `table_alias.name`, `subquery_alias.name`) для правой таблицы. [#2742](https://github.com/yandex/ClickHouse/pull/2742) +* Добавлена возможность включить HTTPS для репликации. [#2760](https://github.com/yandex/ClickHouse/pull/2760) + +### Улучшения: +* Сервер передаёт на клиент также patch-компонент своей версии. Данные о patch компоненте версии добавлены в `system.processes` и `query_log`. [#2646](https://github.com/yandex/ClickHouse/pull/2646) + + +## ClickHouse release 18.5.1, 2018-07-31 + +### Новые возможности: +* Добавлена функция хеширования `murmurHash2_32`. [#2756](https://github.com/yandex/ClickHouse/pull/2756). + +### Улучшения: +* Добавлена возможность указывать значения в конфигурационных файлах из переменных окружения с помощью атрибута `from_env`. [#2741](https://github.com/yandex/ClickHouse/pull/2741). +* Добавлены регистронезависимые версии функций `coalesce`, `ifNull`, `nullIf`. [#2752](https://github.com/yandex/ClickHouse/pull/2752). + +### Исправление ошибок: +* Исправлена возможная ошибка при старте реплики. [#2759](https://github.com/yandex/ClickHouse/pull/2759). + + +## ClickHouse release 18.4.0, 2018-07-28 + +### Новые возможности: +* Добавлены системные таблицы `formats`, `data_type_families`, `aggregate_function_combinators`, `table_functions`, `table_engines`, `collations` [#2721](https://github.com/yandex/ClickHouse/pull/2721). +* Добавлена возможность использования табличной функции вместо таблицы в качестве аргумента табличной функции `remote` и `cluster` [#2708](https://github.com/yandex/ClickHouse/pull/2708). +* Поддержка `HTTP Basic` аутентификации в протоколе репликации [#2727](https://github.com/yandex/ClickHouse/pull/2727). +* В функции `has` добавлена возможность поиска в массиве значений типа `Enum` по числовому значению [Maxim Khrisanfov](https://github.com/yandex/ClickHouse/pull/2699). +* Поддержка добавления произвольных разделителей сообщений в процессе чтения из `Kafka` [Amos Bird](https://github.com/yandex/ClickHouse/pull/2701). + +### Улучшения: +* Запрос `ALTER TABLE t DELETE WHERE` не перезаписывает куски данных, которые не были затронуты условием WHERE [#2694](https://github.com/yandex/ClickHouse/pull/2694). +* Настройка `use_minimalistic_checksums_in_zookeeper` таблиц семейства `ReplicatedMergeTree` включена по-умолчанию. Эта настройка была добавлена в версии 1.1.54378, 2018-04-16. Установка версий, более старых, чем 1.1.54378, становится невозможной. +* Поддерживается запуск запросов `KILL` и `OPTIMIZE` с указанием `ON CLUSTER` [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2689). + +### Исправление ошибок: +* Исправлена ошибка `Column ... is not under aggregate function and not in GROUP BY` в случае агрегации по выражению с оператором IN. Ошибка появилась в версии 18.1.0. ([bbdd780b](https://github.com/yandex/ClickHouse/commit/bbdd780be0be06a0f336775941cdd536878dd2c2)) +* Исправлена ошибка в агрегатной функции `windowFunnel` [Winter Zhang](https://github.com/yandex/ClickHouse/pull/2735). +* Исправлена ошибка в агрегатной функции `anyHeavy` ([a2101df2](https://github.com/yandex/ClickHouse/commit/a2101df25a6a0fba99aa71f8793d762af2b801ee)) +* Исправлено падение сервера при использовании функции `countArray()`. + +### Обратно несовместимые изменения: + +* Список параметров для таблиц `Kafka` был изменён с `Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format[, kafka_schema, kafka_num_consumers])` на `Kafka(kafka_broker_list, kafka_topic_list, kafka_group_name, kafka_format[, kafka_row_delimiter, kafka_schema, kafka_num_consumers])`. Если вы использовали параметры `kafka_schema` или `kafka_num_consumers`, вам необходимо вручную отредактировать файлы с метаданными `path/metadata/database/table.sql`, добавив параметр `kafka_row_delimiter` со значением `''` в соответствующее место. + + +## ClickHouse release 18.1.0, 2018-07-23 + +### Новые возможности: +* Поддержка запроса `ALTER TABLE t DELETE WHERE` для нереплицированных MergeTree-таблиц ([#2634](https://github.com/yandex/ClickHouse/pull/2634)). +* Поддержка произвольных типов для семейства агрегатных функций `uniq*` ([#2010](https://github.com/yandex/ClickHouse/issues/2010)). +* Поддержка произвольных типов в операторах сравнения ([#2026](https://github.com/yandex/ClickHouse/issues/2026)). +* Возможность в `users.xml` указывать маску подсети в формате `10.0.0.1/255.255.255.0`. Это необходимо для использования "дырявых" масок IPv6 сетей ([#2637](https://github.com/yandex/ClickHouse/pull/2637)). +* Добавлена функция `arrayDistinct` ([#2670](https://github.com/yandex/ClickHouse/pull/2670)). +* Движок SummingMergeTree теперь может работать со столбцами типа AggregateFunction ([Constantin S. Pan](https://github.com/yandex/ClickHouse/pull/2566)). + +### Улучшения: +* Изменена схема версионирования релизов. Теперь первый компонент содержит год релиза (A.D.; по московскому времени; из номера вычитается 2000), второй - номер крупных изменений (увеличивается для большинства релизов), третий - патч-версия. Релизы по-прежнему обратно совместимы, если другое не указано в changelog. +* Ускорено преобразование чисел с плавающей точкой в строку ([Amos Bird](https://github.com/yandex/ClickHouse/pull/2664)). +* Теперь, если при вставке из-за ошибок парсинга пропущено некоторое количество строк (такое возможно про включённых настройках `input_allow_errors_num`, `input_allow_errors_ratio`), это количество пишется в лог сервера ([Leonardo Cecchi](https://github.com/yandex/ClickHouse/pull/2669)). + +### Исправление ошибок: +* Исправлена работа команды TRUNCATE для временных таблиц ([Amos Bird](https://github.com/yandex/ClickHouse/pull/2624)). +* Исправлен редкий deadlock в клиентской библиотеке ZooKeeper, который возникал при сетевой ошибке во время вычитывания ответа ([c315200](https://github.com/yandex/ClickHouse/commit/c315200e64b87e44bdf740707fc857d1fdf7e947)). +* Исправлена ошибка при CAST в Nullable типы ([#1322](https://github.com/yandex/ClickHouse/issues/1322)). +* Исправлен неправильный результат функции `maxIntersection()` в случае совпадения границ отрезков ([Michael Furmur](https://github.com/yandex/ClickHouse/pull/2657)). +* Исправлено неверное преобразование цепочки OR-выражений в аргументе функции ([chenxing-xc](https://github.com/yandex/ClickHouse/pull/2663)). +* Исправлена деградация производительности запросов, содержащих выражение `IN (подзапрос)` внутри другого подзапроса ([#2571](https://github.com/yandex/ClickHouse/issues/2571)). +* Исправлена несовместимость серверов разных версий при распределённых запросах, использующих функцию `CAST` не в верхнем регистре ([fe8c4d6](https://github.com/yandex/ClickHouse/commit/fe8c4d64e434cacd4ceef34faa9005129f2190a5)). +* Добавлено недостающее квотирование идентификаторов при запросах к внешним СУБД ([#2635](https://github.com/yandex/ClickHouse/issues/2635)). + +### Обратно несовместимые изменения: +* Не работает преобразование строки, содержащей число ноль, в DateTime. Пример: `SELECT toDateTime('0')`. По той же причине не работает `DateTime DEFAULT '0'` в таблицах, а также `0` в словарях. Решение: заменить `0` на `0000-00-00 00:00:00`. + + +## ClickHouse release 1.1.54394, 2018-07-12 + +### Новые возможности: +* Добавлена агрегатная функция `histogram` ([Михаил Сурин](https://github.com/yandex/ClickHouse/pull/2521)). +* Возможность использования `OPTIMIZE TABLE ... FINAL` без указания партиции для `ReplicatedMergeTree` ([Amos Bird](https://github.com/yandex/ClickHouse/pull/2600)). + +### Исправление ошибок: +* Исправлена ошибка - выставление слишком маленького таймаута у сокетов (одна секунда) для чтения и записи при отправке и скачивании реплицируемых данных, что приводило к невозможности скачать куски достаточно большого размера при наличии некоторой нагрузки на сеть или диск (попытки скачивания кусков циклически повторяются). Ошибка возникла в версии 1.1.54388. +* Исправлена работа при использовании chroot в ZooKeeper, в случае вставки дублирующихся блоков данных в таблицу. +* Исправлена работа функции `has` для случая массива с Nullable элементами ([#2115](https://github.com/yandex/ClickHouse/issues/2521)). +* Исправлена работа таблицы `system.tables` при её использовании в распределённых запросах; столбцы `metadata_modification_time` и `engine_full` сделаны невиртуальными; исправлена ошибка в случае, если из таблицы были запрошены только эти столбцы. +* Исправлена работа пустой таблицы типа `TinyLog` после вставки в неё пустого блока данных ([#2563](https://github.com/yandex/ClickHouse/issues/2563)). +* Таблица `system.zookeeper` работает в случае, если значение узла в ZooKeeper равно NULL. + + +## ClickHouse release 1.1.54390, 2018-07-06 + +### Новые возможности: +* Возможность отправки запроса в формате `multipart/form-data` (в поле `query`), что полезно, если при этом также отправляются внешние данные для обработки запроса ([Ольга Хвостикова](https://github.com/yandex/ClickHouse/pull/2490)). +* Добавлена возможность включить или отключить обработку одинарных или двойных кавычек при чтении данных в формате CSV. Это задаётся настройками `format_csv_allow_single_quotes` и `format_csv_allow_double_quotes` ([Amos Bird](https://github.com/yandex/ClickHouse/pull/2574)) +* Возможность использования `OPTIMIZE TABLE ... FINAL` без указания партиции для не реплицированных вариантов`MergeTree` ([Amos Bird](https://github.com/yandex/ClickHouse/pull/2599)). + +### Улучшения: +* Увеличена производительность, уменьшено потребление памяти, добавлен корректный учёт потребления памяти, при использовании оператора IN в случае, когда для его работы может использоваться индекс таблицы ([#2584](https://github.com/yandex/ClickHouse/pull/2584)). +* Убраны избыточные проверки чексумм при добавлении куска. Это важно в случае большого количества реплик, так как в этом случае суммарное количество проверок было равно N^2. +* Добавлена поддержка аргументов типа `Array(Tuple(...))` для функции `arrayEnumerateUniq` ([#2573](https://github.com/yandex/ClickHouse/pull/2573)). +* Добавлена поддержка `Nullable` для функции `runningDifference`. ([#2594](https://github.com/yandex/ClickHouse/pull/2594)) +* Увеличена производительность анализа запроса в случае очень большого количества выражений ([#2572](https://github.com/yandex/ClickHouse/pull/2572)). +* Более быстрый выбор кусков для слияния в таблицах типа `ReplicatedMergeTree`. Более быстрое восстановление сессии с ZooKeeper. ([#2597](https://github.com/yandex/ClickHouse/pull/2597)). +* Файл `format_version.txt` для таблиц семейства `MergeTree` создаётся заново при его отсутствии, что имеет смысл в случае запуска ClickHouse после копирования структуры директорий без файлов ([Ciprian Hacman](https://github.com/yandex/ClickHouse/pull/2593)). + +### Исправление ошибок: +* Исправлена ошибка при работе с ZooKeeper, которая могла приводить к невозможности восстановления сессии и readonly состояниям таблиц до перезапуска сервера. +* Исправлена ошибка при работе с ZooKeeper, которая могла приводить к неудалению старых узлов при разрыве сессии. +* Исправлена ошибка в функции `quantileTDigest` для Float аргументов (ошибка появилась в версии 1.1.54388) ([Михаил Сурин](https://github.com/yandex/ClickHouse/pull/2553)). +* Исправлена ошибка работы индекса таблиц типа MergeTree, если в условии, столбец первичного ключа расположен внутри функции преобразования типов между знаковым и беззнаковым целым одного размера ([#2603](https://github.com/yandex/ClickHouse/pull/2603)). +* Исправлен segfault, если в конфигурационном файле нет `macros`, но они используются ([#2570](https://github.com/yandex/ClickHouse/pull/2570)). +* Исправлено переключение на базу данных по-умолчанию при переподключении клиента ([#2583](https://github.com/yandex/ClickHouse/pull/2583)). +* Исправлена ошибка в случае отключенной настройки `use_index_for_in_with_subqueries`. + +### Исправления безопасности: +* При соединениях с MySQL удалена возможность отправки файлов (`LOAD DATA LOCAL INFILE`). + + +## ClickHouse release 1.1.54388, 2018-06-28 + +### Новые возможности: +* Добавлена поддержка запроса `ALTER TABLE t DELETE WHERE` для реплицированных таблиц и таблица `system.mutations`. +* Добавлена поддержка запроса `ALTER TABLE t [REPLACE|ATTACH] PARTITION` для *MergeTree-таблиц. +* Добавлена поддержка запроса `TRUNCATE TABLE` ([Winter Zhang](https://github.com/yandex/ClickHouse/pull/2260)) +* Добавлено несколько новых `SYSTEM`-запросов для реплицированных таблиц (`RESTART REPLICAS`, `SYNC REPLICA`, `[STOP|START] [MERGES|FETCHES|REPLICATED SENDS|REPLICATION QUEUES]`). +* Добавлена возможность записи в таблицу с движком MySQL и соответствующую табличную функцию ([sundy-li](https://github.com/yandex/ClickHouse/pull/2294)). +* Добавлена табличная функция `url()` и движок таблиц `URL` ([Александр Сапин](https://github.com/yandex/ClickHouse/pull/2501)). +* Добавлена агрегатная функция `windowFunnel` ([sundy-li](https://github.com/yandex/ClickHouse/pull/2352)). +* Добавлены функции `startsWith` и `endsWith` для строк ([Вадим Плахтинский](https://github.com/yandex/ClickHouse/pull/2429)). +* В табличной функции `numbers()` добавлена возможность указывать offset ([Winter Zhang](https://github.com/yandex/ClickHouse/pull/2535)). +* Добавлена возможность интерактивного ввода пароля в `clickhouse-client`. +* Добавлена возможность отправки логов сервера в syslog ([Александр Крашенинников](https://github.com/yandex/ClickHouse/pull/2459)). +* Добавлена поддержка логирования в словарях с источником shared library ([Александр Сапин](https://github.com/yandex/ClickHouse/pull/2472)). +* Добавлена поддержка произвольного разделителя в формате CSV ([Иван Жуков](https://github.com/yandex/ClickHouse/pull/2263)) +* Добавлена настройка `date_time_input_format`. Если переключить эту настройку в значение `'best_effort'`, значения DateTime будут читаться в широком диапазоне форматов. +* Добавлена утилита `clickhouse-obfuscator` для обфускации данных. Пример использования: публикация данных, используемых в тестах производительности. + +### Экспериментальные возможности: +* Добавлена возможность вычислять аргументы функции `and` только там, где они нужны ([Анастасия Царькова](https://github.com/yandex/ClickHouse/pull/2272)) +* Добавлена возможность JIT-компиляции в нативный код некоторых выражений ([pyos](https://github.com/yandex/ClickHouse/pull/2277)). + +### Исправление ошибок: +* Исправлено появление дублей в запросе с `DISTINCT` и `ORDER BY`. +* Запросы с `ARRAY JOIN` и `arrayFilter` раньше возвращали некорректный результат. +* Исправлена ошибка при чтении столбца-массива из Nested-структуры ([#2066](https://github.com/yandex/ClickHouse/issues/2066)). +* Исправлена ошибка при анализе запросов с секцией HAVING вида `HAVING tuple IN (...)`. +* Исправлена ошибка при анализе запросов с рекурсивными алиасами. +* Исправлена ошибка при чтении из ReplacingMergeTree с условием в PREWHERE, фильтрующим все строки ([#2525](https://github.com/yandex/ClickHouse/issues/2525)). +* Настройки профиля пользователя не применялись при использовании сессий в HTTP-интерфейсе. +* Исправлено применение настроек из параметров командной строки в программе clickhouse-local. +* Клиентская библиотека ZooKeeper теперь использует таймаут сессии, полученный от сервера. +* Исправлена ошибка в клиентской библиотеке ZooKeeper, из-за которой ожидание ответа от сервера могло длиться дольше таймаута. +* Исправлено отсечение ненужных кусков при запросе с условием на столбцы ключа партиционирования ([#2342](https://github.com/yandex/ClickHouse/issues/2342)). +* После `CLEAR COLUMN IN PARTITION` в соответствующей партиции теперь возможны слияния ([#2315](https://github.com/yandex/ClickHouse/issues/2315)). +* Исправлено соответствие типов в табличной функции ODBC ([sundy-li](https://github.com/yandex/ClickHouse/pull/2268)). +* Исправлено некорректное сравнение типов `DateTime` с таймзоной и без неё ([Александр Бочаров](https://github.com/yandex/ClickHouse/pull/2400)). +* Исправлен синтаксический разбор и форматирование оператора `CAST`. +* Исправлена вставка в материализованное представление в случае, если движок таблицы представления - Distributed ([Babacar Diassé](https://github.com/yandex/ClickHouse/pull/2411)). +* Исправлен race condition при записи данных из движка `Kafka` в материализованные представления ([Yangkuan Liu](https://github.com/yandex/ClickHouse/pull/2448)). +* Исправлена SSRF в табличной функции remote(). +* Исправлен выход из `clickhouse-client` в multiline-режиме ([#2510](https://github.com/yandex/ClickHouse/issues/2510)). + +### Улучшения: +* Фоновые задачи в реплицированных таблицах теперь выполняются не в отдельных потоках, а в пуле потоков ([Silviu Caragea](https://github.com/yandex/ClickHouse/pull/1722)) +* Улучшена производительность разжатия LZ4. +* Ускорен анализ запроса с большим числом JOIN-ов и подзапросов. +* DNS-кэш теперь автоматически обновляется при большом числе сетевых ошибок. +* Вставка в таблицу теперь не происходит, если вставка в одно из её материализованных представлений невозможна из-за того, что в нём много кусков. +* Исправлено несоответствие в значениях счётчиков событий `Query`, `SelectQuery`, `InsertQuery`. +* Разрешены выражения вида `tuple IN (SELECT tuple)`, если типы кортежей совпадают. +* Сервер с реплицированными таблицами теперь может стартовать, даже если не сконфигурирован ZooKeeper. +* При расчёте количества доступных ядер CPU теперь учитываются ограничения cgroups ([Atri Sharma](https://github.com/yandex/ClickHouse/pull/2325)). +* Добавлен chown директорий конфигов в конфигурационном файле systemd ([Михаил Ширяев](https://github.com/yandex/ClickHouse/pull/2421)). + +### Изменения сборки: +* Добавлена возможность сборки компилятором gcc8. +* Добавлена возможность сборки llvm из submodule. +* Используемая версия библиотеки librdkafka обновлена до v0.11.4. +* Добавлена возможность использования библиотеки libcpuid из системы, используемая версия библиотеки обновлена до 0.4.0. +* Исправлена сборка с использованием библиотеки vectorclass ([Babacar Diassé](https://github.com/yandex/ClickHouse/pull/2274)). +* Cmake теперь по умолчанию генерирует файлы для ninja (как при использовании `-G Ninja`). +* Добавлена возможность использования библиотеки libtinfo вместо libtermcap ([Георгий Кондратьев](https://github.com/yandex/ClickHouse/pull/2519)). +* Исправлен конфликт заголовочных файлов в Fedora Rawhide ([#2520](https://github.com/yandex/ClickHouse/issues/2520)). + +### Обратно несовместимые изменения: +* Убран escaping в форматах `Vertical` и `Pretty*`, удалён формат `VerticalRaw`. +* Если в распределённых запросах одновременно участвуют серверы версии 1.1.54388 или новее и более старые, то при использовании выражения `cast(x, 'Type')`, записанного без указания `AS`, если слово `cast` указано не в верхнем регистре, возникает ошибка вида `Not found column cast(0, 'UInt8') in block`. Решение: обновить сервер на всём кластере. + + +## ClickHouse release 1.1.54385, 2018-06-01 +### Исправление ошибок: +* Исправлена ошибка, которая в некоторых случаях приводила к блокировке операций с ZooKeeper. + +## ClickHouse release 1.1.54383, 2018-05-22 +### Исправление ошибок: +* Исправлена деградация скорости выполнения очереди репликации при большом количестве реплик + +## ClickHouse release 1.1.54381, 2018-05-14 + +### Исправление ошибок: +* Исправлена ошибка, приводящая к "утеканию" метаданных в ZooKeeper при потере соединения с сервером ZooKeeper. + +## ClickHouse release 1.1.54380, 2018-04-21 + +### Новые возможности: +* Добавлена табличная функция `file(path, format, structure)`. Пример, читающий байты из `/dev/urandom`: `ln -s /dev/urandom /var/lib/clickhouse/user_files/random` `clickhouse-client -q "SELECT * FROM file('random', 'RowBinary', 'd UInt8') LIMIT 10"`. + +### Улучшения: +* Добавлена возможность оборачивать подзапросы скобками `()` для повышения читаемости запросов. Например: `(SELECT 1) UNION ALL (SELECT 1)`. +* Простые запросы `SELECT` из таблицы `system.processes` не учитываются в ограничении `max_concurrent_queries`. + +### Исправление ошибок: +* Исправлена неправильная работа оператора `IN` в `MATERIALIZED VIEW`. +* Исправлена неправильная работа индекса по ключу партиционирования в выражениях типа `partition_key_column IN (...)`. +* Исправлена невозможность выполнить `OPTIMIZE` запрос на лидирующей реплике после выполнения `RENAME` таблицы. +* Исправлены ошибки авторизации возникающие при выполнении запросов `OPTIMIZE` и `ALTER` на нелидирующей реплике. +* Исправлены зависания запросов `KILL QUERY`. +* Исправлена ошибка в клиентской библиотеке ZooKeeper, которая при использовании непустого префикса `chroot` в конфигурации приводила к потере watch'ей, остановке очереди distributed DDL запросов и замедлению репликации. + +### Обратно несовместимые изменения: +* Убрана поддержка выражений типа `(a, b) IN (SELECT (a, b))` (можно использовать эквивалентные выражение `(a, b) IN (SELECT a, b)`). Раньше такие запросы могли приводить к недетерминированной фильтрации в `WHERE`. + + +## ClickHouse release 1.1.54378, 2018-04-16 + +### Новые возможности: + +* Возможность изменения уровня логгирования без перезагрузки сервера. +* Добавлен запрос `SHOW CREATE DATABASE`. +* Возможность передать `query_id` в `clickhouse-client` (elBroom). +* Добавлена настройка `max_network_bandwidth_for_all_users`. +* Добавлена поддержка `ALTER TABLE ... PARTITION ... ` для `MATERIALIZED VIEW`. +* Добавлена информация о размере кусков данных в несжатом виде в системные таблицы. +* Поддержка межсерверного шифрования для distributed таблиц (`1` в конфигурации реплики в ``). +* Добавлена настройка уровня таблицы семейства `ReplicatedMergeTree` для уменьшения объема данных, хранимых в zookeeper: `use_minimalistic_checksums_in_zookeeper = 1` +* Возможность настройки приглашения `clickhouse-client`. По-умолчанию добавлен вывод имени сервера в приглашение. Возможность изменить отображаемое имя сервера. Отправка его в HTTP заголовке `X-ClickHouse-Display-Name` (Kirill Shvakov). +* Возможность указания нескольких `topics` через запятую для движка `Kafka` (Tobias Adamson) +* При остановке запроса по причине `KILL QUERY` или `replace_running_query`, клиент получает исключение `Query was cancelled` вместо неполного результата. + +### Улучшения: + +* Запросы вида `ALTER TABLE ... DROP/DETACH PARTITION` выполняются впереди очереди репликации. +* Возможность использовать `SELECT ... FINAL` и `OPTIMIZE ... FINAL` даже в случае, если данные в таблице представлены одним куском. +* Пересоздание таблицы `query_log` налету в случае если было произведено её удаление вручную (Kirill Shvakov). +* Ускорение функции `lengthUTF8` (zhang2014). +* Улучшена производительность синхронной вставки в `Distributed` таблицы (`insert_distributed_sync = 1`) в случае очень большого количества шардов. +* Сервер принимает настройки `send_timeout` и `receive_timeout` от клиента и применяет их на своей стороне для соединения с клиентом (в переставленном порядке: `send_timeout` у сокета на стороне сервера выставляется в значение `receive_timeout` принятое от клиента, и наоборот). +* Более надёжное восстановление после сбоев при асинхронной вставке в `Distributed` таблицы. +* Возвращаемый тип функции `countEqual` изменён с `UInt32` на `UInt64` (谢磊) + +### Исправление ошибок: + +* Исправлена ошибка c `IN` где левая часть выражения `Nullable`. +* Исправлен неправильный результат при использовании кортежей с `IN` в случае, если часть компоненнтов кортежа есть в индексе таблицы. +* Исправлена работа ограничения `max_execution_time` с распределенными запросами. +* Исправлены ошибки при вычислении размеров составных столбцов в таблице `system.columns`. +* Исправлена ошибка при создании временной таблицы `CREATE TEMPORARY TABLE IF NOT EXISTS` +* Исправлены ошибки в `StorageKafka` (#2075) +* Исправлены падения сервера от некорректных аргументов некоторых аггрегатных функций. +* Исправлена ошибка, из-за которой запрос `DETACH DATABASE` мог не приводить к остановке фоновых задач таблицы типа `ReplicatedMergeTree`. +* Исправлена проблема с появлением `Too many parts` в агрегирующих материализованных представлениях (#2084). +* Исправлена рекурсивная обработка подстановок в конфиге, если после одной подстановки, требуется другая подстановка на том же уровне. +* Исправлена ошибка с неправильным синтаксисом в файле с метаданными при создании `VIEW`, использующих запрос с `UNION ALL`. +* Исправлена работа `SummingMergeTree` в случае суммирования вложенных структур данных с составным ключом. +* Исправлена возможность возникновения race condition при выборе лидера таблиц `ReplicatedMergeTree`. + +### Изменения сборки: + +* Поддержка `ninja` вместо `make` при сборке. `ninja` используется по-умолчанию при сборке релизов. +* Переименованы пакеты `clickhouse-server-base` в `clickhouse-common-static`; `clickhouse-server-common` в `clickhouse-server`; `clickhouse-common-dbg` в `clickhouse-common-static-dbg`. Для установки используйте `clickhouse-server clickhouse-client`. Для совместимости, пакеты со старыми именами продолжают загружаться в репозиторий. + +### Обратно несовместимые изменения: + +* Удалена специальная интерпретация выражения IN, если слева указан массив. Ранее выражение вида `arr IN (set)` воспринималось как "хотя бы один элемент `arr` принадлежит множеству `set`". Для получения такого же поведения в новой версии, напишите `arrayExists(x -> x IN (set), arr)`. +* Отключено ошибочное использование опции сокета `SO_REUSEPORT` (которая по ошибке включена по-умолчанию в библиотеке Poco). Стоит обратить внимание, что на Linux системах теперь не имеет смысла указывать одновременно адреса `::` и `0.0.0.0` для listen - следует использовать лишь адрес `::`, который (с настройками ядра по-умолчанию) позволяет слушать соединения как по IPv4 так и по IPv6. Также вы можете вернуть поведение старых версий, указав в конфиге `1`. + + +## ClickHouse release 1.1.54370, 2018-03-16 + +### Новые возможности: * Добавлена системная таблица `system.macros` и автоматическое обновление макросов при изменении конфигурационного файла. * Добавлен запрос `SYSTEM RELOAD CONFIG`. * Добавлена агрегатная функция `maxIntersections(left_col, right_col)`, возвращающая максимальное количество одновременно пересекающихся интервалов `[left; right]`. Функция `maxIntersectionsPosition(left, right)` возвращает начало такого "максимального" интервала. ([Michael Furmur](https://github.com/yandex/ClickHouse/pull/2012)). -## Улучшения: +### Улучшения: * При вставке данных в `Replicated`-таблицу делается меньше обращений к `ZooKeeper` (также из лога `ZooKeeper` исчезло большинство user-level ошибок). * Добавлена возможность создавать алиасы для множеств. Пример: `WITH (1, 2, 3) AS set SELECT number IN set FROM system.numbers LIMIT 10`. -## Исправление ошибок: +### Исправление ошибок: * Исправлена ошибка `Illegal PREWHERE` при чтении из Merge-таблицы над `Distributed`-таблицами. * Добавлены исправления, позволяющие запускать clickhouse-server в IPv4-only docker-контейнерах. @@ -26,9 +642,9 @@ * Устранено ненужное Error-level логирование `Not found column ... in block`. -# Релиз ClickHouse 1.1.54362, 2018-03-11 +## Релиз ClickHouse 1.1.54362, 2018-03-11 -## Новые возможности: +### Новые возможности: * Агрегация без `GROUP BY` по пустому множеству (как например, `SELECT count(*) FROM table WHERE 0`) теперь возвращает результат из одной строки с нулевыми значениями агрегатных функций, в соответствии со стандартом SQL. Вы можете вернуть старое поведение (возвращать пустой результат), выставив настройку `empty_result_for_aggregation_by_empty_set` в значение 1. * Добавлено приведение типов при `UNION ALL`. Допустимо использование столбцов с разными алиасами в соответствующих позициях `SELECT` в `UNION ALL`, что соответствует стандарту SQL. @@ -66,7 +682,7 @@ * Добавлена настройка `odbc_default_field_size`, позволяющая расширить максимальный размер значения, загружаемого из ODBC источника (по-умолчанию - 1024). * В таблицу `system.processes` и в `SHOW PROCESSLIST` добавлены столбцы `is_cancelled` и `peak_memory_usage`. -## Улучшения: +### Улучшения: * Ограничения на результат и квоты на результат теперь не применяются к промежуточным данным для запросов `INSERT SELECT` и для подзапросов в `SELECT`. * Уменьшено количество ложных срабатываний при проверке состояния `Replicated` таблиц при запуске сервера, приводивших к необходимости выставления флага `force_restore_data`. @@ -82,7 +698,7 @@ * Значения типа `Enum` можно использовать в функциях `min`, `max`, `sum` и некоторых других - в этих случаях используются соответствующие числовые значения. Эта возможность присутствовала ранее, но была потеряна в релизе 1.1.54337. * Добавлено ограничение `max_expanded_ast_elements` действующее на размер AST после рекурсивного раскрытия алиасов. -## Исправление ошибок: +### Исправление ошибок: * Исправлены случаи ошибочного удаления ненужных столбцов из подзапросов, а также отсутствие удаления ненужных столбцов из подзапросов, содержащих `UNION ALL`. * Исправлена ошибка в слияниях для таблиц типа `ReplacingMergeTree`. @@ -110,19 +726,19 @@ * Запрещено использование запросов с `UNION ALL` в `MATERIALIZED VIEW`. * Исправлена ошибка, которая может возникать при инициализации системной таблицы `part_log` при старте сервера (по-умолчанию `part_log` выключен). -## Обратно несовместимые изменения: +### Обратно несовместимые изменения: * Удалена настройка `distributed_ddl_allow_replicated_alter`. Соответствующее поведение включено по-умолчанию. * Удалена настройка `strict_insert_defaults`. Если вы использовали эту функциональность, напишите на `clickhouse-feedback@yandex-team.com`. * Удалён движок таблиц `UnsortedMergeTree`. -# Релиз ClickHouse 1.1.54343, 2018-02-05 +## Релиз ClickHouse 1.1.54343, 2018-02-05 * Добавлена возможность использовать макросы при задании имени кластера в распределенных DLL запросах и создании Distributed-таблиц: `CREATE TABLE distr ON CLUSTER '{cluster}' (...) ENGINE = Distributed('{cluster}', 'db', 'table')`. * Теперь при вычислении запросов вида `SELECT ... FROM table WHERE expr IN (subquery)` используется индекс таблицы `table`. * Улучшена обработка дубликатов при вставке в Replicated-таблицы, теперь они не приводят к излишнему замедлению выполнения очереди репликации. -# Релиз ClickHouse 1.1.54342, 2018-01-22 +## Релиз ClickHouse 1.1.54342, 2018-01-22 Релиз содержит исправление к предыдущему релизу 1.1.54337: * Исправлена регрессия в версии 1.1.54337: если пользователь по-умолчанию имеет readonly доступ, то сервер отказывался стартовать с сообщением `Cannot create database in readonly mode`. @@ -133,9 +749,9 @@ * Таблицы типа Buffer теперь работают при наличии MATERIALIZED столбцов в таблице назначения (by zhang2014). * Исправлена одна из ошибок в реализации NULL. -# Релиз ClickHouse 1.1.54337, 2018-01-18 +## Релиз ClickHouse 1.1.54337, 2018-01-18 -## Новые возможности: +### Новые возможности: * Добавлена поддержка хранения многомерных массивов и кортежей (тип данных `Tuple`) в таблицах. * Поддержка табличных функций для запросов `DESCRIBE` и `INSERT`. Поддержка подзапроса в запросе `DESCRIBE`. Примеры: `DESC TABLE remote('host', default.hits)`; `DESC TABLE (SELECT 1)`; `INSERT INTO TABLE FUNCTION remote('host', default.hits)`. Возможность писать `INSERT INTO TABLE` вместо `INSERT INTO`. @@ -164,9 +780,9 @@ * Добавлена поддержка `ALTER` для таблиц типа `Null` (Anastasiya Tsarkova). * Функция `reinterpretAsString` расширена на все типы данных, значения которых хранятся в памяти непрерывно. * Для программы `clickhouse-local` добавлена опция `--silent` для подавления вывода информации о выполнении запроса в stderr. -* Добавлена поддержка чтения `Date` в текстовом виде в формате, где месяц и день месяца могут быть указаны одной цифрой вместо двух (Amos Bird). +* Добавлена поддержка чтения `Date` в текстовом виде в формате, где месяц и день месяца могут быть указаны одной цифрой вместо двух (Amos Bird). -## Увеличение производительности: +### Увеличение производительности: * Увеличена производительность агрегатных функций `min`, `max`, `any`, `anyLast`, `anyHeavy`, `argMin`, `argMax` от строковых аргументов. * Увеличена производительность функций `isInfinite`, `isFinite`, `isNaN`, `roundToExp2`. @@ -175,7 +791,7 @@ * Уменьшено потребление памяти при `JOIN`, если левая и правая часть содержали столбцы с одинаковым именем, не входящие в `USING`. * Увеличена производительность агрегатных функций `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr` за счёт уменьшения стойкости к вычислительной погрешности. Старые версии функций добавлены под именами `varSampStable`, `varPopStable`, `stddevSampStable`, `stddevPopStable`, `covarSampStable`, `covarPopStable`, `corrStable`. -## Исправления ошибок: +### Исправления ошибок: * Исправлена работа дедупликации блоков после `DROP` или `DETATH PARTITION`. Раньше удаление партиции и вставка тех же самых данных заново не работала, так как вставленные заново блоки считались дубликатами. * Исправлена ошибка, в связи с которой может неправильно обрабатываться `WHERE` для запросов на создание `MATERIALIZED VIEW` с указанием `POPULATE`. @@ -185,7 +801,7 @@ * Добавлена недостающая поддержка типа данных `UUID` для `DISTINCT`, `JOIN`, в агрегатных функциях `uniq` и во внешних словарях (Иванов Евгений). Поддержка `UUID` всё ещё остаётся не полной. * Исправлено поведение `SummingMergeTree` для строк, в которых все значения после суммирования равны нулю. * Многочисленные доработки для движка таблиц `Kafka` (Marek Vavruša). -* Исправлена некорректная работа движка таблиц `Join` (Amos Bird). +* Исправлена некорректная работа движка таблиц `Join` (Amos Bird). * Исправлена работа аллокатора под FreeBSD и OS X. * Функция `extractAll` теперь может доставать пустые вхождения. * Исправлена ошибка, не позволяющая подключить при сборке `libressl` вместо `openssl`. @@ -209,12 +825,12 @@ * Исправлена работа `DISTINCT` при условии, что все столбцы константные. * Исправлено форматирование запроса в случае наличия функции `tupleElement` со сложным константным выражением в качестве номера элемента. * Исправлена работа `Dictionary` таблиц для словарей типа `range_hashed`. -* Исправлена ошибка, приводящая к появлению лишних строк при `FULL` и `RIGHT JOIN` (Amos Bird). +* Исправлена ошибка, приводящая к появлению лишних строк при `FULL` и `RIGHT JOIN` (Amos Bird). * Исправлено падение сервера в случае создания и удаления временных файлов в `config.d` директориях в момент перечитывания конфигурации. * Исправлена работа запроса `SYSTEM DROP DNS CACHE`: ранее сброс DNS кэша не приводил к повторному резолвингу имён хостов кластера. * Исправлено поведение `MATERIALIZED VIEW` после `DETACH TABLE` таблицы, на которую он смотрит (Marek Vavruša). -## Улучшения сборки: +### Улучшения сборки: * Для сборки используется `pbuilder`. Сборка максимально независима от окружения на сборочной машине. * Для разных версий систем выкладывается один и тот же пакет, который совместим с широким диапазоном Linux систем. @@ -228,27 +844,27 @@ * Удалено использование расширений GNU из кода и включена опция `-Wextra`. При сборке с помощью `clang` по-умолчанию используется `libc++` вместо `libstdc++`. * Выделены библиотеки `clickhouse_parsers` и `clickhouse_common_io` для более быстрой сборки утилит. -## Обратно несовместимые изменения: +### Обратно несовместимые изменения: * Формат засечек (marks) для таблиц типа `Log`, содержащих `Nullable` столбцы, изменён обратно-несовместимым образом. В случае наличия таких таблиц, вы можете преобразовать их в `TinyLog` до запуска новой версии сервера. Для этого в соответствующем таблице файле `.sql` в директории `metadata`, замените `ENGINE = Log` на `ENGINE = TinyLog`. Если в таблице нет `Nullable` столбцов или тип таблицы не `Log`, то ничего делать не нужно. * Удалена настройка `experimental_allow_extended_storage_definition_syntax`. Соответствующая функциональность включена по-умолчанию. * Функция `runningIncome` переименована в `runningDifferenceStartingWithFirstValue` во избежание путаницы. -* Удалена возможность написания `FROM ARRAY JOIN arr` без указания таблицы после FROM (Amos Bird). +* Удалена возможность написания `FROM ARRAY JOIN arr` без указания таблицы после FROM (Amos Bird). * Удалён формат `BlockTabSeparated`, использовавшийся лишь для демонстрационных целей. * Изменён формат состояния агрегатных функций `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr`. Если вы использовали эти состояния для хранения в таблицах (тип данных `AggregateFunction` от этих функций или материализованные представления, хранящие эти состояния), напишите на clickhouse-feedback@yandex-team.com. * В предыдущих версиях существовала недокументированная возможность: в типе данных AggregateFunction можно было не указывать параметры для агрегатной функции, которая зависит от параметров. Пример: `AggregateFunction(quantiles, UInt64)` вместо `AggregateFunction(quantiles(0.5, 0.9), UInt64)`. Эта возможность потеряна. Не смотря на то, что возможность не документирована, мы собираемся вернуть её в ближайших релизах. * Значения типа данных Enum не могут быть переданы в агрегатные функции min/max. Возможность будет возвращена обратно в следующем релизе. -## На что обратить внимание при обновлении: +### На что обратить внимание при обновлении: * При обновлении кластера, на время, когда на одних репликах работает новая версия сервера, а на других - старая, репликация будет приостановлена и в логе появятся сообщения вида `unknown parameter 'shard'`. Репликация продолжится после обновления всех реплик кластера. * Если на серверах кластера работают разные версии ClickHouse, то возможен неправильный результат распределённых запросов, использующих функции `varSamp`, `varPop`, `stddevSamp`, `stddevPop`, `covarSamp`, `covarPop`, `corr`. Необходимо обновить все серверы кластера. -# Релиз ClickHouse 1.1.54327, 2017-12-21 +## Релиз ClickHouse 1.1.54327, 2017-12-21 Релиз содержит исправление к предыдущему релизу 1.1.54318: * Исправлена проблема с возможным race condition при репликации, которая может приводить к потере данных. Проблеме подвержены версии 1.1.54310 и 1.1.54318. Если вы их используете и у вас есть Replicated таблицы, то обновление обязательно. Понять, что эта проблема существует, можно по сообщениям в логе Warning вида `Part ... from own log doesn't exist.` Даже если таких сообщений нет, проблема всё-равно актуальна. -# Релиз ClickHouse 1.1.54318, 2017-11-30 +## Релиз ClickHouse 1.1.54318, 2017-11-30 Релиз содержит изменения к предыдущему релизу 1.1.54310 с исправлением следующих багов: * Исправлено некорректное удаление строк при слияниях в движке SummingMergeTree @@ -257,9 +873,9 @@ * Исправлена проблема, приводящая к остановке выполнения очереди репликации * Исправлено ротирование и архивация логов сервера -# Релиз ClickHouse 1.1.54310, 2017-11-01 +## Релиз ClickHouse 1.1.54310, 2017-11-01 -## Новые возможности: +### Новые возможности: * Произвольный ключ партиционирования для таблиц семейства MergeTree. * Движок таблиц [Kafka](https://clickhouse.yandex/docs/en/single/index.html#document-table_engines/kafka). * Возможность загружать модели [CatBoost](https://catboost.yandex/) и применять их к данным, хранящимся в ClickHouse. @@ -275,12 +891,12 @@ * Поддержка входного формата Cap’n Proto. * Возможность задавать уровень сжатия при использовании алгоритма zstd. -## Обратно несовместимые изменения: +### Обратно несовместимые изменения: * Запрещено создание временных таблиц с движком, отличным от Memory. * Запрещено явное создание таблиц с движком View и MaterializedView. * При создании таблицы теперь проверяется, что ключ сэмплирования входит в первичный ключ. -## Исправления ошибок: +### Исправления ошибок: * Исправлено зависание при синхронной вставке в Distributed таблицу. * Исправлена неатомарность при добавлении/удалении кусков в реплицированных таблицах. * Данные, вставляемые в материализованное представление, теперь не подвергаются излишней дедупликации. @@ -290,14 +906,14 @@ * Исправлено зависание при недостатке места на диске в разделе с логами. * Исправлено переполнение в функции toRelativeWeekNum для первой недели Unix-эпохи. -## Улучшения сборки: +### Улучшения сборки: * Несколько сторонних библиотек (в частности, Poco) обновлены и переведены на git submodules. -# Релиз ClickHouse 1.1.54304, 2017-10-19 -## Новые возможности: +## Релиз ClickHouse 1.1.54304, 2017-10-19 +### Новые возможности: * Добавлена поддержка TLS в нативном протоколе (включается заданием `tcp_ssl_port` в `config.xml`) -## Исправления ошибок: +### Исправления ошибок: * `ALTER` для реплицированных таблиц теперь пытается начать выполнение как можно быстрее * Исправлены падения при чтении данных с настройкой `preferred_block_size_bytes=0` * Исправлено падение `clickhouse-client` при нажатии `Page Down` @@ -310,16 +926,16 @@ * Корректное обновление пользователей при невалидном `users.xml` * Корректная обработка случаев, когда executable-словарь возвращает ненулевой код ответа -# Релиз ClickHouse 1.1.54292, 2017-09-20 +## Релиз ClickHouse 1.1.54292, 2017-09-20 -## Новые возможности: +### Новые возможности: * Добавлена функция `pointInPolygon` для работы с координатами на плоскости. * Добавлена агрегатная функция `sumMap`, обеспечивающая суммирование массивов аналогично `SummingMergeTree`. * Добавлена функция `trunc`. Увеличена производительность функций округления `round`, `floor`, `ceil`, `roundToExp2`. Исправлена логика работы функций округления. Изменена логика работы функции `roundToExp2` для дробных и отрицательных чисел. * Ослаблена зависимость исполняемого файла ClickHouse от версии libc. Один и тот же исполняемый файл ClickHouse может запускаться и работать на широком множестве Linux систем. Замечание: зависимость всё ещё присутствует при использовании скомпилированных запросов (настройка `compile = 1`, по-умолчанию не используется). * Уменьшено время динамической компиляции запросов. -## Исправления ошибок: +### Исправления ошибок: * Исправлена ошибка, которая могла приводить к сообщениям `part ... intersects previous part` и нарушению консистентности реплик. * Исправлена ошибка, приводящая к блокировке при завершении работы сервера, если в это время ZooKeeper недоступен. * Удалено избыточное логгирование при восстановлении реплик. @@ -327,9 +943,9 @@ * Исправлена ошибка в функции concat, возникающая в случае, если первый столбец блока имеет тип Array. * Исправлено отображение прогресса в таблице system.merges. -# Релиз ClickHouse 1.1.54289, 2017-09-13 +## Релиз ClickHouse 1.1.54289, 2017-09-13 -## Новые возможности: +### Новые возможности: * Запросы `SYSTEM` для административных действий с сервером: `SYSTEM RELOAD DICTIONARY`, `SYSTEM RELOAD DICTIONARIES`, `SYSTEM DROP DNS CACHE`, `SYSTEM SHUTDOWN`, `SYSTEM KILL`. * Добавлены функции для работы с массивами: `concat`, `arraySlice`, `arrayPushBack`, `arrayPushFront`, `arrayPopBack`, `arrayPopFront`. * Добавлены параметры `root` и `identity` для конфигурации ZooKeeper. Это позволяет изолировать разных пользователей одного ZooKeeper кластера. @@ -344,7 +960,7 @@ * Возможность задать `umask` в конфигурационном файле. * Увеличена производительность запросов с `DISTINCT`. -## Исправления ошибок: +### Исправления ошибок: * Более оптимальная процедура удаления старых нод в ZooKeeper. Ранее в случае очень частых вставок, старые ноды могли не успевать удаляться, что приводило, в том числе, к очень долгому завершению сервера. * Исправлена рандомизация при выборе хостов для соединения с ZooKeeper. * Исправлено отключение отстающей реплики при распределённых запросах, если реплика является localhost. @@ -357,28 +973,28 @@ * Исправлено появление zombie процессов при работе со словарём с источником `executable`. * Исправлен segfault при запросе HEAD. -## Улучшения процесса разработки и сборки ClickHouse: +### Улучшения процесса разработки и сборки ClickHouse: * Возможность сборки с помощью `pbuilder`. * Возможность сборки с использованием `libc++` вместо `libstdc++` под Linux. * Добавлены инструкции для использования статических анализаторов кода `Coverity`, `clang-tidy`, `cppcheck`. -## На что обратить внимание при обновлении: +### На что обратить внимание при обновлении: * Увеличено значение по-умолчанию для настройки MergeTree `max_bytes_to_merge_at_max_space_in_pool` (максимальный суммарный размер кусков в байтах для мержа) со 100 GiB до 150 GiB. Это может привести к запуску больших мержей после обновления сервера, что может вызвать повышенную нагрузку на дисковую подсистему. Если же на серверах, где это происходит, количество свободного места менее чем в два раза больше суммарного объёма выполняющихся мержей, то в связи с этим перестанут выполняться какие-либо другие мержи, включая мержи мелких кусков. Это приведёт к тому, что INSERT-ы будут отклоняться с сообщением "Merges are processing significantly slower than inserts". Для наблюдения, используйте запрос `SELECT * FROM system.merges`. Вы также можете смотреть на метрику `DiskSpaceReservedForMerge` в таблице `system.metrics` или в Graphite. Для исправления этой ситуации можно ничего не делать, так как она нормализуется сама после завершения больших мержей. Если же вас это не устраивает, вы можете вернуть настройку `max_bytes_to_merge_at_max_space_in_pool` в старое значение, прописав в config.xml в секции `` `107374182400` и перезапустить сервер. -# Релиз ClickHouse 1.1.54284, 2017-08-29 +## Релиз ClickHouse 1.1.54284, 2017-08-29 * Релиз содержит изменения к предыдущему релизу 1.1.54282, которые исправляют утечку записей о кусках в ZooKeeper -# Релиз ClickHouse 1.1.54282, 2017-08-23 +## Релиз ClickHouse 1.1.54282, 2017-08-23 Релиз содержит исправления к предыдущему релизу 1.1.54276: * Исправлена ошибка `DB::Exception: Assertion violation: !_path.empty()` при вставке в Distributed таблицу. * Исправлен парсинг при вставке в формате RowBinary, если входные данные начинаются с ';'. * Исправлена ошибка при рантайм-компиляции некоторых агрегатных функций (например, `groupArray()`). -# Релиз ClickHouse 1.1.54276, 2017-08-16 +## Релиз ClickHouse 1.1.54276, 2017-08-16 -## Новые возможности: +### Новые возможности: * Добавлена опциональная секция WITH запроса SELECT. Пример запроса: `WITH 1+1 AS a SELECT a, a*a` * Добавлена возможность синхронной вставки в Distributed таблицу: выдается Ok только после того как все данные записались на все шарды. Активируется настройкой insert_distributed_sync=1 * Добавлен тип данных UUID для работы с 16-байтовыми идентификаторами @@ -388,17 +1004,17 @@ * Добавлена поддержка неконстантных аргументов и отрицательных смещений в функции `substring(str, pos, len)` * Добавлен параметр max_size для агрегатной функции `groupArray(max_size)(column)`, и оптимизирована её производительность -## Основные изменения: +### Основные изменения: * Улучшение безопасности: все файлы сервера создаются с правами 0640 (можно поменять, через параметр в конфиге). * Улучшены сообщения об ошибках в случае синтаксически неверных запросов * Значительно уменьшен расход оперативной памяти и улучшена производительность слияний больших MergeTree-кусков данных * Значительно увеличена производительность слияний данных для движка ReplacingMergeTree * Улучшена производительность асинхронных вставок из Distributed таблицы за счет объединения нескольких исходных вставок. Функциональность включается настройкой distributed_directory_monitor_batch_inserts=1. -## Обратно несовместимые изменения: +### Обратно несовместимые изменения: * Изменился бинарный формат агрегатных состояний функции `groupArray(array_column)` для массивов -## Полный список изменений: +### Полный список изменений: * Добавлена настройка `output_format_json_quote_denormals`, включающая вывод nan и inf значений в формате JSON * Более оптимальное выделение потоков при чтении из Distributed таблиц * Разрешено задавать настройки в режиме readonly, если их значение не изменяется @@ -416,7 +1032,7 @@ * Возможность подключения к MySQL через сокет на файловой системе * В таблицу system.parts добавлен столбец с информацией о размере marks в байтах -## Исправления багов: +### Исправления багов: * Исправлена некорректная работа Distributed таблиц, использующих Merge таблицы, при SELECT с условием на поле _table * Исправлен редкий race condition в ReplicatedMergeTree при проверке кусков данных * Исправлено возможное зависание процедуры leader election при старте сервера @@ -439,15 +1055,15 @@ * Исправлена ошибка "Cannot mremap" при использовании множеств в секциях IN, JOIN, содержащих более 2 млрд. элементов * Исправлен failover для словарей с источником MySQL -## Улучшения процесса разработки и сборки ClickHouse: +### Улучшения процесса разработки и сборки ClickHouse: * Добавлена возмозможность сборки в Arcadia * Добавлена возможность сборки с помощью gcc 7 * Ускорена параллельная сборка с помощью ccache+distcc -# Релиз ClickHouse 1.1.54245, 2017-07-04 +## Релиз ClickHouse 1.1.54245, 2017-07-04 -## Новые возможности: +### Новые возможности: * Распределённые DDL (например, `CREATE TABLE ON CLUSTER`) * Реплицируемый запрос `ALTER TABLE CLEAR COLUMN IN PARTITION` * Движок таблиц Dictionary (доступ к данным словаря в виде таблицы) @@ -458,14 +1074,14 @@ * Сессии в HTTP интерфейсе * Запрос OPTIMIZE для Replicated таблицы теперь можно выполнять не только на лидере -## Обратно несовместимые изменения: +### Обратно несовместимые изменения: * Убрана команда SET GLOBAL -## Мелкие изменения: +### Мелкие изменения: * Теперь после получения сигнала в лог печатается полный стектрейс * Ослаблена проверка на количество повреждённых/лишних кусков при старте (было слишком много ложных срабатываний) -## Исправления багов: +### Исправления багов: * Исправлено залипание плохого соединения при вставке в Distributed таблицу * GLOBAL IN теперь работает при запросе из таблицы Merge, смотрящей в Distributed * Теперь правильно определяется количество ядер на виртуалках Google Compute Engine diff --git a/CMakeLists.txt b/CMakeLists.txt index ca4be1f5893..d215c05250a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,22 +1,26 @@ project (ClickHouse) -cmake_minimum_required (VERSION 2.8) +cmake_minimum_required (VERSION 3.3) -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${ClickHouse_SOURCE_DIR}/cmake/Modules/") +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/") if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") # Require at least gcc 7 if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7 AND NOT CMAKE_VERSION VERSION_LESS 2.8.9) - message (FATAL_ERROR "GCC version must be at least 7! For example, if GCC 7 is available under gcc-7, g++-7 names, do the following: export CC=gcc-7 CXX=g++-7; rm -rf CMakeCache.txt CMakeFiles; and re run cmake or ./release.") + message (FATAL_ERROR "GCC version must be at least 7. For example, if GCC 7 is available under gcc-7, g++-7 names, do the following: export CC=gcc-7 CXX=g++-7; rm -rf CMakeCache.txt CMakeFiles; and re run cmake or ./release.") endif () elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") # Require at least clang 5 if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5) - message (FATAL_ERROR "Clang version must be at least 5!") + message (FATAL_ERROR "Clang version must be at least 5.") endif () else () - message (WARNING "You are using an unsupported compiler! Compilation has only been tested with Clang 5+ and GCC 7+.") + message (WARNING "You are using an unsupported compiler. Compilation has only been tested with Clang 5+ and GCC 7+.") endif () +# Check that submodules are present only if source was downloaded with git +if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git" AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/boost/boost") + message (FATAL_ERROR "Submodules are not initialized. Run\n\tgit submodule update --init --recursive") +endif () # Write compile_commands.json set(CMAKE_EXPORT_COMPILE_COMMANDS 1) @@ -28,15 +32,35 @@ if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None") set (CMAKE_BUILD_TYPE "RELWITHDEBINFO") endif () string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC) -message (STATUS "CMAKE_BUILD_TYPE: " ${CMAKE_BUILD_TYPE} ) +message (STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}") -# ASan - build type with address sanitizer -# UBSan - build type with undefined behaviour sanitizer -# TSan is not supported due to false positive errors in libstdc++ and necessity to rebuild libstdc++ with TSan -set (CMAKE_CONFIGURATION_TYPES "RelWithDebInfo;Debug;Release;MinSizeRel;ASan;UBSan" CACHE STRING "" FORCE) +set (CMAKE_CONFIGURATION_TYPES "RelWithDebInfo;Debug;Release;MinSizeRel" CACHE STRING "" FORCE) + +option (USE_STATIC_LIBRARIES "Set to FALSE to use shared libraries" ON) +option (MAKE_STATIC_LIBRARIES "Set to FALSE to make shared libraries" ${USE_STATIC_LIBRARIES}) +if (NOT MAKE_STATIC_LIBRARIES) + option (SPLIT_SHARED_LIBRARIES "DEV ONLY. Keep all internal libs as separate .so for faster linking" OFF) + option (CLICKHOUSE_SPLIT_BINARY "Make several binaries instead one bundled (clickhouse-server, clickhouse-client, ... )" OFF) +endif () + +if (SPLIT_SHARED_LIBRARIES) + set (LINK_MODE SHARED) +endif () + +if (USE_STATIC_LIBRARIES) + list(REVERSE CMAKE_FIND_LIBRARY_SUFFIXES) +endif () + +include (cmake/sanitize.cmake) include (cmake/arch.cmake) +if (CMAKE_GENERATOR STREQUAL "Ninja") + # Turn on colored output. https://github.com/ninja-build/ninja/wiki/FAQ + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-color=always") +endif () + if (NOT MSVC) set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wall") # -Werror is also added inside directories with our own code. endif () @@ -51,32 +75,13 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wno-unused-command-line-argument") endif () -if (ARCH_LINUX) - set (CXX11_ABI "ENABLE" CACHE STRING "Use C++11 ABI: DEFAULT, ENABLE, DISABLE") -endif () - option (TEST_COVERAGE "Enables flags for test coverage" OFF) -option (ENABLE_TESTS "Enables tests" ${NOT_MSVC}) +option (ENABLE_TESTS "Enables tests" ON) -option (USE_STATIC_LIBRARIES "Set to FALSE to use shared libraries" ON) -option (MAKE_STATIC_LIBRARIES "Set to FALSE to make shared libraries" ${USE_STATIC_LIBRARIES}) -if (NOT MAKE_STATIC_LIBRARIES) - option (SPLIT_SHARED_LIBRARIES "DEV ONLY. Keep all internal libs as separate .so for faster linking" OFF) - option (CLICKHOUSE_SPLIT_BINARY "Make several binaries instead one bundled (clickhouse-server, clickhouse-client, ... )" OFF) -endif () - -if (SPLIT_SHARED_LIBRARIES) - set (SPLIT_SHARED SHARED) -endif () - -if (USE_STATIC_LIBRARIES) - list(REVERSE CMAKE_FIND_LIBRARY_SUFFIXES) -endif () - -if (CMAKE_LIBRARY_ARCHITECTURE MATCHES "amd64.*|x86_64.*|AMD64.*") +if (CMAKE_SYSTEM_PROCESSOR MATCHES "amd64|x86_64") option (USE_INTERNAL_MEMCPY "Use internal implementation of 'memcpy' function instead of provided by libc. Only for x86_64." ON) - if (ARCH_LINUX) + if (OS_LINUX) option (GLIBC_COMPATIBILITY "Set to TRUE to enable compatibility with older glibc libraries. Only for x86_64, Linux. Implies USE_INTERNAL_MEMCPY." ON) endif() endif () @@ -85,15 +90,7 @@ if (GLIBC_COMPATIBILITY) set (USE_INTERNAL_MEMCPY ON) endif () -if (CXX11_ABI STREQUAL ENABLE) - set (CXX11_ABI_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=1") -elseif (CXX11_ABI STREQUAL DISABLE) - set (CXX11_ABI_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=0") -else () - set (CXX11_ABI_FLAGS "") -endif () - -set (COMPILER_FLAGS "${COMPILER_FLAGS} ${CXX11_ABI_FLAGS}") +set (COMPILER_FLAGS "${COMPILER_FLAGS}") string(REGEX MATCH "-?[0-9]+(.[0-9]+)?$" COMPILER_POSTFIX ${CMAKE_CXX_COMPILER}) @@ -118,8 +115,8 @@ endif () include (cmake/test_cpu.cmake) -option (ARCHNATIVE "Enable -march=native compiler flag" OFF) -if (ARCHNATIVE) +option (ARCH_NATIVE "Enable -march=native compiler flag" ${ARCH_ARM}) +if (ARCH_NATIVE) set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native") endif () @@ -129,37 +126,41 @@ endif () #endif () if (CMAKE_VERSION VERSION_LESS "3.8.0") - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1z") + if (NOT MSVC) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1z") + endif () else () set (CMAKE_CXX_STANDARD 17) set (CMAKE_CXX_EXTENSIONS 0) # https://cmake.org/cmake/help/latest/prop_tgt/CXX_EXTENSIONS.html#prop_tgt:CXX_EXTENSIONS set (CMAKE_CXX_STANDARD_REQUIRED ON) set (CXX_FLAGS_INTERNAL_COMPILER "-std=c++1z") - # This needs to propagate to vendored projects in contrib - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") endif () set (CMAKE_BUILD_COLOR_MAKEFILE ON) -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS}") +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS}") #set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${CMAKE_CXX_FLAGS_ADD}") set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_CXX_FLAGS_ADD}") set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_CXX_FLAGS_ADD}") -set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS} ${CMAKE_C_FLAGS_ADD}") +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CMAKE_C_FLAGS_ADD}") #set (CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${CMAKE_C_FLAGS_ADD}") set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_C_FLAGS_ADD}") set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_C_FLAGS_ADD}") -if (MAKE_STATIC_LIBRARIES AND NOT APPLE AND NOT (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND ARCH_FREEBSD)) +if (MAKE_STATIC_LIBRARIES AND NOT APPLE AND NOT (COMPILER_CLANG AND OS_FREEBSD)) set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++") + + # Along with executables, we also build example of shared library for "library dictionary source"; and it also should be self-contained. + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-libgcc -static-libstdc++") endif () set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package (Threads) include (cmake/test_compiler.cmake) -if (ARCH_LINUX AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${GLIBC_COMPATIBILITY_LINK_FLAGS} ${CXX11_ABI_FLAGS}") +if (OS_LINUX AND COMPILER_CLANG) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}") option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++ (only make sense on Linux with Clang)" ${HAVE_LIBCXX}) set (LIBCXX_PATH "" CACHE STRING "Use custom path for libc++. It should be used for MSan.") @@ -174,7 +175,7 @@ if (ARCH_LINUX AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang") endif () if (LIBCXX_PATH) -# include_directories (BEFORE SYSTEM "${LIBCXX_PATH}/include" "${LIBCXX_PATH}/include/c++/v1") +# include_directories (SYSTEM BEFORE "${LIBCXX_PATH}/include" "${LIBCXX_PATH}/include/c++/v1") link_directories ("${LIBCXX_PATH}/lib") endif () endif () @@ -189,8 +190,6 @@ if (NOT MAKE_STATIC_LIBRARIES) set(CMAKE_POSITION_INDEPENDENT_CODE ON) endif () -include (cmake/sanitize.cmake) - # Using "include-what-you-use" tool. option (USE_INCLUDE_WHAT_YOU_USE "Use 'include-what-you-use' tool" OFF) if (USE_INCLUDE_WHAT_YOU_USE) @@ -220,14 +219,14 @@ else () set (CLICKHOUSE_ETC_DIR "${CMAKE_INSTALL_PREFIX}/etc") endif () -option (UNBUNDLED "Try find all libraries in system (if fail - use bundled from contrib/)" OFF) +option (UNBUNDLED "Try find all libraries in system. We recommend to avoid this mode for production builds, because we cannot guarantee exact versions and variants of libraries your system has installed. This mode exists for enthusiastic developers who search for trouble. Also it is useful for maintainers of OS packages." OFF) if (UNBUNDLED) set(NOT_UNBUNDLED 0) else () set(NOT_UNBUNDLED 1) endif () # Using system libs can cause lot of warnings in includes. -if (UNBUNDLED OR NOT (ARCH_LINUX OR APPLE) OR ARCH_32) +if (UNBUNDLED OR NOT (OS_LINUX OR APPLE) OR ARCH_32) option (NO_WERROR "Disable -Werror compiler option" ON) endif () @@ -236,33 +235,31 @@ message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE include(GNUInstallDirs) include (cmake/find_ssl.cmake) -if (NOT OPENSSL_FOUND) - message (FATAL_ERROR "Need openssl for build. debian tip: sudo apt install libssl-dev") -endif () - include (cmake/lib_name.cmake) -include (cmake/find_icu4c.cmake) +include (cmake/find_icu.cmake) include (cmake/find_boost.cmake) -# openssl, zlib before poco include (cmake/find_zlib.cmake) include (cmake/find_zstd.cmake) include (cmake/find_ltdl.cmake) # for odbc -if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/contrib/poco/cmake/FindODBC.cmake) - include (${CMAKE_CURRENT_SOURCE_DIR}/contrib/poco/cmake/FindODBC.cmake) # for poco -else () - include (cmake/find_odbc.cmake) -endif () +include (cmake/find_termcap.cmake) +include (cmake/find_odbc.cmake) +# openssl, zlib, odbc before poco include (cmake/find_poco.cmake) include (cmake/find_lz4.cmake) include (cmake/find_sparsehash.cmake) include (cmake/find_rt.cmake) include (cmake/find_execinfo.cmake) include (cmake/find_readline_edit.cmake) -include (cmake/find_zookeeper.cmake) include (cmake/find_re2.cmake) include (cmake/find_rdkafka.cmake) include (cmake/find_capnp.cmake) include (cmake/find_llvm.cmake) +include (cmake/find_cpuid.cmake) +include (cmake/find_consistent-hashing.cmake) +include (cmake/find_base64.cmake) +if (ENABLE_TESTS) + include (cmake/find_gtest.cmake) +endif () include (cmake/find_contrib_lib.cmake) find_contrib_lib(cityhash) @@ -280,12 +277,6 @@ include (libs/libdaemon/cmake/find_unwind.cmake) include (cmake/print_flags.cmake) -# Directory for Yandex specific files -set (CLICKHOUSE_PRIVATE_DIR ${ClickHouse_SOURCE_DIR}/private/) -if (EXISTS ${CLICKHOUSE_PRIVATE_DIR}) - add_subdirectory (${CLICKHOUSE_PRIVATE_DIR}) -endif () - add_subdirectory (contrib) add_subdirectory (libs) add_subdirectory (utils) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ec8f5735957..adaea133aee 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,32 +1,34 @@ -# Contributing to ClickHouse - -## Technical info -Developer guide for writing code for ClickHouse is published on official website alongside the usage and operations documentation: -https://clickhouse.yandex/docs/en/development/index.html - -## Legal info - -In order for us (YANDEX LLC) to accept patches and other contributions from you, you will have to adopt our Yandex Contributor License Agreement (the "**CLA**"). The current version of the CLA you may find here: -1) https://yandex.ru/legal/cla/?lang=en (in English) and -2) https://yandex.ru/legal/cla/?lang=ru (in Russian). - -By adopting the CLA, you state the following: - -* You obviously wish and are willingly licensing your contributions to us for our open source projects under the terms of the CLA, -* You has read the terms and conditions of the CLA and agree with them in full, -* You are legally able to provide and license your contributions as stated, -* We may use your contributions for our open source projects and for any other our project too, -* We rely on your assurances concerning the rights of third parties in relation to your contributes. - -If you agree with these principles, please read and adopt our CLA. By providing us your contributions, you hereby declare that you has already read and adopt our CLA, and we may freely merge your contributions with our corresponding open source project and use it in further in accordance with terms and conditions of the CLA. - -If you have already adopted terms and conditions of the CLA, you are able to provide your contributes. When you submit your pull request, please add the following information into it: - -``` -I hereby agree to the terms of the CLA available at: [link]. -``` - -Replace the bracketed text as follows: -* [link] is the link at the current version of the CLA (you may add here a link https://yandex.ru/legal/cla/?lang=en (in English) or a link https://yandex.ru/legal/cla/?lang=ru (in Russian). - -It is enough to provide us such notification at once. +# Contributing to ClickHouse + +## Technical info +Developer guide for writing code for ClickHouse is published on official website alongside the usage and operations documentation: +https://clickhouse.yandex/docs/en/development/architecture/ + +## Legal info + +In order for us (YANDEX LLC) to accept patches and other contributions from you, you will have to adopt our Yandex Contributor License Agreement (the "**CLA**"). The current version of the CLA you may find here: +1) https://yandex.ru/legal/cla/?lang=en (in English) and +2) https://yandex.ru/legal/cla/?lang=ru (in Russian). + +By adopting the CLA, you state the following: + +* You obviously wish and are willingly licensing your contributions to us for our open source projects under the terms of the CLA, +* You has read the terms and conditions of the CLA and agree with them in full, +* You are legally able to provide and license your contributions as stated, +* We may use your contributions for our open source projects and for any other our project too, +* We rely on your assurances concerning the rights of third parties in relation to your contributes. + +If you agree with these principles, please read and adopt our CLA. By providing us your contributions, you hereby declare that you has already read and adopt our CLA, and we may freely merge your contributions with our corresponding open source project and use it in further in accordance with terms and conditions of the CLA. + +If you have already adopted terms and conditions of the CLA, you are able to provide your contributes. When you submit your pull request, please add the following information into it: + +``` +I hereby agree to the terms of the CLA available at: [link]. +``` + +Replace the bracketed text as follows: +* [link] is the link at the current version of the CLA (you may add here a link https://yandex.ru/legal/cla/?lang=en (in English) or a link https://yandex.ru/legal/cla/?lang=ru (in Russian). + +It is enough to provide us such notification at once. + +If you don't agree with the CLA, you still can open a pull request to provide your contributions. diff --git a/MacOS.md b/MacOS.md deleted file mode 100644 index 93c04d7c2de..00000000000 --- a/MacOS.md +++ /dev/null @@ -1,39 +0,0 @@ -## How to increase maxfiles on macOS - -To increase maxfiles on macOS, create the following file: - -(Note: you'll need to use sudo) - -/Library/LaunchDaemons/limit.maxfiles.plist: -``` - - - - - Label - limit.maxfiles - ProgramArguments - - launchctl - limit - maxfiles - 524288 - 524288 - - RunAtLoad - - ServiceIPC - - - -``` - -Execute the following command: -``` -sudo chown root:wheel /Library/LaunchDaemons/limit.maxfiles.plist -``` - -Reboot. - -To check if it's working, you can use `ulimit -n` command. diff --git a/PreLoad.cmake b/PreLoad.cmake new file mode 100644 index 00000000000..c050f00e92c --- /dev/null +++ b/PreLoad.cmake @@ -0,0 +1,21 @@ +# Use Ninja instead of Unix Makefiles by default. +# https://stackoverflow.com/questions/11269833/cmake-selecting-a-generator-within-cmakelists-txt +# +# Reason: it have better startup time than make and it parallelize jobs more uniformly. +# (when comparing to make with Makefiles that was generated by CMake) +# +# How to install Ninja on Ubuntu: +# sudo apt-get install ninja-build + +# CLion does not support Ninja +# You can add your vote on CLion task tracker: +# https://youtrack.jetbrains.com/issue/CPP-2659 +# https://youtrack.jetbrains.com/issue/CPP-870 + +string(TOLOWER "${CMAKE_COMMAND}" CMAKE_COMMAND_LOWER) +if (NOT ${CMAKE_COMMAND_LOWER} MATCHES "clion") + find_program(NINJA_PATH ninja) + if (NINJA_PATH) + set(CMAKE_GENERATOR "Ninja" CACHE INTERNAL "" FORCE) + endif () +endif() diff --git a/README.md b/README.md index 8f93b8d9af1..1f4542e6afe 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,12 @@ +# ClickHouse + ClickHouse is an open-source column-oriented database management system that allows generating analytical data reports in real time. -[Read more...](https://clickhouse.yandex/) +## Useful Links -[![Build Status](https://travis-ci.org/yandex/ClickHouse.svg?branch=master)](https://travis-ci.org/yandex/ClickHouse) +* [Official website](https://clickhouse.yandex/) has quick high-level overview of ClickHouse on main page. +* [Tutorial](https://clickhouse.yandex/tutorial.html) shows how to set up and query small ClickHouse cluster. +* [Documentation](https://clickhouse.yandex/docs/en/) provides more in-depth information. +* [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events. +* [Contacts](https://clickhouse.yandex/#contacts) can help to get your questions answered if there are any. +* You can also [fill this form](https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/) to meet Yandex ClickHouse team in person. diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 00000000000..733cbce80c9 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,142 @@ +## Build and test ClickHouse on various plaforms + +Quick and dirty scripts. + +Usage example: +``` +./run-with-docker.sh ubuntu:bionic jobs/quick-build/run.sh +``` + +Another example, check build on ARM 64: +``` +./prepare-docker-image-ubuntu.sh +./run-with-docker.sh multiarch/ubuntu-core:arm64-bionic jobs/quick-build/run.sh +``` + +Another example, check build on FreeBSD: +``` +./prepare-vagrant-image-freebsd.sh +./run-with-vagrant.sh freebsd jobs/quick-build/run.sh +``` + +Look at `default_config` and `jobs/quick-build/run.sh` + +Various possible options. We are not going to automate testing all of them. + +#### CPU architectures: +- x86_64; +- AArch64. + +x86_64 is the main CPU architecture. We also have minimal support for AArch64. + +#### Operating systems: +- Linux; +- FreeBSD. + +We also target Mac OS X, but it's more difficult to test. +Linux is the main. FreeBSD is also supported as production OS. +Mac OS is intended only for development and have minimal support: client should work, server should just start. + +#### Linux distributions: +For build: +- Ubuntu Bionic; +- Ubuntu Trusty. + +For run: +- Ubuntu Hardy; +- CentOS 5 + +We should support almost any Linux to run ClickHouse. That's why we test also on old distributions. + +#### How to obtain sources: +- use sources from local working copy; +- clone sources from github; +- download source tarball. + +#### Compilers: +- gcc-7; +- gcc-8; +- clang-6; +- clang-svn. + +#### Compiler installation: +- from OS packages; +- build from sources. + +#### C++ standard library implementation: +- libc++; +- libstdc++ with C++11 ABI; +- libstdc++ with old ABI. + +When building with clang, libc++ is used. When building with gcc, we choose libstdc++ with C++11 ABI. + +#### Linkers: +- ldd; +- gold; + +When building with clang on x86_64, ldd is used. Otherwise we use gold. + +#### Build types: +- RelWithDebInfo; +- Debug; +- ASan; +- TSan. + +#### Build types, extra: +- -g0 for quick build; +- enable test coverage; +- debug tcmalloc. + +#### What to build: +- only `clickhouse` target; +- all targets; +- debian packages; + +We also have intent to build RPM and simple tgz packages. + +#### Where to get third-party libraries: +- from contrib directory (submodules); +- from OS packages. + +The only production option is to use libraries from contrib directory. +Using libraries from OS packages is discouraged, but we also support this option. + +#### Linkage types: +- static; +- shared; + +Static linking is the only option for production usage. +We also have support for shared linking, but it is indended only for developers. + +#### Make tools: +- make; +- ninja. + +#### Installation options: +- run built `clickhouse` binary directly; +- install from packages. + +#### How to obtain packages: +- build them; +- download from repository. + +#### Sanity checks: +- check that clickhouse binary has no dependencies on unexpected shared libraries; +- check that source code have no style violations. + +#### Tests: +- Functional tests; +- Integration tests; +- Unit tests; +- Simple sh/reference tests; +- Performance tests (note that they require predictable computing power); +- Tests for external dictionaries (should be moved to integration tests); +- Jepsen like tests for quorum inserts (not yet available in opensource). + +#### Tests extra: +- Run functional tests with Valgrind. + +#### Static analyzers: +- CppCheck; +- clang-tidy; +- Coverity. diff --git a/ci/build-clang-from-sources.sh b/ci/build-clang-from-sources.sh new file mode 100755 index 00000000000..fdd7abf5acc --- /dev/null +++ b/ci/build-clang-from-sources.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -e -x + +source default-config + +# TODO Non debian systems +./install-os-packages.sh svn +./install-os-packages.sh cmake + +mkdir "${WORKSPACE}/llvm" + +svn co "http://llvm.org/svn/llvm-project/llvm/${CLANG_SOURCES_BRANCH}" "${WORKSPACE}/llvm/llvm" +svn co "http://llvm.org/svn/llvm-project/cfe/${CLANG_SOURCES_BRANCH}" "${WORKSPACE}/llvm/llvm/tools/clang" +svn co "http://llvm.org/svn/llvm-project/lld/${CLANG_SOURCES_BRANCH}" "${WORKSPACE}/llvm/llvm/tools/lld" +svn co "http://llvm.org/svn/llvm-project/polly/${CLANG_SOURCES_BRANCH}" "${WORKSPACE}/llvm/llvm/tools/polly" +svn co "http://llvm.org/svn/llvm-project/clang-tools-extra/${CLANG_SOURCES_BRANCH}" "${WORKSPACE}/llvm/llvm/tools/clang/tools/extra" +svn co "http://llvm.org/svn/llvm-project/compiler-rt/${CLANG_SOURCES_BRANCH}" "${WORKSPACE}/llvm/llvm/projects/compiler-rt" +svn co "http://llvm.org/svn/llvm-project/libcxx/${CLANG_SOURCES_BRANCH}" "${WORKSPACE}/llvm/llvm/projects/libcxx" +svn co "http://llvm.org/svn/llvm-project/libcxxabi/${CLANG_SOURCES_BRANCH}" "${WORKSPACE}/llvm/llvm/projects/libcxxabi" + +mkdir "${WORKSPACE}/llvm/build" +cd "${WORKSPACE}/llvm/build" + +# NOTE You must build LLVM with the same ABI as ClickHouse. +# For example, if you compile ClickHouse with libc++, you must add +# -DLLVM_ENABLE_LIBCXX=1 +# to the line below. + +cmake -DCMAKE_BUILD_TYPE:STRING=Release -DLLVM_ENABLE_LIBCXX=1 -DLLVM_ENABLE_RTTI=1 ../llvm + +make -j $THREADS +$SUDO make install +hash clang + +cd ../../.. + +export CC=clang +export CXX=clang++ diff --git a/ci/build-debian-packages.sh b/ci/build-debian-packages.sh new file mode 100755 index 00000000000..8726b675c4d --- /dev/null +++ b/ci/build-debian-packages.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -e -x + +source default-config + +[[ -d "${WORKSPACE}/sources" ]] || die "Run get-sources.sh first" + +./sources/release diff --git a/ci/build-gcc-from-sources.sh b/ci/build-gcc-from-sources.sh new file mode 100755 index 00000000000..0734b22335a --- /dev/null +++ b/ci/build-gcc-from-sources.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -e -x + +source default-config + +./install-os-packages.sh curl + +if [[ "${GCC_SOURCES_VERSION}" == "latest" ]]; then + GCC_SOURCES_VERSION=$(curl -sSL https://ftpmirror.gnu.org/gcc/ | grep -oE 'gcc-[0-9]+(\.[0-9]+)+' | sort -Vr | head -n1) +fi + +GCC_VERSION_SHORT=$(echo "$GCC_SOURCES_VERSION" | grep -oE '[0-9]' | head -n1) + +echo "Will download ${GCC_SOURCES_VERSION} (short version: $GCC_VERSION_SHORT)." + +THREADS=$(grep -c ^processor /proc/cpuinfo) + +mkdir "${WORKSPACE}/gcc" +pushd "${WORKSPACE}/gcc" + +wget https://ftpmirror.gnu.org/gcc/${GCC_SOURCES_VERSION}/${GCC_SOURCES_VERSION}.tar.xz +tar xf ${GCC_SOURCES_VERSION}.tar.xz +pushd ${GCC_SOURCES_VERSION} +./contrib/download_prerequisites +popd +mkdir gcc-build +pushd gcc-build +../${GCC_SOURCES_VERSION}/configure --enable-languages=c,c++ --disable-multilib +make -j $THREADS +$SUDO make install + +popd +popd + +$SUDO ln -sf /usr/local/bin/gcc /usr/local/bin/gcc-${GCC_GCC_SOURCES_VERSION_SHORT} +$SUDO ln -sf /usr/local/bin/g++ /usr/local/bin/g++-${GCC_GCC_SOURCES_VERSION_SHORT} +$SUDO ln -sf /usr/local/bin/gcc /usr/local/bin/cc +$SUDO ln -sf /usr/local/bin/g++ /usr/local/bin/c++ + +echo '/usr/local/lib64' | $SUDO tee /etc/ld.so.conf.d/10_local-lib64.conf +$SUDO ldconfig + +hash gcc g++ +gcc --version + +export CC=gcc +export CXX=g++ diff --git a/ci/build-normal.sh b/ci/build-normal.sh new file mode 100755 index 00000000000..b937269c8a3 --- /dev/null +++ b/ci/build-normal.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -e -x + +source default-config + +[[ -d "${WORKSPACE}/sources" ]] || die "Run get-sources.sh first" + +mkdir -p "${WORKSPACE}/build" +pushd "${WORKSPACE}/build" + +if [[ "${ENABLE_EMBEDDED_COMPILER}" == 1 ]]; then + [[ "$USE_LLVM_LIBRARIES_FROM_SYSTEM" == 0 ]] && CMAKE_FLAGS="$CMAKE_FLAGS -DUSE_INTERNAL_LLVM_LIBRARY=1" + [[ "$USE_LLVM_LIBRARIES_FROM_SYSTEM" != 0 ]] && CMAKE_FLAGS="$CMAKE_FLAGS -DUSE_INTERNAL_LLVM_LIBRARY=0" +fi + +cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DENABLE_EMBEDDED_COMPILER=${ENABLE_EMBEDDED_COMPILER} $CMAKE_FLAGS ../sources + +[[ "$BUILD_TARGETS" != 'all' ]] && BUILD_TARGETS_STRING="--target $BUILD_TARGETS" + +cmake --build . $BUILD_TARGETS_STRING -- -j $THREADS + +popd diff --git a/ci/check-docker.sh b/ci/check-docker.sh new file mode 100755 index 00000000000..e4ffb11f643 --- /dev/null +++ b/ci/check-docker.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -e -x + +source default-config + +command -v docker > /dev/null || die "You need to install Docker" +docker ps > /dev/null || die "You need to have access to Docker: run '$SUDO usermod -aG docker $USER' and relogin" diff --git a/ci/check-syntax.sh b/ci/check-syntax.sh new file mode 100755 index 00000000000..df233965f9e --- /dev/null +++ b/ci/check-syntax.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -e -x + +source default-config + +./install-os-packages.sh jq + +[[ -d "${WORKSPACE}/sources" ]] || die "Run get-sources.sh first" + +mkdir -p "${WORKSPACE}/build" +pushd "${WORKSPACE}/build" + +cmake -DCMAKE_BUILD_TYPE=Debug $CMAKE_FLAGS ../sources + +make -j $THREADS re2_st # Generated headers + +jq --raw-output '.[] | .command' compile_commands.json | grep -v -P -- '-c .+/contrib/' | sed -r -e 's/-o\s+\S+/-fsyntax-only/' > syntax-commands +xargs --arg-file=syntax-commands --max-procs=$THREADS --replace /bin/sh -c "{}" + +popd diff --git a/ci/create-sources-tarball.sh b/ci/create-sources-tarball.sh new file mode 100755 index 00000000000..bfbbf61e556 --- /dev/null +++ b/ci/create-sources-tarball.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -e -x + +source default-config + +if [[ -d "${WORKSPACE}/sources" ]]; then + tar -c -z -f "${WORKSPACE}/sources.tar.gz" --directory "${WORKSPACE}/sources" . +else + die "Run get-sources first" +fi diff --git a/ci/default-config b/ci/default-config new file mode 100644 index 00000000000..26e82ddcceb --- /dev/null +++ b/ci/default-config @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +set -e -x + +if [[ -z "$INITIALIZED" ]]; then + +INITIALIZED=1 + +SCRIPTPATH=$(pwd) +WORKSPACE=${SCRIPTPATH}/workspace +PROJECT_ROOT=$(cd $SCRIPTPATH/.. && pwd) + +# Almost all scripts take no arguments. Arguments should be in config. + +# get-sources +SOURCES_METHOD=local # clone, local, tarball +SOURCES_CLONE_URL="https://github.com/yandex/ClickHouse.git" +SOURCES_BRANCH="master" +SOURCES_COMMIT=HEAD # do checkout of this commit after clone + +# prepare-toolchain +COMPILER=gcc # gcc, clang +COMPILER_INSTALL_METHOD=packages # packages, sources +COMPILER_PACKAGE_VERSION=7 # or 6.0 for clang + +# install-compiler-from-sources +CLANG_SOURCES_BRANCH=trunk # or tags/RELEASE_600/final +GCC_SOURCES_VERSION=latest # or gcc-7.1.0 + +# install-libraries +USE_LLVM_LIBRARIES_FROM_SYSTEM=0 # 0 or 1 +ENABLE_EMBEDDED_COMPILER=1 + +# build +BUILD_METHOD=normal # normal, debian +BUILD_TARGETS=clickhouse # tagtet name, all; only for "normal" +BUILD_TYPE=RelWithDebInfo # RelWithDebInfo, Debug, ASan, TSan +CMAKE_FLAGS="" + +# prepare-docker-image-ubuntu +DOCKER_UBUNTU_VERSION=bionic +DOCKER_UBUNTU_ARCH=arm64 # How the architecture is named in a tarball at https://partner-images.canonical.com/core/ +DOCKER_UBUNTU_QUEMU_ARCH=aarch64 # How the architecture is named in QEMU +DOCKER_UBUNTU_TAG_ARCH=arm64 # How the architecture is named in Docker +DOCKER_UBUNTU_QEMU_VER=v2.9.1 +DOCKER_UBUNTU_REPO=multiarch/ubuntu-core + +THREADS=$(grep -c ^processor /proc/cpuinfo || nproc || sysctl -a | grep -F 'hw.ncpu' | grep -oE '[0-9]+') + +# All scripts should return 0 in case of success, 1 in case of permanent error, +# 2 in case of temporary error, any other code in case of permanent error. +function die { + echo ${1:-Error} + exit ${2:1} +} + +[[ $EUID -ne 0 ]] && SUDO=sudo + +./install-os-packages.sh prepare + +# Configuration parameters may be overriden with CONFIG environment variable pointing to config file. +[[ -n "$CONFIG" ]] && source $CONFIG + +mkdir -p $WORKSPACE + +fi diff --git a/ci/docker-multiarch/LICENSE b/ci/docker-multiarch/LICENSE new file mode 100644 index 00000000000..60d0ea8fa82 --- /dev/null +++ b/ci/docker-multiarch/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Multiarch + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ci/docker-multiarch/README.md b/ci/docker-multiarch/README.md new file mode 100644 index 00000000000..9cc30c2dca8 --- /dev/null +++ b/ci/docker-multiarch/README.md @@ -0,0 +1,53 @@ +Source: https://github.com/multiarch/ubuntu-core +Commit: 3972a7794b40a965615abd710759d3ed439c9a55 + +# :earth_africa: ubuntu-core + +![](https://raw.githubusercontent.com/multiarch/dockerfile/master/logo.jpg) + +Multiarch Ubuntu images for Docker. + +Based on https://github.com/tianon/docker-brew-ubuntu-core/ + +* `multiarch/ubuntu-core` on [Docker Hub](https://hub.docker.com/r/multiarch/ubuntu-core/) +* [Available tags](https://hub.docker.com/r/multiarch/ubuntu-core/tags/) + +## Usage + +Once you need to configure binfmt-support on your Docker host. +This works locally or remotely (i.e using boot2docker or swarm). + +```console +# configure binfmt-support on the Docker host (works locally or remotely, i.e: using boot2docker) +$ docker run --rm --privileged multiarch/qemu-user-static:register --reset +``` + +Then you can run an `armhf` image from your `x86_64` Docker host. + +```console +$ docker run -it --rm multiarch/ubuntu-core:armhf-wily +root@a0818570f614:/# uname -a +Linux a0818570f614 4.1.13-boot2docker #1 SMP Fri Nov 20 19:05:50 UTC 2015 armv7l armv7l armv7l GNU/Linux +root@a0818570f614:/# exit +``` + +Or an `x86_64` image from your `x86_64` Docker host, directly, without qemu emulation. + +```console +$ docker run -it --rm multiarch/ubuntu-core:amd64-wily +root@27fe384370c9:/# uname -a +Linux 27fe384370c9 4.1.13-boot2docker #1 SMP Fri Nov 20 19:05:50 UTC 2015 x86_64 x86_64 x86_64 GNU/Linux +root@27fe384370c9:/# +``` + +It also works for `arm64` + +```console +$ docker run -it --rm multiarch/ubuntu-core:arm64-wily +root@723fb9f184fa:/# uname -a +Linux 723fb9f184fa 4.1.13-boot2docker #1 SMP Fri Nov 20 19:05:50 UTC 2015 aarch64 aarch64 aarch64 GNU/Linux +``` + +## License + +MIT diff --git a/ci/docker-multiarch/update.sh b/ci/docker-multiarch/update.sh new file mode 100755 index 00000000000..6e3e18047de --- /dev/null +++ b/ci/docker-multiarch/update.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -e -x + +# A POSIX variable +OPTIND=1 # Reset in case getopts has been used previously in the shell. + +while getopts "a:v:q:u:d:t:" opt; do + case "$opt" in + a) ARCH=$OPTARG + ;; + v) VERSION=$OPTARG + ;; + q) QEMU_ARCH=$OPTARG + ;; + u) QEMU_VER=$OPTARG + ;; + d) DOCKER_REPO=$OPTARG + ;; + t) TAG_ARCH=$OPTARG + ;; + esac +done + +thisTarBase="ubuntu-$VERSION-core-cloudimg-$ARCH" +thisTar="$thisTarBase-root.tar.gz" +baseUrl="https://partner-images.canonical.com/core/$VERSION" + + +# install qemu-user-static +if [ -n "${QEMU_ARCH}" ]; then + if [ ! -f x86_64_qemu-${QEMU_ARCH}-static.tar.gz ]; then + wget -N https://github.com/multiarch/qemu-user-static/releases/download/${QEMU_VER}/x86_64_qemu-${QEMU_ARCH}-static.tar.gz + fi + tar -xvf x86_64_qemu-${QEMU_ARCH}-static.tar.gz -C $ROOTFS/usr/bin/ +fi + + +# get the image +if \ + wget -q --spider "$baseUrl/current" \ + && wget -q --spider "$baseUrl/current/$thisTar" \ + ; then + baseUrl+='/current' +fi +wget -qN "$baseUrl/"{{MD5,SHA{1,256}}SUMS{,.gpg},"$thisTarBase.manifest",'unpacked/build-info.txt'} || true +wget -N "$baseUrl/$thisTar" + +# check checksum +if [ -f SHA256SUMS ]; then + sha256sum="$(sha256sum "$thisTar" | cut -d' ' -f1)" + if ! grep -q "$sha256sum" SHA256SUMS; then + echo >&2 "error: '$thisTar' has invalid SHA256" + exit 1 + fi +fi + +cat > Dockerfile <<-EOF + FROM scratch + ADD $thisTar / + ENV ARCH=${ARCH} UBUNTU_SUITE=${VERSION} DOCKER_REPO=${DOCKER_REPO} +EOF + +# add qemu-user-static binary +if [ -n "${QEMU_ARCH}" ]; then + cat >> Dockerfile <> Dockerfile <<-EOF + # a few minor docker-specific tweaks + # see https://github.com/docker/docker/blob/master/contrib/mkimage/debootstrap + RUN echo '#!/bin/sh' > /usr/sbin/policy-rc.d \\ + && echo 'exit 101' >> /usr/sbin/policy-rc.d \\ + && chmod +x /usr/sbin/policy-rc.d \\ + && dpkg-divert --local --rename --add /sbin/initctl \\ + && cp -a /usr/sbin/policy-rc.d /sbin/initctl \\ + && sed -i 's/^exit.*/exit 0/' /sbin/initctl \\ + && echo 'force-unsafe-io' > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup \\ + && echo 'DPkg::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' > /etc/apt/apt.conf.d/docker-clean \\ + && echo 'APT::Update::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' >> /etc/apt/apt.conf.d/docker-clean \\ + && echo 'Dir::Cache::pkgcache ""; Dir::Cache::srcpkgcache "";' >> /etc/apt/apt.conf.d/docker-clean \\ + && echo 'Acquire::Languages "none";' > /etc/apt/apt.conf.d/docker-no-languages \\ + && echo 'Acquire::GzipIndexes "true"; Acquire::CompressionTypes::Order:: "gz";' > /etc/apt/apt.conf.d/docker-gzip-indexes + + # enable the universe + RUN sed -i 's/^#\s*\(deb.*universe\)$/\1/g' /etc/apt/sources.list + + # overwrite this with 'CMD []' in a dependent Dockerfile + CMD ["/bin/bash"] +EOF + +docker build -t "${DOCKER_REPO}:${TAG_ARCH}-${VERSION}" . +docker run --rm "${DOCKER_REPO}:${TAG_ARCH}-${VERSION}" /bin/bash -ec "echo Hello from Ubuntu!" diff --git a/ci/get-sources.sh b/ci/get-sources.sh new file mode 100755 index 00000000000..ee57b0ec27d --- /dev/null +++ b/ci/get-sources.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -e -x + +source default-config + +if [[ "$SOURCES_METHOD" == "clone" ]]; then + ./install-os-packages.sh git + SOURCES_DIR="${WORKSPACE}/sources" + mkdir -p "${SOURCES_DIR}" + git clone --recursive --branch "$SOURCES_BRANCH" "$SOURCES_CLONE_URL" "${SOURCES_DIR}" + pushd "${SOURCES_DIR}" + git checkout --recurse-submodules "$SOURCES_COMMIT" + popd +elif [[ "$SOURCES_METHOD" == "local" ]]; then + ln -f -s "${PROJECT_ROOT}" "${WORKSPACE}/sources" +else + die "Unknown SOURCES_METHOD" +fi diff --git a/ci/install-compiler-from-packages.sh b/ci/install-compiler-from-packages.sh new file mode 100755 index 00000000000..53909435a06 --- /dev/null +++ b/ci/install-compiler-from-packages.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -e -x + +source default-config + +# TODO Install from PPA on older Ubuntu + +./install-os-packages.sh ${COMPILER}-${COMPILER_PACKAGE_VERSION} + +if [[ "$COMPILER" == "gcc" ]]; then + if command -v gcc-${COMPILER_PACKAGE_VERSION}; then export CC=gcc-${COMPILER_PACKAGE_VERSION} CXX=g++-${COMPILER_PACKAGE_VERSION}; + elif command -v gcc${COMPILER_PACKAGE_VERSION}; then export CC=gcc${COMPILER_PACKAGE_VERSION} CXX=g++${COMPILER_PACKAGE_VERSION}; + elif command -v gcc; then export CC=gcc CXX=g++; + fi +elif [[ "$COMPILER" == "clang" ]]; then + if command -v clang-${COMPILER_PACKAGE_VERSION}; then export CC=clang-${COMPILER_PACKAGE_VERSION} CXX=clang++-${COMPILER_PACKAGE_VERSION}; + elif command -v clang${COMPILER_PACKAGE_VERSION}; then export CC=clang${COMPILER_PACKAGE_VERSION} CXX=clang++${COMPILER_PACKAGE_VERSION}; + elif command -v clang; then export CC=clang CXX=clang++; + fi +else + die "Unknown compiler specified" +fi diff --git a/ci/install-compiler-from-sources.sh b/ci/install-compiler-from-sources.sh new file mode 100755 index 00000000000..235898ed300 --- /dev/null +++ b/ci/install-compiler-from-sources.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -e -x + +source default-config + +if [[ "$COMPILER" == "gcc" ]]; then + . build-gcc-from-sources.sh +elif [[ "$COMPILER" == "clang" ]]; then + . build-clang-from-sources.sh +else + die "Unknown COMPILER" +fi diff --git a/ci/install-libraries.sh b/ci/install-libraries.sh new file mode 100755 index 00000000000..d7fb856dbed --- /dev/null +++ b/ci/install-libraries.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -e -x + +source default-config + +./install-os-packages.sh libicu-dev +./install-os-packages.sh libreadline-dev + +if [[ "$ENABLE_EMBEDDED_COMPILER" == 1 && "$USE_LLVM_LIBRARIES_FROM_SYSTEM" == 1 ]]; then + ./install-os-packages.sh llvm-libs-5.0 +fi diff --git a/ci/install-os-packages.sh b/ci/install-os-packages.sh new file mode 100755 index 00000000000..fe5b4f84833 --- /dev/null +++ b/ci/install-os-packages.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +set -e -x + +# Dispatches package installation on various OS and distributives + +WHAT=$1 + +[[ $EUID -ne 0 ]] && SUDO=sudo + +command -v apt-get && PACKAGE_MANAGER=apt +command -v yum && PACKAGE_MANAGER=yum +command -v pkg && PACKAGE_MANAGER=pkg + + +case $PACKAGE_MANAGER in + apt) + case $WHAT in + prepare) + $SUDO apt-get update + ;; + svn) + $SUDO apt-get install -y subversion + ;; + gcc*) + $SUDO apt-get install -y $WHAT ${WHAT/cc/++} + ;; + clang*) + $SUDO apt-get install -y $WHAT libc++-dev libc++abi-dev + [[ $(uname -m) == "x86_64" ]] && $SUDO apt-get install -y ${WHAT/clang/lld} || true + ;; + git) + $SUDO apt-get install -y git + ;; + cmake) + $SUDO apt-get install -y cmake3 || $SUDO apt-get install -y cmake + ;; + ninja) + $SUDO apt-get install -y ninja-build + ;; + curl) + $SUDO apt-get install -y curl + ;; + jq) + $SUDO apt-get install -y jq + ;; + libicu-dev) + $SUDO apt-get install -y libicu-dev + ;; + libreadline-dev) + $SUDO apt-get install -y libreadline-dev + ;; + llvm-libs*) + $SUDO apt-get install -y ${WHAT/llvm-libs/liblld}-dev ${WHAT/llvm-libs/libclang}-dev + ;; + qemu-user-static) + $SUDO apt-get install -y qemu-user-static + ;; + vagrant-virtualbox) + $SUDO apt-get install -y vagrant virtualbox + ;; + *) + echo "Unknown package"; exit 1; + ;; + esac + ;; + yum) + case $WHAT in + prepare) + ;; + svn) + $SUDO yum install -y subversion + ;; + gcc*) + $SUDO yum install -y gcc gcc-c++ libstdc++-static + ;; + git) + $SUDO yum install -y git + ;; + cmake) + $SUDO yum install -y cmake + ;; + ninja) + $SUDO yum install -y ninja-build + ;; + curl) + $SUDO yum install -y curl + ;; + jq) + $SUDO yum install -y jq + ;; + libicu-dev) + $SUDO yum install -y libicu-devel + ;; + libreadline-dev) + $SUDO yum install -y readline-devel + ;; + *) + echo "Unknown package"; exit 1; + ;; + esac + ;; + pkg) + case $WHAT in + prepare) + ;; + svn) + $SUDO pkg install -y subversion + ;; + gcc*) + $SUDO pkg install -y ${WHAT/-/} + ;; + clang*) + $SUDO pkg install -y clang-devel + ;; + git) + $SUDO pkg install -y git + ;; + cmake) + $SUDO pkg install -y cmake + ;; + ninja) + $SUDO pkg install -y ninja-build + ;; + curl) + $SUDO pkg install -y curl + ;; + jq) + $SUDO pkg install -y jq + ;; + libicu-dev) + $SUDO pkg install -y icu + ;; + libreadline-dev) + $SUDO pkg install -y readline + ;; + *) + echo "Unknown package"; exit 1; + ;; + esac + ;; + *) + echo "Unknown distributive"; exit 1; + ;; +esac diff --git a/ci/jobs/quick-build/README.md b/ci/jobs/quick-build/README.md new file mode 100644 index 00000000000..803acae0f93 --- /dev/null +++ b/ci/jobs/quick-build/README.md @@ -0,0 +1,5 @@ +## Build with debug mode and without many libraries + +This job is intended as first check that build is not broken on wide variety of platforms. + +Results of this build are not intended for production usage. diff --git a/ci/jobs/quick-build/run.sh b/ci/jobs/quick-build/run.sh new file mode 100755 index 00000000000..6a948c560ee --- /dev/null +++ b/ci/jobs/quick-build/run.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -e -x + +# How to run: +# From "ci" directory: +# jobs/quick-build/run.sh +# or: +# ./run-with-docker.sh ubuntu:bionic jobs/quick-build/run.sh + +cd "$(dirname $0)"/../.. + +. default-config + +SOURCES_METHOD=local +COMPILER=clang +COMPILER_INSTALL_METHOD=packages +COMPILER_PACKAGE_VERSION=6.0 +USE_LLVM_LIBRARIES_FROM_SYSTEM=0 +BUILD_METHOD=normal +BUILD_TARGETS=clickhouse +BUILD_TYPE=Debug +ENABLE_EMBEDDED_COMPILER=0 + +CMAKE_FLAGS="-D CMAKE_C_FLAGS_ADD=-g0 -D CMAKE_CXX_FLAGS_ADD=-g0 -D ENABLE_JEMALLOC=0 -D ENABLE_CAPNP=0 -D ENABLE_RDKAFKA=0 -D ENABLE_UNWIND=0 -D ENABLE_ICU=0 -D ENABLE_POCO_MONGODB=0 -D ENABLE_POCO_NETSSL=0 -D ENABLE_POCO_ODBC=0 -D ENABLE_ODBC=0 -D ENABLE_MYSQL=0" + +[[ $(uname) == "FreeBSD" ]] && COMPILER_PACKAGE_VERSION=devel && export COMPILER_PATH=/usr/local/bin + +. get-sources.sh +. prepare-toolchain.sh +. install-libraries.sh +. build-normal.sh diff --git a/ci/prepare-docker-image-ubuntu.sh b/ci/prepare-docker-image-ubuntu.sh new file mode 100755 index 00000000000..2880d7fc1e6 --- /dev/null +++ b/ci/prepare-docker-image-ubuntu.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -e -x + +source default-config + +./check-docker.sh + +# http://fl47l1n3.net/2015/12/24/binfmt/ +./install-os-packages.sh qemu-user-static + +pushd docker-multiarch + +$SUDO ./update.sh \ + -a "$DOCKER_UBUNTU_ARCH" \ + -v "$DOCKER_UBUNTU_VERSION" \ + -q "$DOCKER_UBUNTU_QUEMU_ARCH" \ + -u "$DOCKER_UBUNTU_QEMU_VER" \ + -d "$DOCKER_UBUNTU_REPO" \ + -t "$DOCKER_UBUNTU_TAG_ARCH" + +docker run --rm --privileged multiarch/qemu-user-static:register + +popd diff --git a/ci/prepare-toolchain.sh b/ci/prepare-toolchain.sh new file mode 100755 index 00000000000..5edb19cc430 --- /dev/null +++ b/ci/prepare-toolchain.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -e -x + +source default-config + +./install-os-packages.sh cmake +./install-os-packages.sh ninja + +if [[ "$COMPILER_INSTALL_METHOD" == "packages" ]]; then + . install-compiler-from-packages.sh +elif [[ "$COMPILER_INSTALL_METHOD" == "sources" ]]; then + . install-compiler-from-sources.sh +else + die "Unknown COMPILER_INSTALL_METHOD" +fi diff --git a/ci/prepare-vagrant-image-freebsd.sh b/ci/prepare-vagrant-image-freebsd.sh new file mode 100755 index 00000000000..16c5e58c7c5 --- /dev/null +++ b/ci/prepare-vagrant-image-freebsd.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -e -x + +source default-config + +./install-os-packages.sh vagrant-virtualbox + +pushd "vagrant-freebsd" +vagrant up +vagrant ssh-config > vagrant-ssh +ssh -F vagrant-ssh default 'uname -a' +popd diff --git a/ci/run-clickhouse-from-binaries.sh b/ci/run-clickhouse-from-binaries.sh new file mode 100755 index 00000000000..f16d840316a --- /dev/null +++ b/ci/run-clickhouse-from-binaries.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -e -x + +# Usage example: +# ./run-with-docker.sh centos:centos6 ./run-clickhouse-from-binaries.sh + +source default-config + +SERVER_BIN="${WORKSPACE}/build/dbms/src/Server/clickhouse" +SERVER_CONF="${WORKSPACE}/sources/dbms/src/Server/config.xml" +SERVER_DATADIR="${WORKSPACE}/clickhouse" + +[[ -x "$SERVER_BIN" ]] || die "Run build-normal.sh first" +[[ -r "$SERVER_CONF" ]] || die "Run get-sources.sh first" + +mkdir -p "${SERVER_DATADIR}" + +$SERVER_BIN server --config-file "$SERVER_CONF" --pid-file="${WORKSPACE}/clickhouse.pid" -- --path "$SERVER_DATADIR" diff --git a/ci/run-with-docker.sh b/ci/run-with-docker.sh new file mode 100755 index 00000000000..158961dd5da --- /dev/null +++ b/ci/run-with-docker.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -e -x + +mkdir -p /var/cache/ccache +DOCKER_ENV+=" --mount=type=bind,source=/var/cache/ccache,destination=/ccache -e CCACHE_DIR=/ccache " + +PROJECT_ROOT="$(cd "$(dirname "$0")/.."; pwd -P)" +[[ -n "$CONFIG" ]] && DOCKER_ENV="--env=CONFIG" +docker run -t --network=host --mount=type=bind,source=${PROJECT_ROOT},destination=/ClickHouse --workdir=/ClickHouse/ci $DOCKER_ENV "$1" "$2" diff --git a/ci/run-with-vagrant.sh b/ci/run-with-vagrant.sh new file mode 100755 index 00000000000..620d38071eb --- /dev/null +++ b/ci/run-with-vagrant.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -e -x + +[[ -r "vagrant-${1}/vagrant-ssh" ]] || die "Run prepare-vagrant-image-... first." + +pushd vagrant-$1 + +shopt -s extglob + +vagrant ssh -c "mkdir -p ClickHouse" +scp -q -F vagrant-ssh -r ../../!(*build*) default:~/ClickHouse +vagrant ssh -c "cd ClickHouse/ci; $2" + +popd diff --git a/ci/vagrant-freebsd/.gitignore b/ci/vagrant-freebsd/.gitignore new file mode 100644 index 00000000000..8000dd9db47 --- /dev/null +++ b/ci/vagrant-freebsd/.gitignore @@ -0,0 +1 @@ +.vagrant diff --git a/ci/vagrant-freebsd/Vagrantfile b/ci/vagrant-freebsd/Vagrantfile new file mode 100644 index 00000000000..c01ae5fa6e2 --- /dev/null +++ b/ci/vagrant-freebsd/Vagrantfile @@ -0,0 +1,3 @@ +Vagrant.configure("2") do |config| + config.vm.box = "generic/freebsd11" +end diff --git a/cmake/Modules/FindICU.cmake b/cmake/Modules/FindICU.cmake new file mode 100644 index 00000000000..0e61b3dcf29 --- /dev/null +++ b/cmake/Modules/FindICU.cmake @@ -0,0 +1,394 @@ +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + +#.rst: +# FindICU +# ------- +# +# Find the International Components for Unicode (ICU) libraries and +# programs. +# +# This module supports multiple components. +# Components can include any of: ``data``, ``i18n``, ``io``, ``le``, +# ``lx``, ``test``, ``tu`` and ``uc``. +# +# Note that on Windows ``data`` is named ``dt`` and ``i18n`` is named +# ``in``; any of the names may be used, and the appropriate +# platform-specific library name will be automatically selected. +# +# This module reports information about the ICU installation in +# several variables. General variables:: +# +# ICU_VERSION - ICU release version +# ICU_FOUND - true if the main programs and libraries were found +# ICU_LIBRARIES - component libraries to be linked +# ICU_INCLUDE_DIRS - the directories containing the ICU headers +# +# Imported targets:: +# +# ICU:: +# +# Where ```` is the name of an ICU component, for example +# ``ICU::i18n``. +# +# ICU programs are reported in:: +# +# ICU_GENCNVAL_EXECUTABLE - path to gencnval executable +# ICU_ICUINFO_EXECUTABLE - path to icuinfo executable +# ICU_GENBRK_EXECUTABLE - path to genbrk executable +# ICU_ICU-CONFIG_EXECUTABLE - path to icu-config executable +# ICU_GENRB_EXECUTABLE - path to genrb executable +# ICU_GENDICT_EXECUTABLE - path to gendict executable +# ICU_DERB_EXECUTABLE - path to derb executable +# ICU_PKGDATA_EXECUTABLE - path to pkgdata executable +# ICU_UCONV_EXECUTABLE - path to uconv executable +# ICU_GENCFU_EXECUTABLE - path to gencfu executable +# ICU_MAKECONV_EXECUTABLE - path to makeconv executable +# ICU_GENNORM2_EXECUTABLE - path to gennorm2 executable +# ICU_GENCCODE_EXECUTABLE - path to genccode executable +# ICU_GENSPREP_EXECUTABLE - path to gensprep executable +# ICU_ICUPKG_EXECUTABLE - path to icupkg executable +# ICU_GENCMN_EXECUTABLE - path to gencmn executable +# +# ICU component libraries are reported in:: +# +# ICU__FOUND - ON if component was found +# ICU__LIBRARIES - libraries for component +# +# ICU datafiles are reported in:: +# +# ICU_MAKEFILE_INC - Makefile.inc +# ICU_PKGDATA_INC - pkgdata.inc +# +# Note that ```` is the uppercased name of the component. +# +# This module reads hints about search results from:: +# +# ICU_ROOT - the root of the ICU installation +# +# The environment variable ``ICU_ROOT`` may also be used; the +# ICU_ROOT variable takes precedence. +# +# The following cache variables may also be set:: +# +# ICU_

_EXECUTABLE - the path to executable

+# ICU_INCLUDE_DIR - the directory containing the ICU headers +# ICU__LIBRARY - the library for component +# +# .. note:: +# +# In most cases none of the above variables will require setting, +# unless multiple ICU versions are available and a specific version +# is required. +# +# Other variables one may set to control this module are:: +# +# ICU_DEBUG - Set to ON to enable debug output from FindICU. + +# Written by Roger Leigh + +set(icu_programs + gencnval + icuinfo + genbrk + icu-config + genrb + gendict + derb + pkgdata + uconv + gencfu + makeconv + gennorm2 + genccode + gensprep + icupkg + gencmn) + +set(icu_data + Makefile.inc + pkgdata.inc) + +# The ICU checks are contained in a function due to the large number +# of temporary variables needed. +function(_ICU_FIND) + # Set up search paths, taking compiler into account. Search ICU_ROOT, + # with ICU_ROOT in the environment as a fallback if unset. + if(ICU_ROOT) + list(APPEND icu_roots "${ICU_ROOT}") + else() + if(NOT "$ENV{ICU_ROOT}" STREQUAL "") + file(TO_CMAKE_PATH "$ENV{ICU_ROOT}" NATIVE_PATH) + list(APPEND icu_roots "${NATIVE_PATH}") + set(ICU_ROOT "${NATIVE_PATH}" + CACHE PATH "Location of the ICU installation" FORCE) + endif() + endif() + + # Find include directory + list(APPEND icu_include_suffixes "include") + find_path(ICU_INCLUDE_DIR + NAMES "unicode/utypes.h" + HINTS ${icu_roots} + PATH_SUFFIXES ${icu_include_suffixes} + DOC "ICU include directory") + set(ICU_INCLUDE_DIR "${ICU_INCLUDE_DIR}" PARENT_SCOPE) + + # Get version + if(ICU_INCLUDE_DIR AND EXISTS "${ICU_INCLUDE_DIR}/unicode/uvernum.h") + file(STRINGS "${ICU_INCLUDE_DIR}/unicode/uvernum.h" icu_header_str + REGEX "^#define[\t ]+U_ICU_VERSION[\t ]+\".*\".*") + + string(REGEX REPLACE "^#define[\t ]+U_ICU_VERSION[\t ]+\"([^ \\n]*)\".*" + "\\1" icu_version_string "${icu_header_str}") + set(ICU_VERSION "${icu_version_string}") + set(ICU_VERSION "${icu_version_string}" PARENT_SCOPE) + unset(icu_header_str) + unset(icu_version_string) + endif() + + if(CMAKE_SIZEOF_VOID_P EQUAL 8) + # 64-bit binary directory + set(_bin64 "bin64") + # 64-bit library directory + set(_lib64 "lib64") + endif() + + + # Find all ICU programs + list(APPEND icu_binary_suffixes "${_bin64}" "bin") + foreach(program ${icu_programs}) + string(TOUPPER "${program}" program_upcase) + set(cache_var "ICU_${program_upcase}_EXECUTABLE") + set(program_var "ICU_${program_upcase}_EXECUTABLE") + find_program("${cache_var}" "${program}" + HINTS ${icu_roots} + PATH_SUFFIXES ${icu_binary_suffixes} + DOC "ICU ${program} executable") + mark_as_advanced(cache_var) + set("${program_var}" "${${cache_var}}" PARENT_SCOPE) + endforeach() + + # Find all ICU libraries + list(APPEND icu_library_suffixes "${_lib64}" "lib") + set(ICU_REQUIRED_LIBS_FOUND ON) + foreach(component ${ICU_FIND_COMPONENTS}) + string(TOUPPER "${component}" component_upcase) + set(component_cache "ICU_${component_upcase}_LIBRARY") + set(component_cache_release "${component_cache}_RELEASE") + set(component_cache_debug "${component_cache}_DEBUG") + set(component_found "${component_upcase}_FOUND") + set(component_libnames "icu${component}") + set(component_debug_libnames "icu${component}d") + + # Special case deliberate library naming mismatches between Unix + # and Windows builds + unset(component_libnames) + unset(component_debug_libnames) + list(APPEND component_libnames "icu${component}") + list(APPEND component_debug_libnames "icu${component}d") + if(component STREQUAL "data") + list(APPEND component_libnames "icudt") + # Note there is no debug variant at present + list(APPEND component_debug_libnames "icudtd") + endif() + if(component STREQUAL "dt") + list(APPEND component_libnames "icudata") + # Note there is no debug variant at present + list(APPEND component_debug_libnames "icudatad") + endif() + if(component STREQUAL "i18n") + list(APPEND component_libnames "icuin") + list(APPEND component_debug_libnames "icuind") + endif() + if(component STREQUAL "in") + list(APPEND component_libnames "icui18n") + list(APPEND component_debug_libnames "icui18nd") + endif() + + find_library("${component_cache_release}" ${component_libnames} + HINTS ${icu_roots} + PATH_SUFFIXES ${icu_library_suffixes} + DOC "ICU ${component} library (release)") + find_library("${component_cache_debug}" ${component_debug_libnames} + HINTS ${icu_roots} + PATH_SUFFIXES ${icu_library_suffixes} + DOC "ICU ${component} library (debug)") + include(SelectLibraryConfigurations) + select_library_configurations(ICU_${component_upcase}) + mark_as_advanced("${component_cache_release}" "${component_cache_debug}") + if(${component_cache}) + set("${component_found}" ON) + list(APPEND ICU_LIBRARY "${${component_cache}}") + endif() + mark_as_advanced("${component_found}") + set("${component_cache}" "${${component_cache}}" PARENT_SCOPE) + set("${component_found}" "${${component_found}}" PARENT_SCOPE) + if(${component_found}) + if (ICU_FIND_REQUIRED_${component}) + list(APPEND ICU_LIBS_FOUND "${component} (required)") + else() + list(APPEND ICU_LIBS_FOUND "${component} (optional)") + endif() + else() + if (ICU_FIND_REQUIRED_${component}) + set(ICU_REQUIRED_LIBS_FOUND OFF) + list(APPEND ICU_LIBS_NOTFOUND "${component} (required)") + else() + list(APPEND ICU_LIBS_NOTFOUND "${component} (optional)") + endif() + endif() + endforeach() + set(_ICU_REQUIRED_LIBS_FOUND "${ICU_REQUIRED_LIBS_FOUND}" PARENT_SCOPE) + set(ICU_LIBRARY "${ICU_LIBRARY}" PARENT_SCOPE) + + # Find all ICU data files + if(CMAKE_LIBRARY_ARCHITECTURE) + list(APPEND icu_data_suffixes + "${_lib64}/${CMAKE_LIBRARY_ARCHITECTURE}/icu/${ICU_VERSION}" + "lib/${CMAKE_LIBRARY_ARCHITECTURE}/icu/${ICU_VERSION}" + "${_lib64}/${CMAKE_LIBRARY_ARCHITECTURE}/icu" + "lib/${CMAKE_LIBRARY_ARCHITECTURE}/icu") + endif() + list(APPEND icu_data_suffixes + "${_lib64}/icu/${ICU_VERSION}" + "lib/icu/${ICU_VERSION}" + "${_lib64}/icu" + "lib/icu") + foreach(data ${icu_data}) + string(TOUPPER "${data}" data_upcase) + string(REPLACE "." "_" data_upcase "${data_upcase}") + set(cache_var "ICU_${data_upcase}") + set(data_var "ICU_${data_upcase}") + find_file("${cache_var}" "${data}" + HINTS ${icu_roots} + PATH_SUFFIXES ${icu_data_suffixes} + DOC "ICU ${data} data file") + mark_as_advanced(cache_var) + set("${data_var}" "${${cache_var}}" PARENT_SCOPE) + endforeach() + + if(NOT ICU_FIND_QUIETLY) + if(ICU_LIBS_FOUND) + message(STATUS "Found the following ICU libraries:") + foreach(found ${ICU_LIBS_FOUND}) + message(STATUS " ${found}") + endforeach() + endif() + if(ICU_LIBS_NOTFOUND) + message(STATUS "The following ICU libraries were not found:") + foreach(notfound ${ICU_LIBS_NOTFOUND}) + message(STATUS " ${notfound}") + endforeach() + endif() + endif() + + if(ICU_DEBUG) + message(STATUS "--------FindICU.cmake search debug--------") + message(STATUS "ICU binary path search order: ${icu_roots}") + message(STATUS "ICU include path search order: ${icu_roots}") + message(STATUS "ICU library path search order: ${icu_roots}") + message(STATUS "----------------") + endif() +endfunction() + +_ICU_FIND() + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(ICU + FOUND_VAR ICU_FOUND + REQUIRED_VARS ICU_INCLUDE_DIR + ICU_LIBRARY + _ICU_REQUIRED_LIBS_FOUND + VERSION_VAR ICU_VERSION + FAIL_MESSAGE "Failed to find all ICU components") + +unset(_ICU_REQUIRED_LIBS_FOUND) + +if(ICU_FOUND) + set(ICU_INCLUDE_DIRS "${ICU_INCLUDE_DIR}") + set(ICU_LIBRARIES "${ICU_LIBRARY}") + foreach(_ICU_component ${ICU_FIND_COMPONENTS}) + string(TOUPPER "${_ICU_component}" _ICU_component_upcase) + set(_ICU_component_cache "ICU_${_ICU_component_upcase}_LIBRARY") + set(_ICU_component_cache_release "ICU_${_ICU_component_upcase}_LIBRARY_RELEASE") + set(_ICU_component_cache_debug "ICU_${_ICU_component_upcase}_LIBRARY_DEBUG") + set(_ICU_component_lib "ICU_${_ICU_component_upcase}_LIBRARIES") + set(_ICU_component_found "${_ICU_component_upcase}_FOUND") + set(_ICU_imported_target "ICU::${_ICU_component}") + if(${_ICU_component_found}) + set("${_ICU_component_lib}" "${${_ICU_component_cache}}") + if(NOT TARGET ${_ICU_imported_target}) + add_library(${_ICU_imported_target} UNKNOWN IMPORTED) + if(ICU_INCLUDE_DIR) + set_target_properties(${_ICU_imported_target} PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${ICU_INCLUDE_DIR}") + endif() + if(EXISTS "${${_ICU_component_cache}}") + set_target_properties(${_ICU_imported_target} PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "CXX" + IMPORTED_LOCATION "${${_ICU_component_cache}}") + endif() + if(EXISTS "${${_ICU_component_cache_release}}") + set_property(TARGET ${_ICU_imported_target} APPEND PROPERTY + IMPORTED_CONFIGURATIONS RELEASE) + set_target_properties(${_ICU_imported_target} PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "CXX" + IMPORTED_LOCATION_RELEASE "${${_ICU_component_cache_release}}") + endif() + if(EXISTS "${${_ICU_component_cache_debug}}") + set_property(TARGET ${_ICU_imported_target} APPEND PROPERTY + IMPORTED_CONFIGURATIONS DEBUG) + set_target_properties(${_ICU_imported_target} PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES_DEBUG "CXX" + IMPORTED_LOCATION_DEBUG "${${_ICU_component_cache_debug}}") + endif() + endif() + endif() + unset(_ICU_component_upcase) + unset(_ICU_component_cache) + unset(_ICU_component_lib) + unset(_ICU_component_found) + unset(_ICU_imported_target) + endforeach() +endif() + +if(ICU_DEBUG) + message(STATUS "--------FindICU.cmake results debug--------") + message(STATUS "ICU found: ${ICU_FOUND}") + message(STATUS "ICU_VERSION number: ${ICU_VERSION}") + message(STATUS "ICU_ROOT directory: ${ICU_ROOT}") + message(STATUS "ICU_INCLUDE_DIR directory: ${ICU_INCLUDE_DIR}") + message(STATUS "ICU_LIBRARIES: ${ICU_LIBRARIES}") + + foreach(program IN LISTS icu_programs) + string(TOUPPER "${program}" program_upcase) + set(program_lib "ICU_${program_upcase}_EXECUTABLE") + message(STATUS "${program} program: ${${program_lib}}") + unset(program_upcase) + unset(program_lib) + endforeach() + + foreach(data IN LISTS icu_data) + string(TOUPPER "${data}" data_upcase) + string(REPLACE "." "_" data_upcase "${data_upcase}") + set(data_lib "ICU_${data_upcase}") + message(STATUS "${data} data: ${${data_lib}}") + unset(data_upcase) + unset(data_lib) + endforeach() + + foreach(component IN LISTS ICU_FIND_COMPONENTS) + string(TOUPPER "${component}" component_upcase) + set(component_lib "ICU_${component_upcase}_LIBRARIES") + set(component_found "${component_upcase}_FOUND") + message(STATUS "${component} library found: ${${component_found}}") + message(STATUS "${component} library: ${${component_lib}}") + unset(component_upcase) + unset(component_lib) + unset(component_found) + endforeach() + message(STATUS "----------------") +endif() + +unset(icu_programs) diff --git a/cmake/Modules/FindODBC.cmake b/cmake/Modules/FindODBC.cmake new file mode 100644 index 00000000000..66d43e93d2d --- /dev/null +++ b/cmake/Modules/FindODBC.cmake @@ -0,0 +1,88 @@ +# This file copied from contrib/poco/cmake/FindODBC.cmake to allow build without submodules + +# +# Find the ODBC driver manager includes and library. +# +# ODBC is an open standard for connecting to different databases in a +# semi-vendor-independent fashion. First you install the ODBC driver +# manager. Then you need a driver for each separate database you want +# to connect to (unless a generic one works). VTK includes neither +# the driver manager nor the vendor-specific drivers: you have to find +# those yourself. +# +# This module defines +# ODBC_INCLUDE_DIRECTORIES, where to find sql.h +# ODBC_LIBRARIES, the libraries to link against to use ODBC +# ODBC_FOUND. If false, you cannot build anything that requires ODBC. + +option (ENABLE_ODBC "Enable ODBC" ${OS_LINUX}) +if (OS_LINUX) + option (USE_INTERNAL_ODBC_LIBRARY "Set to FALSE to use system odbc library instead of bundled" ${NOT_UNBUNDLED}) +else () + option (USE_INTERNAL_ODBC_LIBRARY "Set to FALSE to use system odbc library instead of bundled" OFF) +endif () + +if (USE_INTERNAL_ODBC_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/unixodbc/README") + message (WARNING "submodule contrib/unixodbc is missing. to fix try run: \n git submodule update --init --recursive") + set (USE_INTERNAL_ODBC_LIBRARY 0) +endif () + +if (ENABLE_ODBC) + if (USE_INTERNAL_ODBC_LIBRARY) + set (ODBC_LIBRARIES unixodbc) + set (ODBC_INCLUDE_DIRECTORIES ${CMAKE_SOURCE_DIR}/contrib/unixodbc/include) + set (ODBC_FOUND 1) + set (USE_ODBC 1) + else () + find_path(ODBC_INCLUDE_DIRECTORIES + NAMES sql.h + HINTS + /usr/include + /usr/include/iodbc + /usr/include/odbc + /usr/local/include + /usr/local/include/iodbc + /usr/local/include/odbc + /usr/local/iodbc/include + /usr/local/odbc/include + "C:/Program Files/ODBC/include" + "C:/Program Files/Microsoft SDKs/Windows/v7.0/include" + "C:/Program Files/Microsoft SDKs/Windows/v6.0a/include" + "C:/ODBC/include" + DOC "Specify the directory containing sql.h." + ) + + find_library(ODBC_LIBRARIES + NAMES iodbc odbc iodbcinst odbcinst odbc32 + HINTS + /usr/lib + /usr/lib/iodbc + /usr/lib/odbc + /usr/local/lib + /usr/local/lib/iodbc + /usr/local/lib/odbc + /usr/local/iodbc/lib + /usr/local/odbc/lib + "C:/Program Files/ODBC/lib" + "C:/ODBC/lib/debug" + "C:/Program Files (x86)/Microsoft SDKs/Windows/v7.0A/Lib" + DOC "Specify the ODBC driver manager library here." + ) + + # MinGW find usually fails + if(MINGW) + set(ODBC_INCLUDE_DIRECTORIES ".") + set(ODBC_LIBRARIES odbc32) + endif() + + include(FindPackageHandleStandardArgs) + find_package_handle_standard_args(ODBC + DEFAULT_MSG + ODBC_INCLUDE_DIRECTORIES + ODBC_LIBRARIES) + + mark_as_advanced(ODBC_FOUND ODBC_LIBRARIES ODBC_INCLUDE_DIRECTORIES) + endif () +endif () + +message (STATUS "Using odbc: ${ODBC_INCLUDE_DIRECTORIES} : ${ODBC_LIBRARIES}") diff --git a/cmake/Modules/FindPoco.cmake b/cmake/Modules/FindPoco.cmake index 33f0bd82ba0..e56e2f70fb5 100644 --- a/cmake/Modules/FindPoco.cmake +++ b/cmake/Modules/FindPoco.cmake @@ -51,6 +51,7 @@ set(Poco_HINTS /usr/local + /usr/local/include/Poco C:/AppliedInformatics ${Poco_DIR} $ENV{Poco_DIR} @@ -116,28 +117,35 @@ foreach( component ${components} ) # include directory for the component if(NOT Poco_${component}_INCLUDE_DIR) + set (component_alt "${component}") + set (component_root "${component}") if (${component} STREQUAL "DataODBC") - set (component_in_data "ODBC") - else () - set (component_in_data "") + set (component_top "Data") + set (component_in "ODBC") + set (component_root "Data/ODBC") + endif () + if (${component} STREQUAL "SQLODBC") + set (component_top "SQL") + set (component_in "ODBC") + set (component_root "SQL/ODBC") endif () if (${component} STREQUAL "NetSSL") set (component_alt "Net") - else () - set (component_alt ${component}) + set (component_root "NetSSL_OpenSSL") endif () find_path(Poco_${component}_INCLUDE_DIR NAMES Poco/${component}.h # e.g. Foundation.h Poco/${component}/${component}.h # e.g. OSP/OSP.h Util/Util.h Poco/${component_alt}/${component}.h # e.g. Net/NetSSL.h - Poco/Data/${component_in_data}/${component_in_data}.h # e.g. Data/ODBC/ODBC.h + Poco/${component_top}/${component_in}/${component_in}.h # e.g. Data/ODBC/ODBC.h HINTS ${Poco_ROOT_DIR} PATH_SUFFIXES include - ${component}/include + ${component_root}/include ) + # message(STATUS "poco include debug: {component}: ${Poco_${component}_INCLUDE_DIR}") endif() if(NOT Poco_${component}_INCLUDE_DIR) message(WARNING "Poco_${component}_INCLUDE_DIR NOT FOUND") @@ -189,6 +197,8 @@ foreach( component ${components} ) elseif(NOT Poco_FIND_QUIETLY) message(WARNING "Could not find Poco component ${component}!") endif() + + # message(STATUS "Poco component ${component}: Poco_${component}_LIBRARY : Poco_${component}_INCLUDE_DIR") endforeach() if(Poco_DataODBC_LIBRARY) @@ -196,6 +206,11 @@ if(Poco_DataODBC_LIBRARY) list(APPEND Poco_INCLUDE_DIRS ${ODBC_INCLUDE_DIRECTORIES}) endif() +if(Poco_SQLODBC_LIBRARY) + list(APPEND Poco_SQLODBC_LIBRARY ${ODBC_LIBRARIES} ${LTDL_LIBRARY}) + list(APPEND Poco_INCLUDE_DIRS ${ODBC_INCLUDE_DIRECTORIES}) +endif() + if(Poco_NetSSL_LIBRARY) list(APPEND Poco_NetSSL_LIBRARY ${OPENSSL_LIBRARIES}) list(APPEND Poco_INCLUDE_DIRS ${OPENSSL_INCLUDE_DIR}) @@ -230,5 +245,3 @@ if(${Poco_OSP_FOUND}) endif() message(STATUS "Found Poco: ${Poco_LIBRARIES}") - - diff --git a/cmake/arch.cmake b/cmake/arch.cmake index f61bac96ab0..abc30d99e32 100644 --- a/cmake/arch.cmake +++ b/cmake/arch.cmake @@ -7,19 +7,16 @@ endif () if (CMAKE_LIBRARY_ARCHITECTURE MATCHES "i386") set (ARCH_I386 1) endif () -if ( ( ARCH_ARM AND NOT ARCH_AARCH64 ) OR ARCH_I386) +if ((ARCH_ARM AND NOT ARCH_AARCH64) OR ARCH_I386) set (ARCH_32 1) - message (WARNING "Support for 32bit platforms is highly experimental") -endif () -if (CMAKE_SYSTEM MATCHES "Linux") - set (ARCH_LINUX 1) -endif () -if (CMAKE_SYSTEM MATCHES "FreeBSD") - set (ARCH_FREEBSD 1) + message (FATAL_ERROR "32bit platforms are not supported") endif () -if (NOT MSVC) - set (NOT_MSVC 1) +if (CMAKE_SYSTEM MATCHES "Linux") + set (OS_LINUX 1) +endif () +if (CMAKE_SYSTEM MATCHES "FreeBSD") + set (OS_FREEBSD 1) endif () if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") diff --git a/cmake/dbms_generate_function.cmake b/cmake/dbms_generate_function.cmake deleted file mode 100644 index 0ed4d6002a5..00000000000 --- a/cmake/dbms_generate_function.cmake +++ /dev/null @@ -1,13 +0,0 @@ - -function(generate_function_register FUNCTION_AREA) - - foreach(FUNCTION IN LISTS ARGN) - configure_file (registerFunction.h.in register${FUNCTION}.h) - configure_file (registerFunction.cpp.in register${FUNCTION}.cpp) - set(REGISTER_HEADERS "${REGISTER_HEADERS} #include \"register${FUNCTION}.h\"\n") - set(REGISTER_FUNCTIONS "${REGISTER_FUNCTIONS} register${FUNCTION}(factory);\n") - endforeach() - - configure_file (registerFunctions_area.cpp.in registerFunctions${FUNCTION_AREA}.cpp) - -endfunction() diff --git a/cmake/find_base64.cmake b/cmake/find_base64.cmake new file mode 100644 index 00000000000..ad71d11c1b4 --- /dev/null +++ b/cmake/find_base64.cmake @@ -0,0 +1,12 @@ +option (ENABLE_BASE64 "Enable base64" ON) + +if (ENABLE_BASE64) + if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64") + message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive") + else() + set (BASE64_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64/include) + set (BASE64_LIBRARY base64) + set (USE_BASE64 1) + endif() +endif () + diff --git a/cmake/find_boost.cmake b/cmake/find_boost.cmake index a99211efe08..5a024d4d3db 100644 --- a/cmake/find_boost.cmake +++ b/cmake/find_boost.cmake @@ -26,9 +26,9 @@ endif () if (NOT Boost_SYSTEM_LIBRARY) set (USE_INTERNAL_BOOST_LIBRARY 1) - set (Boost_PROGRAM_OPTIONS_LIBRARY boost_program_options_internal) set (Boost_SYSTEM_LIBRARY boost_system_internal) - set (Boost_FILESYSTEM_LIBRARY boost_filesystem_internal) + set (Boost_PROGRAM_OPTIONS_LIBRARY boost_program_options_internal) + set (Boost_FILESYSTEM_LIBRARY boost_filesystem_internal ${Boost_SYSTEM_LIBRARY}) set (Boost_INCLUDE_DIRS) diff --git a/cmake/find_capnp.cmake b/cmake/find_capnp.cmake index 22301acc3ee..ec591afdc38 100644 --- a/cmake/find_capnp.cmake +++ b/cmake/find_capnp.cmake @@ -1,7 +1,6 @@ -option (ENABLE_CAPNP "Enable Cap'n Proto" ${NOT_MSVC}) +option (ENABLE_CAPNP "Enable Cap'n Proto" ON) if (ENABLE_CAPNP) - # cmake 3.5.1 bug: # capnproto uses this cmake feature: # target_compile_features(kj PUBLIC cxx_constexpr) @@ -30,7 +29,7 @@ if (ENABLE_CAPNP) find_library (CAPNP capnp PATHS ${CAPNP_PATHS}) find_library (CAPNPC capnpc PATHS ${CAPNP_PATHS}) find_library (KJ kj PATHS ${CAPNP_PATHS}) - set (CAPNP_LIBRARY ${CAPNP} ${CAPNPC} ${KJ}) + set (CAPNP_LIBRARY ${CAPNPC} ${CAPNP} ${KJ}) find_path (CAPNP_INCLUDE_DIR NAMES capnp/schema-parser.h PATHS ${CAPNP_INCLUDE_PATHS}) endif () @@ -42,7 +41,6 @@ if (ENABLE_CAPNP) set (CAPNP_LIBRARY capnpc) set (USE_CAPNP 1) endif () - endif () if (USE_CAPNP) diff --git a/cmake/find_consistent-hashing.cmake b/cmake/find_consistent-hashing.cmake new file mode 100644 index 00000000000..23e709d2228 --- /dev/null +++ b/cmake/find_consistent-hashing.cmake @@ -0,0 +1,14 @@ +option (USE_INTERNAL_CONSISTENT_HASHING_LIBRARY "Set to FALSE to use consistent-hashing library from Arcadia (Yandex internal repository) instead of bundled" ${NOT_UNBUNDLED}) + +if (NOT USE_INTERNAL_CONSISTENT_HASHING_LIBRARY) + find_library (CONSISTENT_HASHING_LIBRARY consistent-hashing) + find_path (CONSISTENT_HASHING_INCLUDE_DIR NAMES consistent_hashing.h PATHS ${CONSISTENT_HASHING_INCLUDE_PATHS}) +endif () + +if (CONSISTENT_HASHING_LIBRARY AND CONSISTENT_HASHING_INCLUDE_DIR) +else () + set (USE_INTERNAL_CONSISTENT_HASHING_LIBRARY 1) + set (CONSISTENT_HASHING_LIBRARY consistent-hashing) +endif () + +message (STATUS "Using consistent-hashing: ${CONSISTENT_HASHING_INCLUDE_DIR} : ${CONSISTENT_HASHING_LIBRARY}") diff --git a/cmake/find_cpuid.cmake b/cmake/find_cpuid.cmake new file mode 100644 index 00000000000..cda88433a1c --- /dev/null +++ b/cmake/find_cpuid.cmake @@ -0,0 +1,29 @@ +if (NOT ARCH_ARM) + option (USE_INTERNAL_CPUID_LIBRARY "Set to FALSE to use system cpuid library instead of bundled" ${NOT_UNBUNDLED}) +endif () + +#if (USE_INTERNAL_CPUID_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcpuid/include/cpuid/libcpuid.h") +# message (WARNING "submodule contrib/libcpuid is missing. to fix try run: \n git submodule update --init --recursive") +# set (USE_INTERNAL_CPUID_LIBRARY 0) +# set (MISSING_INTERNAL_CPUID_LIBRARY 1) +#endif () + +if (NOT USE_INTERNAL_CPUID_LIBRARY) + find_library (CPUID_LIBRARY cpuid) + find_path (CPUID_INCLUDE_DIR NAMES libcpuid/libcpuid.h PATHS ${CPUID_INCLUDE_PATHS}) +endif () + +if (CPUID_LIBRARY AND CPUID_INCLUDE_DIR) + if (OS_FREEBSD) + # need in /usr/local/include/libcpuid/libcpuid_types.h + # Freebsd: /usr/local/include/libcpuid/libcpuid_types.h:61:29: error: conflicting declaration 'typedef long long int int64_t' + add_definitions(-DHAVE_STDINT_H) + # TODO: make virtual target cpuid:cpuid with COMPILE_DEFINITIONS property + endif () +elseif (NOT MISSING_INTERNAL_CPUID_LIBRARY) + set (CPUID_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcpuid/include) + set (USE_INTERNAL_CPUID_LIBRARY 1) + set (CPUID_LIBRARY cpuid) +endif () + +message (STATUS "Using cpuid: ${CPUID_INCLUDE_DIR} : ${CPUID_LIBRARY}") diff --git a/cmake/find_execinfo.cmake b/cmake/find_execinfo.cmake index 6d7428a166f..650d279983c 100644 --- a/cmake/find_execinfo.cmake +++ b/cmake/find_execinfo.cmake @@ -1,6 +1,9 @@ -if (ARCH_FREEBSD) +if (OS_FREEBSD) find_library (EXECINFO_LIBRARY execinfo) + find_library (ELF_LIBRARY elf) message (STATUS "Using execinfo: ${EXECINFO_LIBRARY}") + message (STATUS "Using elf: ${ELF_LIBRARY}") else () set (EXECINFO_LIBRARY "") + set (ELF_LIBRARY "") endif () diff --git a/cmake/find_gtest.cmake b/cmake/find_gtest.cmake index 5a861e7ec24..ce0a3d32785 100644 --- a/cmake/find_gtest.cmake +++ b/cmake/find_gtest.cmake @@ -9,20 +9,12 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest/CMakeList endif () if (NOT USE_INTERNAL_GTEST_LIBRARY) - find_path (GTEST_INCLUDE_DIR NAMES /gtest/gtest.h PATHS ${GTEST_INCLUDE_PATHS}) - find_path (GTEST_ROOT NAMES src/gtest-all.cc PATHS /usr/src/googletest/googletest /usr/src/gtest) + find_package (GTest) endif () -if (GTEST_INCLUDE_DIR AND GTEST_ROOT) - # googletest package have no lib - add_library(gtest ${GTEST_ROOT}/src/gtest-all.cc) - add_library(gtest_main ${GTEST_ROOT}/src/gtest_main.cc) - target_include_directories(gtest PRIVATE ${GTEST_ROOT}) - target_link_libraries(gtest_main gtest) - set (GTEST_LIBRARY gtest_main) -elseif (NOT MISSING_INTERNAL_GTEST_LIBRARY) +if (NOT GTEST_INCLUDE_DIRS AND NOT MISSING_INTERNAL_GTEST_LIBRARY) set (USE_INTERNAL_GTEST_LIBRARY 1) - set (GTEST_LIBRARY gtest_main) + set (GTEST_MAIN_LIBRARIES gtest_main) endif () -message (STATUS "Using gtest: ${GTEST_INCLUDE_DIR} : ${GTEST_LIBRARY}") +message (STATUS "Using gtest: ${GTEST_INCLUDE_DIRS} : ${GTEST_MAIN_LIBRARIES}") diff --git a/cmake/find_icu.cmake b/cmake/find_icu.cmake new file mode 100644 index 00000000000..e677b11b037 --- /dev/null +++ b/cmake/find_icu.cmake @@ -0,0 +1,16 @@ +option (ENABLE_ICU "Enable ICU" ON) + +if (ENABLE_ICU) + find_package(ICU COMPONENTS data i18n uc) # TODO: remove Modules/FindICU.cmake after cmake 3.7 + #set (ICU_LIBRARIES ${ICU_I18N_LIBRARY} ${ICU_UC_LIBRARY} ${ICU_DATA_LIBRARY} CACHE STRING "") + set (ICU_LIBRARIES ICU::i18n ICU::uc ICU::data CACHE STRING "") + if (ICU_FOUND) + set(USE_ICU 1) + endif () +endif () + +if (USE_ICU) + message (STATUS "Using icu=${USE_ICU}: ${ICU_INCLUDE_DIR} : ${ICU_LIBRARIES}") +else () + message (STATUS "Build without ICU (support for collations and charset conversion functions will be disabled)") +endif () diff --git a/cmake/find_icu4c.cmake b/cmake/find_icu4c.cmake deleted file mode 100644 index e39f1c4c1dd..00000000000 --- a/cmake/find_icu4c.cmake +++ /dev/null @@ -1,21 +0,0 @@ -option (ENABLE_ICU "Enable ICU" ON) - -if (ENABLE_ICU) - set (ICU_PATHS "/usr/local/opt/icu4c/lib") - set (ICU_INCLUDE_PATHS "/usr/local/opt/icu4c/include") - find_library (ICUI18N icui18n PATHS ${ICU_PATHS}) - find_library (ICUUC icuuc PATHS ${ICU_PATHS}) - find_library (ICUDATA icudata PATHS ${ICU_PATHS}) - set (ICU_LIBS ${ICUI18N} ${ICUUC} ${ICUDATA}) - - find_path (ICU_INCLUDE_DIR NAMES unicode/unistr.h PATHS ${ICU_INCLUDE_PATHS}) - if (ICU_INCLUDE_DIR AND ICU_LIBS) - set(USE_ICU 1) - endif () -endif () - -if (USE_ICU) - message (STATUS "Using icu=${USE_ICU}: ${ICU_INCLUDE_DIR} : ${ICU_LIBS}") -else () - message (STATUS "Build without ICU (support for collations and charset conversion functions will be disabled)") -endif () diff --git a/cmake/find_llvm.cmake b/cmake/find_llvm.cmake index dafb533a0f9..81866b720c9 100644 --- a/cmake/find_llvm.cmake +++ b/cmake/find_llvm.cmake @@ -1,107 +1,88 @@ -option (ENABLE_EMBEDDED_COMPILER "Set to TRUE to enable support for 'compile' option for query execution" FALSE) +# Broken in macos. TODO: update clang, re-test, enable +if (NOT APPLE) + option (ENABLE_EMBEDDED_COMPILER "Set to TRUE to enable support for 'compile' option for query execution" 1) + option (USE_INTERNAL_LLVM_LIBRARY "Use bundled or system LLVM library. Default: system library for quicker developer builds." ${APPLE}) +endif () if (ENABLE_EMBEDDED_COMPILER) - # Based on source code of YT. - # Authors: Ivan Puzyrevskiy, Alexey Lukyanchikov, Ruslan Savchenko. + if (USE_INTERNAL_LLVM_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm/CMakeLists.txt") + message (WARNING "submodule contrib/llvm is missing. to fix try run: \n git submodule update --init --recursive") + set (USE_INTERNAL_LLVM_LIBRARY 0) + endif () - # Find LLVM includes and libraries. - # - # LLVM_VERSION - LLVM version. - # LLVM_INCLUDE_DIRS - Directory containing LLVM headers. - # LLVM_LIBRARY_DIRS - Directory containing LLVM libraries. - # LLVM_CXXFLAGS - C++ compiler flags for files that include LLVM headers. - # LLVM_FOUND - True if LLVM was found. + if (NOT USE_INTERNAL_LLVM_LIBRARY) + set (LLVM_PATHS "/usr/local/lib/llvm") - # llvm_map_components_to_libraries - Maps LLVM used components to required libraries. - # Usage: llvm_map_components_to_libraries(REQUIRED_LLVM_LIBRARIES core jit interpreter native ...) + if (LLVM_VERSION) + find_package(LLVM ${LLVM_VERSION} CONFIG PATHS ${LLVM_PATHS}) + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + find_package(LLVM ${CMAKE_CXX_COMPILER_VERSION} CONFIG PATHS ${LLVM_PATHS}) + else () + find_package (LLVM 7 CONFIG PATHS ${LLVM_PATHS}) + if (NOT LLVM_FOUND) + find_package (LLVM 6 CONFIG PATHS ${LLVM_PATHS}) + endif () + if (NOT LLVM_FOUND) + find_package (LLVM 5 CONFIG PATHS ${LLVM_PATHS}) + endif () + endif () - if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - set(LLVM_VERSION_POSTFIX "${COMPILER_POSTFIX}" CACHE STRING "") - else() - if (ARCH_FREEBSD) - set(LLVM_VERSION_POSTFIX "50" CACHE STRING "") + if (LLVM_FOUND) + find_library (LLD_LIBRARY_TEST lldCore PATHS ${LLVM_LIBRARY_DIRS}) + find_path (LLD_INCLUDE_DIR_TEST NAMES lld/Core/AbsoluteAtom.h PATHS ${LLVM_INCLUDE_DIRS}) + if (NOT LLD_LIBRARY_TEST OR NOT LLD_INCLUDE_DIR_TEST) + set (LLVM_FOUND 0) + message(WARNING "liblld (${LLD_LIBRARY_TEST}, ${LLD_INCLUDE_DIR_TEST}) not found in ${LLVM_INCLUDE_DIRS} ${LLVM_LIBRARY_DIRS}. Disabling internal compiler.") + endif () + endif () + + if (LLVM_FOUND) + # Remove dynamically-linked zlib and libedit from LLVM's dependencies: + set_target_properties(LLVMSupport PROPERTIES INTERFACE_LINK_LIBRARIES "-lpthread;LLVMDemangle;${ZLIB_LIBRARIES}") + set_target_properties(LLVMLineEditor PROPERTIES INTERFACE_LINK_LIBRARIES "LLVMSupport") + + option(LLVM_HAS_RTTI "Enable if LLVM was build with RTTI enabled" ON) + set (USE_EMBEDDED_COMPILER 1) else() - set(LLVM_VERSION_POSTFIX "-5.0" CACHE STRING "") + set (USE_EMBEDDED_COMPILER 0) endif() - endif() - find_program(LLVM_CONFIG_EXECUTABLE - NAMES llvm-config${LLVM_VERSION_POSTFIX} llvm-config llvm-config-devel - PATHS $ENV{LLVM_ROOT}/bin) - - mark_as_advanced(LLVM_CONFIG_EXECUTABLE) - - if(NOT LLVM_CONFIG_EXECUTABLE) - message(FATAL_ERROR "Cannot find LLVM (looking for `llvm-config${LLVM_VERSION_POSTFIX}`, `llvm-config`, `llvm-config-devel`). Please, provide LLVM_ROOT environment variable.") + if (LLVM_FOUND AND OS_LINUX AND USE_LIBCXX) + message(WARNING "Option USE_INTERNAL_LLVM_LIBRARY is not set but the LLVM library from OS packages in Linux is incompatible with libc++ ABI. LLVM Will be disabled.") + set (LLVM_FOUND 0) + set (USE_EMBEDDED_COMPILER 0) + endif () else() - set(LLVM_FOUND TRUE) - - execute_process( - COMMAND ${LLVM_CONFIG_EXECUTABLE} --version - OUTPUT_VARIABLE LLVM_VERSION - OUTPUT_STRIP_TRAILING_WHITESPACE) - - if(LLVM_VERSION VERSION_LESS "5") - message(FATAL_ERROR "LLVM 5+ is required. You have ${LLVM_VERSION} (${LLVM_CONFIG_EXECUTABLE})") - endif() - - message(STATUS "LLVM config: ${LLVM_CONFIG_EXECUTABLE}; version: ${LLVM_VERSION}") - - execute_process( - COMMAND ${LLVM_CONFIG_EXECUTABLE} --includedir - OUTPUT_VARIABLE LLVM_INCLUDE_DIRS - OUTPUT_STRIP_TRAILING_WHITESPACE) - - execute_process( - COMMAND ${LLVM_CONFIG_EXECUTABLE} --libdir - OUTPUT_VARIABLE LLVM_LIBRARY_DIRS - OUTPUT_STRIP_TRAILING_WHITESPACE) - - execute_process( - COMMAND ${LLVM_CONFIG_EXECUTABLE} --cxxflags - OUTPUT_VARIABLE LLVM_CXXFLAGS - OUTPUT_STRIP_TRAILING_WHITESPACE) - - execute_process( - COMMAND ${LLVM_CONFIG_EXECUTABLE} --targets-built - OUTPUT_VARIABLE LLVM_TARGETS_BUILT - OUTPUT_STRIP_TRAILING_WHITESPACE) - - string(REPLACE " " ";" LLVM_TARGETS_BUILT "${LLVM_TARGETS_BUILT}") - - if (USE_STATIC_LIBRARIES) - set (LLVM_CONFIG_ADD "--link-static") - endif() - - # Get the link libs we need. - function(llvm_map_components_to_libraries RESULT) - execute_process( - COMMAND ${LLVM_CONFIG_EXECUTABLE} ${LLVM_CONFIG_ADD} --libs ${ARGN} - OUTPUT_VARIABLE _tmp - OUTPUT_STRIP_TRAILING_WHITESPACE) - - string(REPLACE " " ";" _libs_module "${_tmp}") - - #message(STATUS "LLVM Libraries for '${ARGN}': ${_libs_module}") - - execute_process( - COMMAND ${LLVM_CONFIG_EXECUTABLE} --system-libs ${ARGN} - OUTPUT_VARIABLE _libs_system - OUTPUT_STRIP_TRAILING_WHITESPACE) - - string(REPLACE "\n" " " _libs_system "${_libs_system}") - string(REPLACE " " " " _libs_system "${_libs_system}") - string(REPLACE " " ";" _libs_system "${_libs_system}") - - set(${RESULT} ${_libs_module} ${_libs_system} PARENT_SCOPE) - endfunction(llvm_map_components_to_libraries) - - message(STATUS "LLVM Include Directory: ${LLVM_INCLUDE_DIRS}") - message(STATUS "LLVM Library Directory: ${LLVM_LIBRARY_DIRS}") - message(STATUS "LLVM C++ Compiler: ${LLVM_CXXFLAGS}") + set (LLVM_FOUND 1) + set (USE_EMBEDDED_COMPILER 1) + set (LLVM_VERSION "7.0.0bundled") + set (LLVM_INCLUDE_DIRS + ${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm/include + ${ClickHouse_BINARY_DIR}/contrib/llvm/llvm/include + ${ClickHouse_SOURCE_DIR}/contrib/llvm/clang/include + ${ClickHouse_BINARY_DIR}/contrib/llvm/clang/include + ${ClickHouse_BINARY_DIR}/contrib/llvm/llvm/tools/clang/include + ${ClickHouse_SOURCE_DIR}/contrib/llvm/lld/include + ${ClickHouse_BINARY_DIR}/contrib/llvm/lld/include + ${ClickHouse_BINARY_DIR}/contrib/llvm/llvm/tools/lld/include) + set (LLVM_LIBRARY_DIRS ${ClickHouse_BINARY_DIR}/contrib/llvm/llvm) endif() - if (LLVM_FOUND AND LLVM_INCLUDE_DIRS AND LLVM_LIBRARY_DIRS) - set(USE_EMBEDDED_COMPILER TRUE) + if (LLVM_FOUND) + message(STATUS "LLVM version: ${LLVM_PACKAGE_VERSION}") + message(STATUS "LLVM include Directory: ${LLVM_INCLUDE_DIRS}") + message(STATUS "LLVM library Directory: ${LLVM_LIBRARY_DIRS}") + message(STATUS "LLVM C++ compiler flags: ${LLVM_CXXFLAGS}") endif() endif() + + +function(llvm_libs_all REQUIRED_LLVM_LIBRARIES) + llvm_map_components_to_libnames (result all) + list (REMOVE_ITEM result "LTO" "LLVM") + if (TERMCAP_LIBRARY) + list (APPEND result ${TERMCAP_LIBRARY}) + endif () + list (APPEND result ${CMAKE_DL_LIBS} ${ZLIB_LIBRARIES}) + set (${REQUIRED_LLVM_LIBRARIES} ${result} PARENT_SCOPE) +endfunction() diff --git a/cmake/find_ltdl.cmake b/cmake/find_ltdl.cmake index 935de0d4124..18003618dbd 100644 --- a/cmake/find_ltdl.cmake +++ b/cmake/find_ltdl.cmake @@ -1,3 +1,5 @@ -set (LTDL_PATHS "/usr/local/opt/libtool/lib") -find_library (LTDL_LIBRARY ltdl PATHS ${LTDL_PATHS}) -message (STATUS "Using ltdl: ${LTDL_LIBRARY}") +if (ENABLE_ODBC AND NOT USE_INTERNAL_ODBC_LIBRARY) + set (LTDL_PATHS "/usr/local/opt/libtool/lib") + find_library (LTDL_LIBRARY ltdl PATHS ${LTDL_PATHS}) + message (STATUS "Using ltdl: ${LTDL_LIBRARY}") +endif () diff --git a/cmake/find_odbc.cmake b/cmake/find_odbc.cmake index 338108910bf..32a410c6f11 100644 --- a/cmake/find_odbc.cmake +++ b/cmake/find_odbc.cmake @@ -13,54 +13,81 @@ # This module defines # ODBC_INCLUDE_DIRECTORIES, where to find sql.h # ODBC_LIBRARIES, the libraries to link against to use ODBC -# ODBC_FOUND. If false, you cannot build anything that requires MySQL. +# ODBC_FOUND. If false, you cannot build anything that requires ODBC. -find_path(ODBC_INCLUDE_DIRECTORIES - NAMES sql.h - HINTS - /usr/include - /usr/include/odbc - /usr/include/iodbc - /usr/local/include - /usr/local/include/odbc - /usr/local/include/iodbc - /usr/local/odbc/include - /usr/local/iodbc/include - "C:/Program Files/ODBC/include" - "C:/Program Files/Microsoft SDKs/Windows/v7.0/include" - "C:/Program Files/Microsoft SDKs/Windows/v6.0a/include" - "C:/ODBC/include" - DOC "Specify the directory containing sql.h." -) +option (ENABLE_ODBC "Enable ODBC" ${OS_LINUX}) +if (OS_LINUX) + option (USE_INTERNAL_ODBC_LIBRARY "Set to FALSE to use system odbc library instead of bundled" ${NOT_UNBUNDLED}) +else () + option (USE_INTERNAL_ODBC_LIBRARY "Set to FALSE to use system odbc library instead of bundled" OFF) +endif () -find_library(ODBC_LIBRARIES - NAMES iodbc odbc iodbcinst odbcinst odbc32 - HINTS - /usr/lib - /usr/lib/odbc - /usr/lib/iodbc - /usr/local/lib - /usr/local/lib/odbc - /usr/local/lib/iodbc - /usr/local/odbc/lib - /usr/local/iodbc/lib - "C:/Program Files/ODBC/lib" - "C:/ODBC/lib/debug" - "C:/Program Files (x86)/Microsoft SDKs/Windows/v7.0A/Lib" - DOC "Specify the ODBC driver manager library here." -) +if (USE_INTERNAL_ODBC_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/unixodbc/README") + message (WARNING "submodule contrib/unixodbc is missing. to fix try run: \n git submodule update --init --recursive") + set (USE_INTERNAL_ODBC_LIBRARY 0) +endif () -# MinGW find usually fails -if(MINGW) - set(ODBC_INCLUDE_DIRECTORIES ".") - set(ODBC_LIBRARIES odbc32) -endif() - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(ODBC - DEFAULT_MSG - ODBC_INCLUDE_DIRECTORIES - ODBC_LIBRARIES - ) +set (ODBC_INCLUDE_DIRECTORIES ) # Include directories will be either used automatically by target_include_directories or set later. -mark_as_advanced(ODBC_FOUND ODBC_LIBRARIES ODBC_INCLUDE_DIRECTORIES) +if (ENABLE_ODBC) + if (USE_INTERNAL_ODBC_LIBRARY) + set (ODBC_LIBRARIES unixodbc) + set (ODBC_FOUND 1) + set (USE_ODBC 1) + else () + find_path(ODBC_INCLUDE_DIRECTORIES + NAMES sql.h + HINTS + /usr/include + /usr/include/iodbc + /usr/include/odbc + /usr/local/include + /usr/local/include/iodbc + /usr/local/include/odbc + /usr/local/iodbc/include + /usr/local/odbc/include + "C:/Program Files/ODBC/include" + "C:/Program Files/Microsoft SDKs/Windows/v7.0/include" + "C:/Program Files/Microsoft SDKs/Windows/v6.0a/include" + "C:/ODBC/include" + DOC "Specify the directory containing sql.h." + ) + + find_library(ODBC_LIBRARIES + NAMES iodbc odbc iodbcinst odbcinst odbc32 + HINTS + /usr/lib + /usr/lib/iodbc + /usr/lib/odbc + /usr/local/lib + /usr/local/lib/iodbc + /usr/local/lib/odbc + /usr/local/iodbc/lib + /usr/local/odbc/lib + "C:/Program Files/ODBC/lib" + "C:/ODBC/lib/debug" + "C:/Program Files (x86)/Microsoft SDKs/Windows/v7.0A/Lib" + DOC "Specify the ODBC driver manager library here." + ) + + # MinGW find usually fails + if (MINGW) + set(ODBC_INCLUDE_DIRECTORIES ".") + set(ODBC_LIBRARIES odbc32) + endif () + + include(FindPackageHandleStandardArgs) + find_package_handle_standard_args(ODBC + DEFAULT_MSG + ODBC_INCLUDE_DIRECTORIES + ODBC_LIBRARIES) + + if (USE_STATIC_LIBRARIES) + list(APPEND ODBC_LIBRARIES ${LTDL_LIBRARY}) + endif () + + mark_as_advanced(ODBC_FOUND ODBC_LIBRARIES ODBC_INCLUDE_DIRECTORIES) + endif () +endif () + +message (STATUS "Using odbc=${ODBC_FOUND}: ${ODBC_INCLUDE_DIRECTORIES} : ${ODBC_LIBRARIES}") diff --git a/cmake/find_poco.cmake b/cmake/find_poco.cmake index 315e27b798f..012f269d48d 100644 --- a/cmake/find_poco.cmake +++ b/cmake/find_poco.cmake @@ -8,13 +8,25 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/poco/CMakeLists.txt") set (MISSING_INTERNAL_POCO_LIBRARY 1) endif () +set (POCO_COMPONENTS Net XML SQL Data) +if (NOT DEFINED ENABLE_POCO_NETSSL OR ENABLE_POCO_NETSSL) + list (APPEND POCO_COMPONENTS Crypto NetSSL) +endif () +if (NOT DEFINED ENABLE_POCO_MONGODB OR ENABLE_POCO_MONGODB) + list (APPEND POCO_COMPONENTS MongoDB) +endif () +# TODO: after new poco release with SQL library rename ENABLE_POCO_ODBC -> ENABLE_POCO_SQLODBC +if (NOT DEFINED ENABLE_POCO_ODBC OR ENABLE_POCO_ODBC) + list (APPEND POCO_COMPONENTS DataODBC) + list (APPEND POCO_COMPONENTS SQLODBC) +endif () + if (NOT USE_INTERNAL_POCO_LIBRARY) - find_package (Poco COMPONENTS Net NetSSL XML Data Crypto DataODBC MongoDB) + find_package (Poco COMPONENTS ${POCO_COMPONENTS}) endif () if (Poco_INCLUDE_DIRS AND Poco_Foundation_LIBRARY) elseif (NOT MISSING_INTERNAL_POCO_LIBRARY) - set (USE_INTERNAL_POCO_LIBRARY 1) set (ENABLE_ZIP 0 CACHE BOOL "") @@ -24,6 +36,15 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY) set (ENABLE_DATA_SQLITE 0 CACHE BOOL "") set (ENABLE_DATA_MYSQL 0 CACHE BOOL "") set (ENABLE_DATA_POSTGRESQL 0 CACHE BOOL "") + # new after 2.0.0: + set (POCO_ENABLE_ZIP 0 CACHE BOOL "") + set (POCO_ENABLE_PAGECOMPILER 0 CACHE BOOL "") + set (POCO_ENABLE_PAGECOMPILER_FILE2PAGE 0 CACHE BOOL "") + set (POCO_ENABLE_REDIS 0 CACHE BOOL "") + set (POCO_ENABLE_SQL_SQLITE 0 CACHE BOOL "") + set (POCO_ENABLE_SQL_MYSQL 0 CACHE BOOL "") + set (POCO_ENABLE_SQL_POSTGRESQL 0 CACHE BOOL "") + set (POCO_UNBUNDLED 1 CACHE BOOL "") set (POCO_UNBUNDLED_PCRE 0 CACHE BOOL "") set (POCO_UNBUNDLED_EXPAT 0 CACHE BOOL "") @@ -37,23 +58,43 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY) "${ClickHouse_SOURCE_DIR}/contrib/poco/Util/include/" ) - if (NOT DEFINED POCO_ENABLE_MONGODB OR POCO_ENABLE_MONGODB) - set (Poco_MongoDB_FOUND 1) + if (NOT DEFINED ENABLE_POCO_MONGODB OR ENABLE_POCO_MONGODB) set (Poco_MongoDB_LIBRARY PocoMongoDB) - set (Poco_MongoDB_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/poco/MongoDB/include/") + set (Poco_MongoDB_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/poco/MongoDB/include/") endif () - if (ODBC_FOUND) - set (Poco_DataODBC_FOUND 1) - set (Poco_DataODBC_LIBRARY PocoDataODBC ${ODBC_LIBRARIES} ${LTDL_LIBRARY}) - set (Poco_DataODBC_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/poco/Data/ODBC/include/") + if (EXISTS "${ClickHouse_SOURCE_DIR}/contrib/poco/SQL/ODBC/include/") + set (Poco_SQL_FOUND 1) + set (Poco_SQL_LIBRARY PocoSQL) + set (Poco_SQL_INCLUDE_DIR + "${ClickHouse_SOURCE_DIR}/contrib/poco/SQL/include" + "${ClickHouse_SOURCE_DIR}/contrib/poco/Data/include" + ) + if ((NOT DEFINED POCO_ENABLE_SQL_ODBC OR POCO_ENABLE_SQL_ODBC) AND ODBC_FOUND) + set (Poco_SQLODBC_INCLUDE_DIR + "${ClickHouse_SOURCE_DIR}/contrib/poco/SQL/ODBC/include/" + "${ClickHouse_SOURCE_DIR}/contrib/poco/Data/ODBC/include/" + ${ODBC_INCLUDE_DIRECTORIES} + ) + set (Poco_SQLODBC_LIBRARY PocoSQLODBC ${ODBC_LIBRARIES} ${LTDL_LIBRARY}) + endif () + else () + set (Poco_Data_FOUND 1) + set (Poco_Data_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/poco/Data/include") + set (Poco_Data_LIBRARY PocoData) + if ((NOT DEFINED ENABLE_DATA_ODBC OR ENABLE_DATA_ODBC) AND ODBC_FOUND) + set (USE_POCO_DATAODBC 1) + set (Poco_DataODBC_INCLUDE_DIR + "${ClickHouse_SOURCE_DIR}/contrib/poco/Data/ODBC/include/" + ${ODBC_INCLUDE_DIRECTORIES} + ) + set (Poco_DataODBC_LIBRARY PocoDataODBC ${ODBC_LIBRARIES} ${LTDL_LIBRARY}) + endif () endif () - # TODO! fix internal ssl - if (OPENSSL_FOUND AND NOT USE_INTERNAL_SSL_LIBRARY) - set (Poco_NetSSL_FOUND 1) - set (Poco_NetSSL_LIBRARY PocoNetSSL) - set (Poco_Crypto_LIBRARY PocoCrypto) + if (OPENSSL_FOUND AND (NOT DEFINED ENABLE_POCO_NETSSL OR ENABLE_POCO_NETSSL)) + set (Poco_NetSSL_LIBRARY PocoNetSSL ${OPENSSL_LIBRARIES}) + set (Poco_Crypto_LIBRARY PocoCrypto ${OPENSSL_LIBRARIES}) endif () if (USE_STATIC_LIBRARIES AND USE_INTERNAL_ZLIB_LIBRARY) @@ -66,11 +107,23 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY) set (Poco_Foundation_LIBRARY PocoFoundation) set (Poco_Util_LIBRARY PocoUtil) set (Poco_Net_LIBRARY PocoNet) - set (Poco_Data_LIBRARY PocoData) set (Poco_XML_LIBRARY PocoXML) endif () -message(STATUS "Using Poco: ${Poco_INCLUDE_DIRS} : ${Poco_Foundation_LIBRARY},${Poco_Util_LIBRARY},${Poco_Net_LIBRARY},${Poco_NetSSL_LIBRARY},${Poco_XML_LIBRARY},${Poco_Data_LIBRARY},${Poco_DataODBC_LIBRARY},${Poco_MongoDB_LIBRARY}; MongoDB=${Poco_MongoDB_FOUND}, DataODBC=${Poco_DataODBC_FOUND}, NetSSL=${Poco_NetSSL_FOUND}") +if (Poco_NetSSL_LIBRARY AND Poco_Crypto_LIBRARY) + set (USE_POCO_NETSSL 1) +endif () +if (Poco_MongoDB_LIBRARY) + set (USE_POCO_MONGODB 1) +endif () +if (Poco_DataODBC_LIBRARY AND ODBC_FOUND) + set (USE_POCO_DATAODBC 1) +endif () +if (Poco_SQLODBC_LIBRARY AND ODBC_FOUND) + set (USE_POCO_SQLODBC 1) +endif () + +message(STATUS "Using Poco: ${Poco_INCLUDE_DIRS} : ${Poco_Foundation_LIBRARY},${Poco_Util_LIBRARY},${Poco_Net_LIBRARY},${Poco_NetSSL_LIBRARY},${Poco_Crypto_LIBRARY},${Poco_XML_LIBRARY},${Poco_Data_LIBRARY},${Poco_DataODBC_LIBRARY},${Poco_SQL_LIBRARY},${Poco_SQLODBC_LIBRARY},${Poco_MongoDB_LIBRARY}; MongoDB=${USE_POCO_MONGODB}, DataODBC=${USE_POCO_DATAODBC}, NetSSL=${USE_POCO_NETSSL}") # How to make sutable poco: # use branch: @@ -86,3 +139,5 @@ message(STATUS "Using Poco: ${Poco_INCLUDE_DIRS} : ${Poco_Foundation_LIBRARY},${ # ClickHouse-Extras/clickhouse_warning # ClickHouse-Extras/clickhouse-purge-logs-on-no-space # ClickHouse-Extras/clickhouse_freebsd +# ClickHouse-Extras/clikhouse_no_zlib +# ClickHouse-Extras/clickhouse-fix-atomic diff --git a/cmake/find_rdkafka.cmake b/cmake/find_rdkafka.cmake index e5964b51dc7..a978ff026f7 100644 --- a/cmake/find_rdkafka.cmake +++ b/cmake/find_rdkafka.cmake @@ -1,8 +1,12 @@ -option (ENABLE_RDKAFKA "Enable kafka" ${NOT_MSVC}) +if (NOT ARCH_ARM) + option (ENABLE_RDKAFKA "Enable kafka" ON) +endif () if (ENABLE_RDKAFKA) -option (USE_INTERNAL_RDKAFKA_LIBRARY "Set to FALSE to use system librdkafka instead of the bundled" ${NOT_UNBUNDLED}) +if (OS_LINUX) + option (USE_INTERNAL_RDKAFKA_LIBRARY "Set to FALSE to use system librdkafka instead of the bundled" ${NOT_UNBUNDLED}) +endif () if (USE_INTERNAL_RDKAFKA_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/CMakeLists.txt") message (WARNING "submodule contrib/librdkafka is missing. to fix try run: \n git submodule update --init --recursive") @@ -13,7 +17,7 @@ endif () if (NOT USE_INTERNAL_RDKAFKA_LIBRARY) find_library (RDKAFKA_LIB rdkafka) find_path (RDKAFKA_INCLUDE_DIR NAMES librdkafka/rdkafka.h PATHS ${RDKAFKA_INCLUDE_PATHS}) - if (USE_STATIC_LIBRARIES AND NOT ARCH_FREEBSD) + if (USE_STATIC_LIBRARIES AND NOT OS_FREEBSD) find_library (SASL2_LIBRARY sasl2) endif () endif () @@ -24,6 +28,9 @@ if (RDKAFKA_LIB AND RDKAFKA_INCLUDE_DIR) if (SASL2_LIBRARY) list (APPEND RDKAFKA_LIBRARY ${SASL2_LIBRARY}) endif () + if (LZ4_LIBRARY) + list (APPEND RDKAFKA_LIBRARY ${LZ4_LIBRARY}) + endif () elseif (NOT MISSING_INTERNAL_RDKAFKA_LIBRARY) set (USE_INTERNAL_RDKAFKA_LIBRARY 1) set (RDKAFKA_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/src") diff --git a/cmake/find_readline_edit.cmake b/cmake/find_readline_edit.cmake index 5ec30b8e314..c2bba6cbfab 100644 --- a/cmake/find_readline_edit.cmake +++ b/cmake/find_readline_edit.cmake @@ -1,6 +1,9 @@ include (CMakePushCheckState) cmake_push_check_state () +option (ENABLE_READLINE "Enable readline" 1) +if (ENABLE_READLINE) + set (READLINE_PATHS "/usr/local/opt/readline/lib") # First try find custom lib for macos users (default lib without history support) find_library (READLINE_LIB NAMES readline PATHS ${READLINE_PATHS} NO_DEFAULT_PATH) @@ -10,30 +13,31 @@ endif () list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES .so.2) -find_library (TERMCAP_LIB NAMES termcap) find_library (EDIT_LIB NAMES edit) set(READLINE_INCLUDE_PATHS "/usr/local/opt/readline/include") -if (READLINE_LIB AND TERMCAP_LIB) +if (READLINE_LIB AND TERMCAP_LIBRARY) find_path (READLINE_INCLUDE_DIR NAMES readline/readline.h PATHS ${READLINE_INCLUDE_PATHS} NO_DEFAULT_PATH) if (NOT READLINE_INCLUDE_DIR) find_path (READLINE_INCLUDE_DIR NAMES readline/readline.h PATHS ${READLINE_INCLUDE_PATHS}) endif () if (READLINE_INCLUDE_DIR AND READLINE_LIB) set (USE_READLINE 1) - set (LINE_EDITING_LIBS ${READLINE_LIB} ${TERMCAP_LIB}) + set (LINE_EDITING_LIBS ${READLINE_LIB} ${TERMCAP_LIBRARY}) message (STATUS "Using line editing libraries (readline): ${READLINE_INCLUDE_DIR} : ${LINE_EDITING_LIBS}") endif () -elseif (EDIT_LIB AND TERMCAP_LIB) +elseif (EDIT_LIB AND TERMCAP_LIBRARY) find_library (CURSES_LIB NAMES curses) find_path (READLINE_INCLUDE_DIR NAMES editline/readline.h PATHS ${READLINE_INCLUDE_PATHS}) if (CURSES_LIB AND READLINE_INCLUDE_DIR) set (USE_LIBEDIT 1) - set (LINE_EDITING_LIBS ${EDIT_LIB} ${CURSES_LIB} ${TERMCAP_LIB}) + set (LINE_EDITING_LIBS ${EDIT_LIB} ${CURSES_LIB} ${TERMCAP_LIBRARY}) message (STATUS "Using line editing libraries (edit): ${READLINE_INCLUDE_DIR} : ${LINE_EDITING_LIBS}") endif () endif () +endif () + if (LINE_EDITING_LIBS AND READLINE_INCLUDE_DIR) include (CheckCXXSourceRuns) diff --git a/cmake/find_rt.cmake b/cmake/find_rt.cmake index 56c7f3cd777..25614fe55eb 100644 --- a/cmake/find_rt.cmake +++ b/cmake/find_rt.cmake @@ -1,15 +1,10 @@ if (APPLE) # lib from libs/libcommon set (RT_LIBRARY "apple_rt") -else () +elseif (OS_FREEBSD) find_library (RT_LIBRARY rt) +else () + set (RT_LIBRARY "") endif () message(STATUS "Using rt: ${RT_LIBRARY}") - -function (target_link_rt_by_force TARGET) - if (NOT APPLE) - set (FLAGS "-Wl,-no-as-needed -lrt -Wl,-as-needed") - set_property (TARGET ${TARGET} APPEND PROPERTY LINK_FLAGS "${FLAGS}") - endif () -endfunction () diff --git a/cmake/find_ssl.cmake b/cmake/find_ssl.cmake index ec40e498da1..6d630c7198c 100644 --- a/cmake/find_ssl.cmake +++ b/cmake/find_ssl.cmake @@ -1,4 +1,6 @@ -option (USE_INTERNAL_SSL_LIBRARY "Set to FALSE to use system *ssl library instead of bundled" ${MSVC}) +#if (OS_LINUX) +option (USE_INTERNAL_SSL_LIBRARY "Set to FALSE to use system *ssl library instead of bundled" ${NOT_UNBUNDLED}) +#endif () set (OPENSSL_USE_STATIC_LIBS ${USE_STATIC_LIBRARIES}) diff --git a/cmake/find_termcap.cmake b/cmake/find_termcap.cmake new file mode 100644 index 00000000000..47b772331bb --- /dev/null +++ b/cmake/find_termcap.cmake @@ -0,0 +1,5 @@ +find_library (TERMCAP_LIBRARY termcap) +if (NOT TERMCAP_LIBRARY) + find_library (TERMCAP_LIBRARY tinfo) +endif() +message (STATUS "Using termcap: ${TERMCAP_LIBRARY}") diff --git a/cmake/find_zlib.cmake b/cmake/find_zlib.cmake index 17350f9fd58..f6b2d268291 100644 --- a/cmake/find_zlib.cmake +++ b/cmake/find_zlib.cmake @@ -1,4 +1,6 @@ -option (USE_INTERNAL_ZLIB_LIBRARY "Set to FALSE to use system zlib library instead of bundled" ${NOT_UNBUNDLED}) +if (NOT OS_FREEBSD) + option (USE_INTERNAL_ZLIB_LIBRARY "Set to FALSE to use system zlib library instead of bundled" ${NOT_UNBUNDLED}) +endif () if (NOT USE_INTERNAL_ZLIB_LIBRARY) find_package (ZLIB) @@ -15,15 +17,6 @@ if (NOT ZLIB_FOUND) endif () set (USE_INTERNAL_ZLIB_LIBRARY 1) - set (ZLIB_COMPAT 1) # for zlib-ng, also enables WITH_GZFILEOP - set (WITH_NATIVE_INSTRUCTIONS ${ARCHNATIVE}) - if (ARCH_FREEBSD OR ARCH_I386) - set (WITH_OPTIM 0 CACHE INTERNAL "") # Bug in assembler - endif () - if (ARCH_AARCH64) - set(WITH_NEON 1 CACHE INTERNAL "") - set(WITH_ACLE 1 CACHE INTERNAL "") - endif () set (ZLIB_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}" "${ClickHouse_BINARY_DIR}/contrib/${INTERNAL_ZLIB_NAME}") # generated zconf.h set (ZLIB_INCLUDE_DIRS ${ZLIB_INCLUDE_DIR}) # for poco set (ZLIB_FOUND 1) # for poco diff --git a/cmake/find_zookeeper.cmake b/cmake/find_zookeeper.cmake deleted file mode 100644 index f667c310eb4..00000000000 --- a/cmake/find_zookeeper.cmake +++ /dev/null @@ -1,35 +0,0 @@ -if (ARCH_ARM) - # bundled have some asm broken for arm, use package libzookeeper-mt-dev - set(USE_INTERNAL_ZOOKEEPER_LIBRARY 0 CACHE BOOL "") -endif () - -option (USE_INTERNAL_ZOOKEEPER_LIBRARY "Set to FALSE to use system zookeeper library instead of bundled" ${NOT_UNBUNDLED}) - -if (USE_INTERNAL_ZOOKEEPER_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/zookeeper/src/c/CMakeLists.txt") - message (WARNING "submodule contrib/zookeeper is missing. to fix try run: \n git submodule update --init --recursive") - set (USE_INTERNAL_ZOOKEEPER_LIBRARY 0) -endif () - -if (NOT USE_INTERNAL_ZOOKEEPER_LIBRARY) - find_library (ZOOKEEPER_LIBRARY zookeeper_mt) - find_path (ZOOKEEPER_INCLUDE_DIR NAMES zookeeper/zookeeper.h PATHS ${ZOOKEEPER_INCLUDE_PATHS}) - set(ZOOKEEPER_INCLUDE_DIR "${ZOOKEEPER_INCLUDE_DIR}/zookeeper") -endif () - -if (ZOOKEEPER_LIBRARY AND ZOOKEEPER_INCLUDE_DIR) -else () - set (USE_INTERNAL_ZOOKEEPER_LIBRARY 1) - set(WANT_CPPUNIT 0 CACHE BOOL "") - set (ZOOKEEPER_LIBRARY zookeeper) -endif () - -message (STATUS "Using zookeeper: ${ZOOKEEPER_INCLUDE_DIR} : ${ZOOKEEPER_LIBRARY}") - - -# how to make clickhouse branch of https://github.com/ClickHouse-Extras/zookeeper.git : -# git remote add upstream https://github.com/apache/zookeeper.git -# git checkhout upstream/master -# git branch -D clickhouse -# git checkout -b clickhouse -# git merge clickhouse_misc -# git merge clickhouse_706 diff --git a/cmake/generate_code.cmake b/cmake/generate_code.cmake new file mode 100644 index 00000000000..8eb9da24d1d --- /dev/null +++ b/cmake/generate_code.cmake @@ -0,0 +1,5 @@ +function(generate_code TEMPLATE_FILE) + foreach(NAME IN LISTS ARGN) + configure_file (${TEMPLATE_FILE}.cpp.in ${CMAKE_CURRENT_BINARY_DIR}/generated/${TEMPLATE_FILE}_${NAME}.cpp) + endforeach() +endfunction() diff --git a/cmake/lib_name.cmake b/cmake/lib_name.cmake index 79e1b3e19e4..5c919b263e6 100644 --- a/cmake/lib_name.cmake +++ b/cmake/lib_name.cmake @@ -1,5 +1,4 @@ set(DIVIDE_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libdivide) -set(CITYHASH_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcityhash/include) set(COMMON_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/libs/libcommon/include ${ClickHouse_BINARY_DIR}/libs/libcommon/include) set(DBMS_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/dbms/src ${ClickHouse_BINARY_DIR}/dbms/src) set(DOUBLE_CONVERSION_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/double-conversion) diff --git a/cmake/print_include_directories.cmake b/cmake/print_include_directories.cmake index 41c4773cfa3..c4c5d00c54f 100644 --- a/cmake/print_include_directories.cmake +++ b/cmake/print_include_directories.cmake @@ -10,6 +10,9 @@ list(APPEND dirs ${dirs1}) get_property (dirs1 TARGET common PROPERTY INCLUDE_DIRECTORIES) list(APPEND dirs ${dirs1}) +get_property (dirs1 TARGET cityhash PROPERTY INCLUDE_DIRECTORIES) +list(APPEND dirs ${dirs1}) + if (USE_INTERNAL_BOOST_LIBRARY) get_property (dirs1 TARGET ${Boost_PROGRAM_OPTIONS_LIBRARY} PROPERTY INCLUDE_DIRECTORIES) list(APPEND dirs ${dirs1}) diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake index bac27578663..9cce2da6725 100644 --- a/cmake/sanitize.cmake +++ b/cmake/sanitize.cmake @@ -1,27 +1,37 @@ +option (SANITIZE "Enable sanitizer: address, memory, thread, undefined" "") + set (SAN_FLAGS "${SAN_FLAGS} -g -fno-omit-frame-pointer -DSANITIZER") -if (SAN_DEBUG) - set (SAN_FLAGS "${SAN_FLAGS} -O0") -else () - set (SAN_FLAGS "${SAN_FLAGS} -O3") -endif () -set (CMAKE_CXX_FLAGS_ASAN "${CMAKE_CXX_FLAGS_ASAN} ${SAN_FLAGS} -fsanitize=address") -set (CMAKE_C_FLAGS_ASAN "${CMAKE_C_FLAGS_ASAN} ${SAN_FLAGS} -fsanitize=address") -set (CMAKE_EXE_LINKER_FLAGS_ASAN "${CMAKE_EXE_LINKER_FLAGS_ASAN} -fsanitize=address") -set (CMAKE_CXX_FLAGS_UBSAN "${CMAKE_CXX_FLAGS_UBSAN} ${SAN_FLAGS} -fsanitize=undefined") -set (CMAKE_C_FLAGS_UBSAN "${CMAKE_C_FLAGS_UBSAN} ${SAN_FLAGS} -fsanitize=undefined") -set (CMAKE_EXE_LINKER_FLAGS_UBSAN "${CMAKE_EXE_LINKER_FLAGS_UBSAN} -fsanitize=undefined") -set (CMAKE_CXX_FLAGS_MSAN "${CMAKE_CXX_FLAGS_MSAN} ${SAN_FLAGS} -fsanitize=memory") -set (CMAKE_C_FLAGS_MSAN "${CMAKE_C_FLAGS_MSAN} ${SAN_FLAGS} -fsanitize=memory") -set (CMAKE_EXE_LINKER_FLAGS_MSAN "${CMAKE_EXE_LINKER_FLAGS_MSAN} -fsanitize=memory") -set (CMAKE_CXX_FLAGS_TSAN "${CMAKE_CXX_FLAGS_TSAN} ${SAN_FLAGS} -fsanitize=thread") -set (CMAKE_C_FLAGS_TSAN "${CMAKE_C_FLAGS_TSAN} ${SAN_FLAGS} -fsanitize=thread") -set (CMAKE_EXE_LINKER_FLAGS_TSAN "${CMAKE_EXE_LINKER_FLAGS_TSAN} -fsanitize=thread") - -# clang use static linking by default -if (MAKE_STATIC_LIBRARIES AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_EXE_LINKER_FLAGS_ASAN "${CMAKE_EXE_LINKER_FLAGS_ASAN} -static-libasan") - set (CMAKE_EXE_LINKER_FLAGS_UBSAN "${CMAKE_EXE_LINKER_FLAGS_UBSAN} -static-libubsan") - set (CMAKE_EXE_LINKER_FLAGS_MSAN "${CMAKE_EXE_LINKER_FLAGS_MSAN} -static-libmsan") - set (CMAKE_EXE_LINKER_FLAGS_TSAN "${CMAKE_EXE_LINKER_FLAGS_TSAN} -static-libtsan") -endif () +if (SANITIZE) + if (SANITIZE STREQUAL "address") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=address -fsanitize-address-use-after-scope") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=address -fsanitize-address-use-after-scope") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address -fsanitize-address-use-after-scope") + if (MAKE_STATIC_LIBRARIES AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libasan") + endif () + elseif (SANITIZE STREQUAL "memory") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=memory") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=memory") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=memory") + if (MAKE_STATIC_LIBRARIES AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libmsan") + endif () + elseif (SANITIZE STREQUAL "thread") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=thread") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=thread") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread") + if (MAKE_STATIC_LIBRARIES AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libtsan") + endif () + elseif (SANITIZE STREQUAL "undefined") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=undefined") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=undefined") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined") + if (MAKE_STATIC_LIBRARIES AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libubsan") + endif () + else () + message (FATAL_ERROR "Unknown sanitizer type: ${SANITIZE}") + endif () +endif() diff --git a/cmake/test_compiler.cmake b/cmake/test_compiler.cmake index ba69155914a..cba7a04e5e3 100644 --- a/cmake/test_compiler.cmake +++ b/cmake/test_compiler.cmake @@ -14,6 +14,7 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") endif () set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG}") + set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} Threads::Threads) check_cxx_source_compiles(" #include @@ -25,8 +26,6 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") else () - cmake_push_check_state () - set (TEST_FLAG "-no-pie") set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG}") @@ -40,7 +39,6 @@ else () set (FLAG_NO_PIE ${TEST_FLAG}) endif () - endif () cmake_pop_check_state () diff --git a/cmake/test_cpu.cmake b/cmake/test_cpu.cmake index 8a301983963..c360de5b962 100644 --- a/cmake/test_cpu.cmake +++ b/cmake/test_cpu.cmake @@ -14,11 +14,12 @@ cmake_push_check_state () #define __SSE4_1__ 1 set (TEST_FLAG "-msse4.1") -set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG}") +set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0") check_cxx_source_compiles(" #include int main() { - _mm_insert_epi8(__m128i(), 0, 0); + auto a = _mm_insert_epi8(__m128i(), 0, 0); + (void)a; return 0; } " HAVE_SSE41) @@ -31,11 +32,12 @@ endif () #define __SSE4_2__ 1 set (TEST_FLAG "-msse4.2") -set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG}") +set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0") check_cxx_source_compiles(" #include int main() { - _mm_crc32_u64(0, 0); + auto a = _mm_crc32_u64(0, 0); + (void)a; return 0; } " HAVE_SSE42) @@ -43,16 +45,49 @@ if (HAVE_SSE42) set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}") endif () +set (TEST_FLAG "-mssse3") +set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0") +check_cxx_source_compiles(" + #include + int main() { + __m64 a = _mm_abs_pi8(__m64()); + (void)a; + return 0; + } +" HAVE_SSSE3) + +set (TEST_FLAG "-mavx") +set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0") +check_cxx_source_compiles(" + #include + int main() { + auto a = _mm256_insert_epi8(__m256i(), 0, 0); + (void)a; + return 0; + } +" HAVE_AVX) + +set (TEST_FLAG "-mavx2") +set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0") +check_cxx_source_compiles(" + #include + int main() { + auto a = _mm256_add_epi16(__m256i(), __m256i()); + (void)a; + return 0; + } +" HAVE_AVX2) # gcc -dM -E -mpopcnt - < /dev/null | sort > gcc-dump-popcnt #define __POPCNT__ 1 set (TEST_FLAG "-mpopcnt") -set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG}") +set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0") check_cxx_source_compiles(" int main() { - __builtin_popcountll(0); + auto a = __builtin_popcountll(0); + (void)a; return 0; } " HAVE_POPCNT) @@ -62,5 +97,3 @@ if (HAVE_POPCNT AND NOT ARCH_AARCH64) endif () cmake_pop_check_state () - -# TODO: add here sse3 test if you want use it diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 9015f7e7d36..b58d211af41 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -1,5 +1,11 @@ -if (NOT MSVC) - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast") +# Third-party libraries may have substandard code. + +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-stringop-overflow") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-implicit-fallthrough -Wno-class-memaccess -std=c++1z") +elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-format -Wno-parentheses-equality") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-format -std=c++1z") endif () if (USE_INTERNAL_BOOST_LIBRARY) @@ -21,16 +27,12 @@ if (USE_INTERNAL_RE2_LIBRARY) endif () if (USE_INTERNAL_DOUBLE_CONVERSION_LIBRARY) - set (BUILD_TESTING ${ENABLE_TESTS} CACHE INTERNAL "") + set (BUILD_TESTING 0 CACHE INTERNAL "") add_subdirectory (double-conversion) endif () -if (USE_INTERNAL_ZOOKEEPER_LIBRARY) - add_subdirectory (zookeeper/src/c) -endif () - if (USE_INTERNAL_CITYHASH_LIBRARY) - add_subdirectory (libcityhash) + add_subdirectory (cityhash102) endif () if (USE_INTERNAL_FARMHASH_LIBRARY) @@ -41,6 +43,8 @@ if (USE_INTERNAL_METROHASH_LIBRARY) add_subdirectory (libmetrohash) endif () +add_subdirectory (murmurhash) + if (USE_INTERNAL_BTRIE_LIBRARY) add_subdirectory (libbtrie) endif () @@ -50,14 +54,24 @@ if (USE_INTERNAL_UNWIND_LIBRARY) endif () if (USE_INTERNAL_ZLIB_LIBRARY) + set (ZLIB_COMPAT 1 CACHE INTERNAL "") # also enables WITH_GZFILEOP + set (WITH_NATIVE_INSTRUCTIONS ${ARCH_NATIVE} CACHE INTERNAL "") + if (OS_FREEBSD OR ARCH_I386) + set (WITH_OPTIM 0 CACHE INTERNAL "") # Bug in assembler + endif () + if (ARCH_AARCH64) + set(WITH_NEON 1 CACHE INTERNAL "") + set(WITH_ACLE 1 CACHE INTERNAL "") + endif () + add_subdirectory (${INTERNAL_ZLIB_NAME}) # todo: make pull to Dead2/zlib-ng and remove: # We should use same defines when including zlib.h as used when zlib compiled target_compile_definitions (zlib PUBLIC ZLIB_COMPAT WITH_GZFILEOP) target_compile_definitions (zlibstatic PUBLIC ZLIB_COMPAT WITH_GZFILEOP) if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "AMD64") - target_compile_definitions (zlib PUBLIC X86_64) - target_compile_definitions (zlibstatic PUBLIC X86_64) + target_compile_definitions (zlib PUBLIC X86_64 UNALIGNED_OK) + target_compile_definitions (zlibstatic PUBLIC X86_64 UNALIGNED_OK) endif () set_target_properties(example PROPERTIES EXCLUDE_FROM_ALL 1) @@ -79,7 +93,11 @@ if (ENABLE_TCMALLOC AND USE_INTERNAL_GPERFTOOLS_LIBRARY) add_subdirectory (libtcmalloc) endif () -if (NOT ARCH_ARM) +if (ENABLE_JEMALLOC AND USE_INTERNAL_JEMALLOC_LIBRARY) + add_subdirectory (jemalloc-cmake) +endif () + +if (USE_INTERNAL_CPUID_LIBRARY) add_subdirectory (libcpuid) endif () @@ -88,40 +106,30 @@ if (USE_INTERNAL_SSL_LIBRARY) set (BUILD_SHARED 1) endif () set (USE_SHARED ${USE_STATIC_LIBRARIES}) + set (LIBRESSL_SKIP_INSTALL 1) add_subdirectory (ssl) - target_include_directories(${OPENSSL_CRYPTO_LIBRARY} PUBLIC ${OPENSSL_INCLUDE_DIR}) - target_include_directories(${OPENSSL_SSL_LIBRARY} PUBLIC ${OPENSSL_INCLUDE_DIR}) + target_include_directories(${OPENSSL_CRYPTO_LIBRARY} SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR}) + target_include_directories(${OPENSSL_SSL_LIBRARY} SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR}) +endif () + +if (ENABLE_MYSQL AND USE_INTERNAL_MYSQL_LIBRARY) + add_subdirectory (mariadb-connector-c-cmake) + target_include_directories(mysqlclient BEFORE PRIVATE ${ZLIB_INCLUDE_DIR}) + target_include_directories(mysqlclient BEFORE PRIVATE ${OPENSSL_INCLUDE_DIR}) endif () if (USE_INTERNAL_RDKAFKA_LIBRARY) - set (RDKAFKA_BUILD_EXAMPLES OFF CACHE INTERNAL "") - set (RDKAFKA_BUILD_TESTS OFF CACHE INTERNAL "") - set (RDKAFKA_BUILD_STATIC ${MAKE_STATIC_LIBRARIES} CACHE INTERNAL "") - mark_as_advanced (ZLIB_INCLUDE_DIR) + add_subdirectory (librdkafka-cmake) + target_include_directories(rdkafka BEFORE PRIVATE ${ZLIB_INCLUDE_DIR}) + target_include_directories(rdkafka BEFORE PRIVATE ${OPENSSL_INCLUDE_DIR}) +endif () - if (USE_INTERNAL_SSL_LIBRARY) - if (MAKE_STATIC_LIBRARIES) - add_library(bundled-ssl ALIAS ${OPENSSL_SSL_LIBRARY}) - set (WITH_BUNDLED_SSL 1 CACHE INTERNAL "") - else () - set (WITH_SSL 0 CACHE INTERNAL "") - endif () - endif () - - add_subdirectory (librdkafka) - - if (USE_INTERNAL_SSL_LIBRARY AND MAKE_STATIC_LIBRARIES) - target_include_directories(rdkafka PRIVATE BEFORE ${OPENSSL_INCLUDE_DIR}) - endif () - target_include_directories(rdkafka PRIVATE BEFORE ${ZLIB_INCLUDE_DIR}) +if (ENABLE_ODBC AND USE_INTERNAL_ODBC_LIBRARY) + add_subdirectory (unixodbc-cmake) endif () if (USE_INTERNAL_CAPNP_LIBRARY) - if (APPLE) # tests never end - set (BUILD_TESTING 0 CACHE INTERNAL "") - else () - set (BUILD_TESTING ${ENABLE_TESTS} CACHE INTERNAL "") - endif () + set (BUILD_TESTING 0 CACHE INTERNAL "") set (_save ${CMAKE_CXX_EXTENSIONS}) set (CMAKE_CXX_EXTENSIONS) add_subdirectory (capnproto/c++) @@ -129,18 +137,13 @@ if (USE_INTERNAL_CAPNP_LIBRARY) target_include_directories(${CAPNP_LIBRARY} PUBLIC $) endif () - if (USE_INTERNAL_POCO_LIBRARY) set (save_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) set (save_CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) set (_save ${ENABLE_TESTS}) set (ENABLE_TESTS 0) + set (POCO_ENABLE_TESTS 0) set (CMAKE_DISABLE_FIND_PACKAGE_ZLIB 1) - if (USE_INTERNAL_SSL_LIBRARY) - set (DISABLE_INTERNAL_OPENSSL 1 CACHE INTERNAL "") - set (ENABLE_NETSSL 0 CACHE INTERNAL "") # TODO! - set (ENABLE_CRYPTO 0 CACHE INTERNAL "") # TODO! - endif () if (MSVC) set (ENABLE_DATA_ODBC 0 CACHE INTERNAL "") # TODO (build fail) endif () @@ -149,8 +152,45 @@ if (USE_INTERNAL_POCO_LIBRARY) set (ENABLE_TESTS ${_save}) set (CMAKE_CXX_FLAGS ${save_CMAKE_CXX_FLAGS}) set (CMAKE_C_FLAGS ${save_CMAKE_C_FLAGS}) - if (OPENSSL_FOUND AND TARGET Crypto) + + if (OPENSSL_FOUND AND TARGET Crypto AND (NOT DEFINED ENABLE_POCO_NETSSL OR ENABLE_POCO_NETSSL)) # Bug in poco https://github.com/pocoproject/poco/pull/2100 found on macos - target_include_directories(Crypto PUBLIC ${OPENSSL_INCLUDE_DIR}) + target_include_directories(Crypto SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR}) endif () endif () + +if (USE_INTERNAL_GTEST_LIBRARY) + # Google Test from sources + add_subdirectory(${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest ${CMAKE_CURRENT_BINARY_DIR}/googletest) + # avoid problems with + target_compile_definitions (gtest INTERFACE GTEST_HAS_POSIX_RE=0) + target_include_directories (gtest SYSTEM INTERFACE ${ClickHouse_SOURCE_DIR}/contrib/googletest/include) +endif () + +if (USE_INTERNAL_LLVM_LIBRARY) + file(GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/empty.cpp CONTENT " ") + add_library(LLVM0 ${CMAKE_CURRENT_BINARY_DIR}/empty.cpp) # silly cmake bug fix + # ld: unknown option: --color-diagnostics + if (APPLE) + set (LINKER_SUPPORTS_COLOR_DIAGNOSTICS 0 CACHE INTERNAL "") + endif () + set (LLVM_ENABLE_EH 1 CACHE INTERNAL "") + set (LLVM_ENABLE_RTTI 1 CACHE INTERNAL "") + set (LLVM_INCLUDE_TESTS 0 CACHE INTERNAL "") + set (LLVM_INCLUDE_EXAMPLES 0 CACHE INTERNAL "") + set (LLVM_INCLUDE_TOOLS 0 CACHE INTERNAL "") + set (LLVM_INSTALL_TOOLCHAIN_ONLY 0 CACHE INTERNAL "") + set (CLANG_BUILT_STANDALONE 0 CACHE INTERNAL "") + set (LLDB_BUILT_STANDALONE 0 CACHE INTERNAL "") + set (CLANG_ENABLE_STATIC_ANALYZER 0 CACHE INTERNAL "") + set (CLANG_ENABLE_ARCMT 0 CACHE INTERNAL "") + set (CLANG_BUILD_TOOLS 0 CACHE INTERNAL "") + set (BENCHMARK_ENABLE_GTEST_TESTS 0 CACHE INTERNAL "") + set (BENCHMARK_ENABLE_ASSEMBLY_TESTS 0 CACHE INTERNAL "") + set (LLVM_TARGETS_TO_BUILD "X86;AArch64" CACHE INTERNAL "") + add_subdirectory (llvm/llvm) +endif () + +if (USE_BASE64) + add_subdirectory (base64-cmake) +endif() diff --git a/contrib/base64 b/contrib/base64 new file mode 160000 index 00000000000..a27c565d1b6 --- /dev/null +++ b/contrib/base64 @@ -0,0 +1 @@ +Subproject commit a27c565d1b6c676beaf297fe503c4518185666f7 diff --git a/contrib/base64-cmake/.gitignore b/contrib/base64-cmake/.gitignore new file mode 100644 index 00000000000..0e56cf2f8c1 --- /dev/null +++ b/contrib/base64-cmake/.gitignore @@ -0,0 +1 @@ +config.h diff --git a/contrib/base64-cmake/CMakeLists.txt b/contrib/base64-cmake/CMakeLists.txt new file mode 100644 index 00000000000..09abb3b02b9 --- /dev/null +++ b/contrib/base64-cmake/CMakeLists.txt @@ -0,0 +1,52 @@ +SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/base64) + +set(base64_compile_instructions "") +LIST(LENGTH base64_compile_instructions 0) +macro(cast_to_bool var instruction) + if (HAVE_${var}) + set(base64_${var} 1) + set(base64_${var}_opt ${instruction}) + else() + set(base64_${var} 0) + endif() +endmacro() + +cast_to_bool(SSSE3 "-mssse3") +cast_to_bool(SSE41 "-msse4.1") +cast_to_bool(SSE42 "-msse4.2") +cast_to_bool(AVX "-mavx") +cast_to_bool(AVX2 "-mavx2") + +# write config.h file, to include it in application +file(READ config-header.tpl header) +file(WRITE config.h ${header}) +file(APPEND config.h "#define HAVE_SSSE3 ${base64_SSSE3}\n") +file(APPEND config.h "#define HAVE_SSE41 ${base64_SSE41}\n") +file(APPEND config.h "#define HAVE_SSE42 ${base64_SSE42}\n") +file(APPEND config.h "#define HAVE_AVX ${base64_AVX}\n") +file(APPEND config.h "#define HAVE_AVX2 ${base64_AVX2}\n") + +set(HAVE_FAST_UNALIGNED_ACCESS 0) +if (${base64_SSSE3} OR ${base64_SSE41} OR ${base64_SSE42} OR ${base64_AVX} OR ${base64_AVX2}) + set(HAVE_FAST_UNALIGNED_ACCESS 1) +endif () + +file(APPEND config.h "#define HAVE_FAST_UNALIGNED_ACCESS " ${HAVE_FAST_UNALIGNED_ACCESS} "\n") + +add_library(base64 ${LINK_MODE} + ${LIBRARY_DIR}/lib/lib.c + ${LIBRARY_DIR}/lib/codec_choose.c + ${LIBRARY_DIR}/lib/arch/avx/codec.c + ${LIBRARY_DIR}/lib/arch/avx2/codec.c + ${LIBRARY_DIR}/lib/arch/generic/codec.c + ${LIBRARY_DIR}/lib/arch/neon32/codec.c + ${LIBRARY_DIR}/lib/arch/neon64/codec.c + ${LIBRARY_DIR}/lib/arch/sse41/codec.c + ${LIBRARY_DIR}/lib/arch/sse42/codec.c + ${LIBRARY_DIR}/lib/arch/ssse3/codec.c + + ${LIBRARY_DIR}/lib/codecs.h + config.h) + +target_compile_options(base64 PRIVATE ${base64_SSSE3_opt} ${base64_SSE41_opt} ${base64_SSE42_opt} ${base64_AVX_opt} ${base64_AVX2_opt}) +target_include_directories(base64 PRIVATE ${LIBRARY_DIR}/include .) \ No newline at end of file diff --git a/contrib/base64-cmake/config-header.tpl b/contrib/base64-cmake/config-header.tpl new file mode 100644 index 00000000000..c978ca0c082 --- /dev/null +++ b/contrib/base64-cmake/config-header.tpl @@ -0,0 +1,2 @@ +#define HAVE_NEON32 0 +#define HAVE_NEON64 0 diff --git a/contrib/boost b/contrib/boost index 5121cc9d037..2d5cb2c86f6 160000 --- a/contrib/boost +++ b/contrib/boost @@ -1 +1 @@ -Subproject commit 5121cc9d0375c7b81b24b6087a51684e6cd62ded +Subproject commit 2d5cb2c86f61126f4e1efe9ab97332efd44e7dea diff --git a/contrib/boost-cmake/CMakeLists.txt b/contrib/boost-cmake/CMakeLists.txt index 2a89293c902..49d50036e4b 100644 --- a/contrib/boost-cmake/CMakeLists.txt +++ b/contrib/boost-cmake/CMakeLists.txt @@ -16,7 +16,7 @@ if (NOT MSVC) add_definitions(-Wno-unused-variable -Wno-deprecated-declarations) endif () -add_library(boost_program_options_internal +add_library(boost_program_options_internal ${LINK_MODE} ${LIBRARY_DIR}/libs/program_options/src/cmdline.cpp ${LIBRARY_DIR}/libs/program_options/src/config_file.cpp ${LIBRARY_DIR}/libs/program_options/src/convert.cpp @@ -29,7 +29,7 @@ ${LIBRARY_DIR}/libs/program_options/src/value_semantic.cpp ${LIBRARY_DIR}/libs/program_options/src/variables_map.cpp ${LIBRARY_DIR}/libs/program_options/src/winmain.cpp) -add_library(boost_filesystem_internal +add_library(boost_filesystem_internal ${LINK_MODE} ${LIBRARY_DIR}/libs/filesystem/src/codecvt_error_category.cpp ${LIBRARY_DIR}/libs/filesystem/src/operations.cpp ${LIBRARY_DIR}/libs/filesystem/src/path.cpp @@ -39,12 +39,14 @@ ${LIBRARY_DIR}/libs/filesystem/src/unique_path.cpp ${LIBRARY_DIR}/libs/filesystem/src/utf8_codecvt_facet.cpp ${LIBRARY_DIR}/libs/filesystem/src/windows_file_codecvt.cpp) -add_library(boost_system_internal +add_library(boost_system_internal ${LINK_MODE} ${LIBRARY_DIR}/libs/system/src/error_code.cpp) -target_include_directories (boost_program_options_internal BEFORE PUBLIC ${Boost_INCLUDE_DIRS}) -target_include_directories (boost_filesystem_internal BEFORE PUBLIC ${Boost_INCLUDE_DIRS}) -target_include_directories (boost_system_internal BEFORE PUBLIC ${Boost_INCLUDE_DIRS}) +target_link_libraries (boost_filesystem_internal PUBLIC boost_system_internal) + +target_include_directories (boost_program_options_internal SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS}) +target_include_directories (boost_filesystem_internal SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS}) +target_include_directories (boost_system_internal SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS}) target_compile_definitions (boost_program_options_internal PUBLIC BOOST_SYSTEM_NO_DEPRECATED) target_compile_definitions (boost_filesystem_internal PUBLIC BOOST_SYSTEM_NO_DEPRECATED) diff --git a/contrib/boost-cmake/README b/contrib/boost-cmake/README index 7eec9460acd..69b266c7c7d 100644 --- a/contrib/boost-cmake/README +++ b/contrib/boost-cmake/README @@ -3,7 +3,7 @@ Motivation For reproducible build, we need to control, what exactly version of boost we build, because different versions of boost obviously have slightly different behaviour. -You may already have installed arbitary version of boost on your system, to build another projects. +You may already have installed arbitrary version of boost on your system, to build another projects. We need to have all libraries with C++ interface to be located in tree and to be build together. This is needed to allow quickly changing build options, that could introduce changes in ABI of that libraries. diff --git a/contrib/capnproto b/contrib/capnproto index c949a18da5f..a00ccd91b37 160000 --- a/contrib/capnproto +++ b/contrib/capnproto @@ -1 +1 @@ -Subproject commit c949a18da5f041a36cc218c5c4b79c7705999b4f +Subproject commit a00ccd91b3746ef2ab51d40fe3265829949d1ace diff --git a/contrib/cctz-cmake/CMakeLists.txt b/contrib/cctz-cmake/CMakeLists.txt index 9c2f6d9a658..5057c2594be 100644 --- a/contrib/cctz-cmake/CMakeLists.txt +++ b/contrib/cctz-cmake/CMakeLists.txt @@ -1,6 +1,6 @@ SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/cctz) -add_library(cctz +add_library(cctz ${LINK_MODE} ${LIBRARY_DIR}/src/civil_time_detail.cc ${LIBRARY_DIR}/src/time_zone_fixed.cc ${LIBRARY_DIR}/src/time_zone_format.cc diff --git a/contrib/cityhash102/CMakeLists.txt b/contrib/cityhash102/CMakeLists.txt new file mode 100644 index 00000000000..c3f53a8f878 --- /dev/null +++ b/contrib/cityhash102/CMakeLists.txt @@ -0,0 +1,8 @@ +add_library(cityhash + src/city.cc + include/citycrc.h + include/city.h + src/config.h) + +target_include_directories(cityhash BEFORE PUBLIC include) +target_include_directories(cityhash PRIVATE src) diff --git a/contrib/libcityhash/COPYING b/contrib/cityhash102/COPYING similarity index 100% rename from contrib/libcityhash/COPYING rename to contrib/cityhash102/COPYING diff --git a/contrib/libcityhash/include/city.h b/contrib/cityhash102/include/city.h similarity index 100% rename from contrib/libcityhash/include/city.h rename to contrib/cityhash102/include/city.h diff --git a/contrib/libcityhash/include/citycrc.h b/contrib/cityhash102/include/citycrc.h similarity index 100% rename from contrib/libcityhash/include/citycrc.h rename to contrib/cityhash102/include/citycrc.h diff --git a/contrib/libcityhash/src/city.cc b/contrib/cityhash102/src/city.cc similarity index 100% rename from contrib/libcityhash/src/city.cc rename to contrib/cityhash102/src/city.cc diff --git a/contrib/libcityhash/src/config.h b/contrib/cityhash102/src/config.h similarity index 97% rename from contrib/libcityhash/src/config.h rename to contrib/cityhash102/src/config.h index 61c75fca818..cca744a35c0 100644 --- a/contrib/libcityhash/src/config.h +++ b/contrib/cityhash102/src/config.h @@ -116,3 +116,10 @@ /* Define to the type of an unsigned integer type of width exactly 8 bits if such a type exists and the standard includes do not define it. */ /* #undef uint8_t */ + +#ifdef _MSC_VER + #include + typedef SSIZE_T ssize_t; +#else + #include +#endif diff --git a/contrib/jemalloc b/contrib/jemalloc new file mode 160000 index 00000000000..cd2931ad9bb --- /dev/null +++ b/contrib/jemalloc @@ -0,0 +1 @@ +Subproject commit cd2931ad9bbd78208565716ab102e86d858c2fff diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt new file mode 100644 index 00000000000..94b84884f41 --- /dev/null +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -0,0 +1,62 @@ +set(JEMALLOC_SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/jemalloc) + +set(SRCS +${JEMALLOC_SOURCE_DIR}/src/arena.c +${JEMALLOC_SOURCE_DIR}/src/background_thread.c +${JEMALLOC_SOURCE_DIR}/src/base.c +${JEMALLOC_SOURCE_DIR}/src/bin.c +${JEMALLOC_SOURCE_DIR}/src/bitmap.c +${JEMALLOC_SOURCE_DIR}/src/ckh.c +${JEMALLOC_SOURCE_DIR}/src/ctl.c +${JEMALLOC_SOURCE_DIR}/src/div.c +${JEMALLOC_SOURCE_DIR}/src/extent.c +${JEMALLOC_SOURCE_DIR}/src/extent_dss.c +${JEMALLOC_SOURCE_DIR}/src/extent_mmap.c +${JEMALLOC_SOURCE_DIR}/src/hash.c +${JEMALLOC_SOURCE_DIR}/src/hook.c +${JEMALLOC_SOURCE_DIR}/src/jemalloc.c +${JEMALLOC_SOURCE_DIR}/src/jemalloc_cpp.cpp +${JEMALLOC_SOURCE_DIR}/src/large.c +${JEMALLOC_SOURCE_DIR}/src/log.c +${JEMALLOC_SOURCE_DIR}/src/malloc_io.c +${JEMALLOC_SOURCE_DIR}/src/mutex.c +${JEMALLOC_SOURCE_DIR}/src/mutex_pool.c +${JEMALLOC_SOURCE_DIR}/src/nstime.c +${JEMALLOC_SOURCE_DIR}/src/pages.c +${JEMALLOC_SOURCE_DIR}/src/prng.c +${JEMALLOC_SOURCE_DIR}/src/prof.c +${JEMALLOC_SOURCE_DIR}/src/rtree.c +${JEMALLOC_SOURCE_DIR}/src/sc.c +${JEMALLOC_SOURCE_DIR}/src/stats.c +${JEMALLOC_SOURCE_DIR}/src/sz.c +${JEMALLOC_SOURCE_DIR}/src/tcache.c +${JEMALLOC_SOURCE_DIR}/src/test_hooks.c +${JEMALLOC_SOURCE_DIR}/src/ticker.c +${JEMALLOC_SOURCE_DIR}/src/tsd.c +${JEMALLOC_SOURCE_DIR}/src/witness.c +) + +if(CMAKE_SYSTEM_NAME MATCHES "Darwin") + list(APPEND SRCS ${JEMALLOC_SOURCE_DIR}/src/zone.c) +endif() + +add_library(jemalloc STATIC ${SRCS}) + +target_include_directories(jemalloc PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/include + ${CMAKE_CURRENT_SOURCE_DIR}/include_linux_x86_64) # jemalloc.h + +target_include_directories(jemalloc PRIVATE + ${JEMALLOC_SOURCE_DIR}/include) + +target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE) + +if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") + target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_DEBUG=1 -DJEMALLOC_PROF=1) + + if (USE_UNWIND) + target_compile_definitions (jemalloc PRIVATE -DJEMALLOC_PROF_LIBUNWIND=1) + target_include_directories (jemalloc BEFORE PRIVATE ${UNWIND_INCLUDE_DIR}) + target_link_libraries (jemalloc PRIVATE ${UNWIND_LIBRARY}) + endif () +endif () diff --git a/contrib/jemalloc-cmake/README b/contrib/jemalloc-cmake/README new file mode 100644 index 00000000000..0af9c4f0e45 --- /dev/null +++ b/contrib/jemalloc-cmake/README @@ -0,0 +1 @@ +It allows to integrate JEMalloc into CMake project. diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc.h new file mode 100644 index 00000000000..d06243c5239 --- /dev/null +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc.h @@ -0,0 +1,16 @@ +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include + +#ifdef __cplusplus +} +#endif + diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h new file mode 100644 index 00000000000..a2ea2dd3533 --- /dev/null +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_rename.h @@ -0,0 +1,29 @@ +/* + * Name mangling for public symbols is controlled by --with-mangling and + * --with-jemalloc-prefix. With default settings the je_ prefix is stripped by + * these macro definitions. + */ +#ifndef JEMALLOC_NO_RENAME +# define je_aligned_alloc aligned_alloc +# define je_calloc calloc +# define je_dallocx dallocx +# define je_free free +# define je_mallctl mallctl +# define je_mallctlbymib mallctlbymib +# define je_mallctlnametomib mallctlnametomib +# define je_malloc malloc +# define je_malloc_conf malloc_conf +# define je_malloc_message malloc_message +# define je_malloc_stats_print malloc_stats_print +# define je_malloc_usable_size malloc_usable_size +# define je_mallocx mallocx +# define je_nallocx nallocx +# define je_posix_memalign posix_memalign +# define je_rallocx rallocx +# define je_realloc realloc +# define je_sallocx sallocx +# define je_sdallocx sdallocx +# define je_xallocx xallocx +# define je_memalign memalign +# define je_valloc valloc +#endif diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/README b/contrib/jemalloc-cmake/include_linux_x86_64/README new file mode 100644 index 00000000000..bf7663bda8d --- /dev/null +++ b/contrib/jemalloc-cmake/include_linux_x86_64/README @@ -0,0 +1,7 @@ +Here are pre-generated files from jemalloc on Linux x86_64. +You can obtain these files by running ./autogen.sh inside jemalloc source directory. + +Added #define GNU_SOURCE +Added JEMALLOC_OVERRIDE___POSIX_MEMALIGN because why not. +Removed JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF because it's non standard. +Removed JEMALLOC_PURGE_MADVISE_FREE because it's available only from Linux 4.5. diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h new file mode 100644 index 00000000000..43936e8eba0 --- /dev/null +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h @@ -0,0 +1,373 @@ +/* include/jemalloc/internal/jemalloc_internal_defs.h. Generated from jemalloc_internal_defs.h.in by configure. */ +#ifndef JEMALLOC_INTERNAL_DEFS_H_ +#define JEMALLOC_INTERNAL_DEFS_H_ + +#ifndef _GNU_SOURCE + #define _GNU_SOURCE +#endif + +/* + * If JEMALLOC_PREFIX is defined via --with-jemalloc-prefix, it will cause all + * public APIs to be prefixed. This makes it possible, with some care, to use + * multiple allocators simultaneously. + */ +/* #undef JEMALLOC_PREFIX */ +/* #undef JEMALLOC_CPREFIX */ + +/* + * Define overrides for non-standard allocator-related functions if they are + * present on the system. + */ +#define JEMALLOC_OVERRIDE___LIBC_CALLOC +#define JEMALLOC_OVERRIDE___LIBC_FREE +#define JEMALLOC_OVERRIDE___LIBC_MALLOC +#define JEMALLOC_OVERRIDE___LIBC_MEMALIGN +#define JEMALLOC_OVERRIDE___LIBC_REALLOC +#define JEMALLOC_OVERRIDE___LIBC_VALLOC +#define JEMALLOC_OVERRIDE___POSIX_MEMALIGN + +/* + * JEMALLOC_PRIVATE_NAMESPACE is used as a prefix for all library-private APIs. + * For shared libraries, symbol visibility mechanisms prevent these symbols + * from being exported, but for static libraries, naming collisions are a real + * possibility. + */ +#define JEMALLOC_PRIVATE_NAMESPACE je_ + +/* + * Hyper-threaded CPUs may need a special instruction inside spin loops in + * order to yield to another virtual CPU. + */ +#define CPU_SPINWAIT __asm__ volatile("pause") +/* 1 if CPU_SPINWAIT is defined, 0 otherwise. */ +#define HAVE_CPU_SPINWAIT 1 + +/* + * Number of significant bits in virtual addresses. This may be less than the + * total number of bits in a pointer, e.g. on x64, for which the uppermost 16 + * bits are the same as bit 47. + */ +#define LG_VADDR 48 + +/* Defined if C11 atomics are available. */ +#define JEMALLOC_C11_ATOMICS 1 + +/* Defined if GCC __atomic atomics are available. */ +#define JEMALLOC_GCC_ATOMIC_ATOMICS 1 + +/* Defined if GCC __sync atomics are available. */ +#define JEMALLOC_GCC_SYNC_ATOMICS 1 + +/* + * Defined if __sync_add_and_fetch(uint32_t *, uint32_t) and + * __sync_sub_and_fetch(uint32_t *, uint32_t) are available, despite + * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 not being defined (which means the + * functions are defined in libgcc instead of being inlines). + */ +/* #undef JE_FORCE_SYNC_COMPARE_AND_SWAP_4 */ + +/* + * Defined if __sync_add_and_fetch(uint64_t *, uint64_t) and + * __sync_sub_and_fetch(uint64_t *, uint64_t) are available, despite + * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 not being defined (which means the + * functions are defined in libgcc instead of being inlines). + */ +/* #undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8 */ + +/* + * Defined if __builtin_clz() and __builtin_clzl() are available. + */ +#define JEMALLOC_HAVE_BUILTIN_CLZ + +/* + * Defined if os_unfair_lock_*() functions are available, as provided by Darwin. + */ +/* #undef JEMALLOC_OS_UNFAIR_LOCK */ + +/* + * Defined if OSSpin*() functions are available, as provided by Darwin, and + * documented in the spinlock(3) manual page. + */ +/* #undef JEMALLOC_OSSPIN */ + +/* Defined if syscall(2) is usable. */ +#define JEMALLOC_USE_SYSCALL + +/* + * Defined if secure_getenv(3) is available. + */ +// Don't want dependency on newer GLIBC +//#define JEMALLOC_HAVE_SECURE_GETENV + +/* + * Defined if issetugid(2) is available. + */ +/* #undef JEMALLOC_HAVE_ISSETUGID */ + +/* Defined if pthread_atfork(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_ATFORK + +/* Defined if pthread_setname_np(3) is available. */ +#define JEMALLOC_HAVE_PTHREAD_SETNAME_NP + +/* + * Defined if clock_gettime(CLOCK_MONOTONIC_COARSE, ...) is available. + */ +#define JEMALLOC_HAVE_CLOCK_MONOTONIC_COARSE 1 + +/* + * Defined if clock_gettime(CLOCK_MONOTONIC, ...) is available. + */ +#define JEMALLOC_HAVE_CLOCK_MONOTONIC 1 + +/* + * Defined if mach_absolute_time() is available. + */ +/* #undef JEMALLOC_HAVE_MACH_ABSOLUTE_TIME */ + +/* + * Defined if _malloc_thread_cleanup() exists. At least in the case of + * FreeBSD, pthread_key_create() allocates, which if used during malloc + * bootstrapping will cause recursion into the pthreads library. Therefore, if + * _malloc_thread_cleanup() exists, use it as the basis for thread cleanup in + * malloc_tsd. + */ +/* #undef JEMALLOC_MALLOC_THREAD_CLEANUP */ + +/* + * Defined if threaded initialization is known to be safe on this platform. + * Among other things, it must be possible to initialize a mutex without + * triggering allocation in order for threaded allocation to be safe. + */ +#define JEMALLOC_THREADED_INIT + +/* + * Defined if the pthreads implementation defines + * _pthread_mutex_init_calloc_cb(), in which case the function is used in order + * to avoid recursive allocation during mutex initialization. + */ +/* #undef JEMALLOC_MUTEX_INIT_CB */ + +/* Non-empty if the tls_model attribute is supported. */ +#define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) + +/* + * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables + * inline functions. + */ +/* #undef JEMALLOC_DEBUG */ + +/* JEMALLOC_STATS enables statistics calculation. */ +#define JEMALLOC_STATS + +/* JEMALLOC_PROF enables allocation profiling. */ +/* #undef JEMALLOC_PROF */ + +/* Use libunwind for profile backtracing if defined. */ +/* #undef JEMALLOC_PROF_LIBUNWIND */ + +/* Use libgcc for profile backtracing if defined. */ +/* #undef JEMALLOC_PROF_LIBGCC */ + +/* Use gcc intrinsics for profile backtracing if defined. */ +/* #undef JEMALLOC_PROF_GCC */ + +/* + * JEMALLOC_DSS enables use of sbrk(2) to allocate extents from the data storage + * segment (DSS). + */ +#define JEMALLOC_DSS + +/* Support memory filling (junk/zero). */ +#define JEMALLOC_FILL + +/* Support utrace(2)-based tracing. */ +/* #undef JEMALLOC_UTRACE */ + +/* Support optional abort() on OOM. */ +/* #undef JEMALLOC_XMALLOC */ + +/* Support lazy locking (avoid locking unless a second thread is launched). */ +/* #undef JEMALLOC_LAZY_LOCK */ + +/* + * Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size + * classes). + */ +/* #undef LG_QUANTUM */ + +/* One page is 2^LG_PAGE bytes. */ +#define LG_PAGE 12 + +/* + * One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the + * system does not explicitly support huge pages; system calls that require + * explicit huge page support are separately configured. + */ +#define LG_HUGEPAGE 21 + +/* + * If defined, adjacent virtual memory mappings with identical attributes + * automatically coalesce, and they fragment when changes are made to subranges. + * This is the normal order of things for mmap()/munmap(), but on Windows + * VirtualAlloc()/VirtualFree() operations must be precisely matched, i.e. + * mappings do *not* coalesce/fragment. + */ +#define JEMALLOC_MAPS_COALESCE + +/* + * If defined, retain memory for later reuse by default rather than using e.g. + * munmap() to unmap freed extents. This is enabled on 64-bit Linux because + * common sequences of mmap()/munmap() calls will cause virtual memory map + * holes. + */ +#define JEMALLOC_RETAIN + +/* TLS is used to map arenas and magazine caches to threads. */ +#define JEMALLOC_TLS + +/* + * Used to mark unreachable code to quiet "end of non-void" compiler warnings. + * Don't use this directly; instead use unreachable() from util.h + */ +#define JEMALLOC_INTERNAL_UNREACHABLE __builtin_unreachable + +/* + * ffs*() functions to use for bitmapping. Don't use these directly; instead, + * use ffs_*() from util.h. + */ +#define JEMALLOC_INTERNAL_FFSLL __builtin_ffsll +#define JEMALLOC_INTERNAL_FFSL __builtin_ffsl +#define JEMALLOC_INTERNAL_FFS __builtin_ffs + +/* + * If defined, explicitly attempt to more uniformly distribute large allocation + * pointer alignments across all cache indices. + */ +#define JEMALLOC_CACHE_OBLIVIOUS + +/* + * If defined, enable logging facilities. We make this a configure option to + * avoid taking extra branches everywhere. + */ +/* #undef JEMALLOC_LOG */ + +/* + * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings. + */ +/* #undef JEMALLOC_ZONE */ + +/* + * Methods for determining whether the OS overcommits. + * JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY: Linux's + * /proc/sys/vm.overcommit_memory file. + * JEMALLOC_SYSCTL_VM_OVERCOMMIT: FreeBSD's vm.overcommit sysctl. + */ +/* #undef JEMALLOC_SYSCTL_VM_OVERCOMMIT */ +#define JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY + +/* Defined if madvise(2) is available. */ +#define JEMALLOC_HAVE_MADVISE + +/* + * Defined if transparent huge pages are supported via the MADV_[NO]HUGEPAGE + * arguments to madvise(2). + */ +#define JEMALLOC_HAVE_MADVISE_HUGE + +/* + * Methods for purging unused pages differ between operating systems. + * + * madvise(..., MADV_FREE) : This marks pages as being unused, such that they + * will be discarded rather than swapped out. + * madvise(..., MADV_DONTNEED) : If JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS is + * defined, this immediately discards pages, + * such that new pages will be demand-zeroed if + * the address region is later touched; + * otherwise this behaves similarly to + * MADV_FREE, though typically with higher + * system overhead. + */ +//#define JEMALLOC_PURGE_MADVISE_FREE +#define JEMALLOC_PURGE_MADVISE_DONTNEED +#define JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS + +/* Defined if madvise(2) is available but MADV_FREE is not (x86 Linux only). */ +/* #undef JEMALLOC_DEFINE_MADVISE_FREE */ + +/* + * Defined if MADV_DO[NT]DUMP is supported as an argument to madvise. + */ +#define JEMALLOC_MADVISE_DONTDUMP + +/* + * Defined if transparent huge pages (THPs) are supported via the + * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. + */ +/* #undef JEMALLOC_THP */ + +/* Define if operating system has alloca.h header. */ +#define JEMALLOC_HAS_ALLOCA_H 1 + +/* C99 restrict keyword supported. */ +#define JEMALLOC_HAS_RESTRICT 1 + +/* For use by hash code. */ +/* #undef JEMALLOC_BIG_ENDIAN */ + +/* sizeof(int) == 2^LG_SIZEOF_INT. */ +#define LG_SIZEOF_INT 2 + +/* sizeof(long) == 2^LG_SIZEOF_LONG. */ +#define LG_SIZEOF_LONG 3 + +/* sizeof(long long) == 2^LG_SIZEOF_LONG_LONG. */ +#define LG_SIZEOF_LONG_LONG 3 + +/* sizeof(intmax_t) == 2^LG_SIZEOF_INTMAX_T. */ +#define LG_SIZEOF_INTMAX_T 3 + +/* glibc malloc hooks (__malloc_hook, __realloc_hook, __free_hook). */ +#define JEMALLOC_GLIBC_MALLOC_HOOK + +/* glibc memalign hook. */ +#define JEMALLOC_GLIBC_MEMALIGN_HOOK + +/* pthread support */ +#define JEMALLOC_HAVE_PTHREAD + +/* dlsym() support */ +#define JEMALLOC_HAVE_DLSYM + +/* Adaptive mutex support in pthreads. */ +#define JEMALLOC_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP + +/* GNU specific sched_getcpu support */ +#define JEMALLOC_HAVE_SCHED_GETCPU + +/* GNU specific sched_setaffinity support */ +#define JEMALLOC_HAVE_SCHED_SETAFFINITY + +/* + * If defined, all the features necessary for background threads are present. + */ +#define JEMALLOC_BACKGROUND_THREAD 1 + +/* + * If defined, jemalloc symbols are not exported (doesn't work when + * JEMALLOC_PREFIX is not defined). + */ +/* #undef JEMALLOC_EXPORT */ + +/* config.malloc_conf options string. */ +#define JEMALLOC_CONFIG_MALLOC_CONF "" + +/* If defined, jemalloc takes the malloc/free/etc. symbol names. */ +#define JEMALLOC_IS_MALLOC 1 + +/* + * Defined if strerror_r returns char * if _GNU_SOURCE is defined. + */ +#define JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE + +#endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_preamble.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_preamble.h new file mode 100644 index 00000000000..c150785fb4a --- /dev/null +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_preamble.h @@ -0,0 +1,194 @@ +#ifndef JEMALLOC_PREAMBLE_H +#define JEMALLOC_PREAMBLE_H + +#include "jemalloc_internal_defs.h" +#include "jemalloc/internal/jemalloc_internal_decls.h" + +#ifdef JEMALLOC_UTRACE +#include +#endif + +#define JEMALLOC_NO_DEMANGLE +#ifdef JEMALLOC_JET +# undef JEMALLOC_IS_MALLOC +# define JEMALLOC_N(n) jet_##n +# include "jemalloc/internal/public_namespace.h" +# define JEMALLOC_NO_RENAME +# include "jemalloc/jemalloc.h" +# undef JEMALLOC_NO_RENAME +#else +# define JEMALLOC_N(n) je_##n +# include "jemalloc/jemalloc.h" +#endif + +#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN)) +#include +#endif + +#ifdef JEMALLOC_ZONE +#include +#include +#include +#endif + +#include "jemalloc/internal/jemalloc_internal_macros.h" + +/* + * Note that the ordering matters here; the hook itself is name-mangled. We + * want the inclusion of hooks to happen early, so that we hook as much as + * possible. + */ +#ifndef JEMALLOC_NO_PRIVATE_NAMESPACE +# ifndef JEMALLOC_JET +# include "jemalloc/internal/private_namespace.h" +# else +# include "jemalloc/internal/private_namespace_jet.h" +# endif +#endif +#include "jemalloc/internal/test_hooks.h" + +#ifdef JEMALLOC_DEFINE_MADVISE_FREE +# define JEMALLOC_MADV_FREE 8 +#endif + +static const bool config_debug = +#ifdef JEMALLOC_DEBUG + true +#else + false +#endif + ; +static const bool have_dss = +#ifdef JEMALLOC_DSS + true +#else + false +#endif + ; +static const bool have_madvise_huge = +#ifdef JEMALLOC_HAVE_MADVISE_HUGE + true +#else + false +#endif + ; +static const bool config_fill = +#ifdef JEMALLOC_FILL + true +#else + false +#endif + ; +static const bool config_lazy_lock = +#ifdef JEMALLOC_LAZY_LOCK + true +#else + false +#endif + ; +static const char * const config_malloc_conf = JEMALLOC_CONFIG_MALLOC_CONF; +static const bool config_prof = +#ifdef JEMALLOC_PROF + true +#else + false +#endif + ; +static const bool config_prof_libgcc = +#ifdef JEMALLOC_PROF_LIBGCC + true +#else + false +#endif + ; +static const bool config_prof_libunwind = +#ifdef JEMALLOC_PROF_LIBUNWIND + true +#else + false +#endif + ; +static const bool maps_coalesce = +#ifdef JEMALLOC_MAPS_COALESCE + true +#else + false +#endif + ; +static const bool config_stats = +#ifdef JEMALLOC_STATS + true +#else + false +#endif + ; +static const bool config_tls = +#ifdef JEMALLOC_TLS + true +#else + false +#endif + ; +static const bool config_utrace = +#ifdef JEMALLOC_UTRACE + true +#else + false +#endif + ; +static const bool config_xmalloc = +#ifdef JEMALLOC_XMALLOC + true +#else + false +#endif + ; +static const bool config_cache_oblivious = +#ifdef JEMALLOC_CACHE_OBLIVIOUS + true +#else + false +#endif + ; +/* + * Undocumented, for jemalloc development use only at the moment. See the note + * in jemalloc/internal/log.h. + */ +static const bool config_log = +#ifdef JEMALLOC_LOG + true +#else + false +#endif + ; +#ifdef JEMALLOC_HAVE_SCHED_GETCPU +/* Currently percpu_arena depends on sched_getcpu. */ +#define JEMALLOC_PERCPU_ARENA +#endif +static const bool have_percpu_arena = +#ifdef JEMALLOC_PERCPU_ARENA + true +#else + false +#endif + ; +/* + * Undocumented, and not recommended; the application should take full + * responsibility for tracking provenance. + */ +static const bool force_ivsalloc = +#ifdef JEMALLOC_FORCE_IVSALLOC + true +#else + false +#endif + ; +static const bool have_background_thread = +#ifdef JEMALLOC_BACKGROUND_THREAD + true +#else + false +#endif + ; + +#endif /* JEMALLOC_PREAMBLE_H */ diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_defs.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_defs.h new file mode 100644 index 00000000000..d1389237a77 --- /dev/null +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_defs.h @@ -0,0 +1,43 @@ +/* include/jemalloc/jemalloc_defs.h. Generated from jemalloc_defs.h.in by configure. */ +/* Defined if __attribute__((...)) syntax is supported. */ +#define JEMALLOC_HAVE_ATTR + +/* Defined if alloc_size attribute is supported. */ +#define JEMALLOC_HAVE_ATTR_ALLOC_SIZE + +/* Defined if format(printf, ...) attribute is supported. */ +#define JEMALLOC_HAVE_ATTR_FORMAT_PRINTF + +/* + * Define overrides for non-standard allocator-related functions if they are + * present on the system. + */ +#define JEMALLOC_OVERRIDE_MEMALIGN +#define JEMALLOC_OVERRIDE_VALLOC + +/* + * At least Linux omits the "const" in: + * + * size_t malloc_usable_size(const void *ptr); + * + * Match the operating system's prototype. + */ +#define JEMALLOC_USABLE_SIZE_CONST + +/* + * If defined, specify throw() for the public function prototypes when compiling + * with C++. The only justification for this is to match the prototypes that + * glibc defines. + */ +#define JEMALLOC_USE_CXX_THROW + +#ifdef _MSC_VER +# ifdef _WIN64 +# define LG_SIZEOF_PTR_WIN 3 +# else +# define LG_SIZEOF_PTR_WIN 2 +# endif +#endif + +/* sizeof(void *) == 2^LG_SIZEOF_PTR. */ +#define LG_SIZEOF_PTR 3 diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_macros.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_macros.h new file mode 100644 index 00000000000..7432f1cda53 --- /dev/null +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_macros.h @@ -0,0 +1,122 @@ +#include +#include +#include +#include +#include + +#define JEMALLOC_VERSION "5.1.0-56-g41b7372eadee941b9164751b8d4963f915d3ceae" +#define JEMALLOC_VERSION_MAJOR 5 +#define JEMALLOC_VERSION_MINOR 1 +#define JEMALLOC_VERSION_BUGFIX 0 +#define JEMALLOC_VERSION_NREV 56 +#define JEMALLOC_VERSION_GID "41b7372eadee941b9164751b8d4963f915d3ceae" + +#define MALLOCX_LG_ALIGN(la) ((int)(la)) +#if LG_SIZEOF_PTR == 2 +# define MALLOCX_ALIGN(a) ((int)(ffs((int)(a))-1)) +#else +# define MALLOCX_ALIGN(a) \ + ((int)(((size_t)(a) < (size_t)INT_MAX) ? ffs((int)(a))-1 : \ + ffs((int)(((size_t)(a))>>32))+31)) +#endif +#define MALLOCX_ZERO ((int)0x40) +/* + * Bias tcache index bits so that 0 encodes "automatic tcache management", and 1 + * encodes MALLOCX_TCACHE_NONE. + */ +#define MALLOCX_TCACHE(tc) ((int)(((tc)+2) << 8)) +#define MALLOCX_TCACHE_NONE MALLOCX_TCACHE(-1) +/* + * Bias arena index bits so that 0 encodes "use an automatically chosen arena". + */ +#define MALLOCX_ARENA(a) ((((int)(a))+1) << 20) + +/* + * Use as arena index in "arena..{purge,decay,dss}" and + * "stats.arenas..*" mallctl interfaces to select all arenas. This + * definition is intentionally specified in raw decimal format to support + * cpp-based string concatenation, e.g. + * + * #define STRINGIFY_HELPER(x) #x + * #define STRINGIFY(x) STRINGIFY_HELPER(x) + * + * mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", NULL, NULL, NULL, + * 0); + */ +#define MALLCTL_ARENAS_ALL 4096 +/* + * Use as arena index in "stats.arenas..*" mallctl interfaces to select + * destroyed arenas. + */ +#define MALLCTL_ARENAS_DESTROYED 4097 + +#if defined(__cplusplus) && defined(JEMALLOC_USE_CXX_THROW) +# define JEMALLOC_CXX_THROW throw() +#else +# define JEMALLOC_CXX_THROW +#endif + +#if defined(_MSC_VER) +# define JEMALLOC_ATTR(s) +# define JEMALLOC_ALIGNED(s) __declspec(align(s)) +# define JEMALLOC_ALLOC_SIZE(s) +# define JEMALLOC_ALLOC_SIZE2(s1, s2) +# ifndef JEMALLOC_EXPORT +# ifdef DLLEXPORT +# define JEMALLOC_EXPORT __declspec(dllexport) +# else +# define JEMALLOC_EXPORT __declspec(dllimport) +# endif +# endif +# define JEMALLOC_FORMAT_PRINTF(s, i) +# define JEMALLOC_NOINLINE __declspec(noinline) +# ifdef __cplusplus +# define JEMALLOC_NOTHROW __declspec(nothrow) +# else +# define JEMALLOC_NOTHROW +# endif +# define JEMALLOC_SECTION(s) __declspec(allocate(s)) +# define JEMALLOC_RESTRICT_RETURN __declspec(restrict) +# if _MSC_VER >= 1900 && !defined(__EDG__) +# define JEMALLOC_ALLOCATOR __declspec(allocator) +# else +# define JEMALLOC_ALLOCATOR +# endif +#elif defined(JEMALLOC_HAVE_ATTR) +# define JEMALLOC_ATTR(s) __attribute__((s)) +# define JEMALLOC_ALIGNED(s) JEMALLOC_ATTR(aligned(s)) +# ifdef JEMALLOC_HAVE_ATTR_ALLOC_SIZE +# define JEMALLOC_ALLOC_SIZE(s) JEMALLOC_ATTR(alloc_size(s)) +# define JEMALLOC_ALLOC_SIZE2(s1, s2) JEMALLOC_ATTR(alloc_size(s1, s2)) +# else +# define JEMALLOC_ALLOC_SIZE(s) +# define JEMALLOC_ALLOC_SIZE2(s1, s2) +# endif +# ifndef JEMALLOC_EXPORT +# define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default")) +# endif +# ifdef JEMALLOC_HAVE_ATTR_FORMAT_GNU_PRINTF +# define JEMALLOC_FORMAT_PRINTF(s, i) JEMALLOC_ATTR(format(gnu_printf, s, i)) +# elif defined(JEMALLOC_HAVE_ATTR_FORMAT_PRINTF) +# define JEMALLOC_FORMAT_PRINTF(s, i) JEMALLOC_ATTR(format(printf, s, i)) +# else +# define JEMALLOC_FORMAT_PRINTF(s, i) +# endif +# define JEMALLOC_NOINLINE JEMALLOC_ATTR(noinline) +# define JEMALLOC_NOTHROW JEMALLOC_ATTR(nothrow) +# define JEMALLOC_SECTION(s) JEMALLOC_ATTR(section(s)) +# define JEMALLOC_RESTRICT_RETURN +# define JEMALLOC_ALLOCATOR +#else +# define JEMALLOC_ATTR(s) +# define JEMALLOC_ALIGNED(s) +# define JEMALLOC_ALLOC_SIZE(s) +# define JEMALLOC_ALLOC_SIZE2(s1, s2) +# define JEMALLOC_EXPORT +# define JEMALLOC_FORMAT_PRINTF(s, i) +# define JEMALLOC_NOINLINE +# define JEMALLOC_NOTHROW +# define JEMALLOC_SECTION(s) +# define JEMALLOC_RESTRICT_RETURN +# define JEMALLOC_ALLOCATOR +#endif diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_protos.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_protos.h new file mode 100644 index 00000000000..ff025e30fa7 --- /dev/null +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_protos.h @@ -0,0 +1,66 @@ +/* + * The je_ prefix on the following public symbol declarations is an artifact + * of namespace management, and should be omitted in application code unless + * JEMALLOC_NO_DEMANGLE is defined (see jemalloc_mangle.h). + */ +extern JEMALLOC_EXPORT const char *je_malloc_conf; +extern JEMALLOC_EXPORT void (*je_malloc_message)(void *cbopaque, + const char *s); + +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *je_malloc(size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *je_calloc(size_t num, size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE2(1, 2); +JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_posix_memalign(void **memptr, + size_t alignment, size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(nonnull(1)); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *je_aligned_alloc(size_t alignment, + size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc) + JEMALLOC_ALLOC_SIZE(2); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *je_realloc(void *ptr, size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ALLOC_SIZE(2); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_free(void *ptr) + JEMALLOC_CXX_THROW; + +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *je_mallocx(size_t size, int flags) + JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1); +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *je_rallocx(void *ptr, size_t size, + int flags) JEMALLOC_ALLOC_SIZE(2); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_xallocx(void *ptr, size_t size, + size_t extra, int flags); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_sallocx(const void *ptr, + int flags) JEMALLOC_ATTR(pure); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_dallocx(void *ptr, int flags); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_sdallocx(void *ptr, size_t size, + int flags); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_nallocx(size_t size, int flags) + JEMALLOC_ATTR(pure); + +JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_mallctl(const char *name, + void *oldp, size_t *oldlenp, void *newp, size_t newlen); +JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_mallctlnametomib(const char *name, + size_t *mibp, size_t *miblenp); +JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_mallctlbymib(const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen); +JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_malloc_stats_print( + void (*write_cb)(void *, const char *), void *je_cbopaque, + const char *opts); +JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW je_malloc_usable_size( + JEMALLOC_USABLE_SIZE_CONST void *ptr) JEMALLOC_CXX_THROW; + +#ifdef JEMALLOC_OVERRIDE_MEMALIGN +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *je_memalign(size_t alignment, size_t size) + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc); +#endif + +#ifdef JEMALLOC_OVERRIDE_VALLOC +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN + void JEMALLOC_NOTHROW *je_valloc(size_t size) JEMALLOC_CXX_THROW + JEMALLOC_ATTR(malloc); +#endif diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_typedefs.h b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_typedefs.h new file mode 100644 index 00000000000..1a58874306e --- /dev/null +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/jemalloc_typedefs.h @@ -0,0 +1,77 @@ +typedef struct extent_hooks_s extent_hooks_t; + +/* + * void * + * extent_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size, + * size_t alignment, bool *zero, bool *commit, unsigned arena_ind); + */ +typedef void *(extent_alloc_t)(extent_hooks_t *, void *, size_t, size_t, bool *, + bool *, unsigned); + +/* + * bool + * extent_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size, + * bool committed, unsigned arena_ind); + */ +typedef bool (extent_dalloc_t)(extent_hooks_t *, void *, size_t, bool, + unsigned); + +/* + * void + * extent_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size, + * bool committed, unsigned arena_ind); + */ +typedef void (extent_destroy_t)(extent_hooks_t *, void *, size_t, bool, + unsigned); + +/* + * bool + * extent_commit(extent_hooks_t *extent_hooks, void *addr, size_t size, + * size_t offset, size_t length, unsigned arena_ind); + */ +typedef bool (extent_commit_t)(extent_hooks_t *, void *, size_t, size_t, size_t, + unsigned); + +/* + * bool + * extent_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size, + * size_t offset, size_t length, unsigned arena_ind); + */ +typedef bool (extent_decommit_t)(extent_hooks_t *, void *, size_t, size_t, + size_t, unsigned); + +/* + * bool + * extent_purge(extent_hooks_t *extent_hooks, void *addr, size_t size, + * size_t offset, size_t length, unsigned arena_ind); + */ +typedef bool (extent_purge_t)(extent_hooks_t *, void *, size_t, size_t, size_t, + unsigned); + +/* + * bool + * extent_split(extent_hooks_t *extent_hooks, void *addr, size_t size, + * size_t size_a, size_t size_b, bool committed, unsigned arena_ind); + */ +typedef bool (extent_split_t)(extent_hooks_t *, void *, size_t, size_t, size_t, + bool, unsigned); + +/* + * bool + * extent_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, + * void *addr_b, size_t size_b, bool committed, unsigned arena_ind); + */ +typedef bool (extent_merge_t)(extent_hooks_t *, void *, size_t, void *, size_t, + bool, unsigned); + +struct extent_hooks_s { + extent_alloc_t *alloc; + extent_dalloc_t *dalloc; + extent_destroy_t *destroy; + extent_commit_t *commit; + extent_decommit_t *decommit; + extent_purge_t *purge_lazy; + extent_purge_t *purge_forced; + extent_split_t *split; + extent_merge_t *merge; +}; diff --git a/contrib/libcityhash/CMakeLists.txt b/contrib/libcityhash/CMakeLists.txt deleted file mode 100644 index eafa6f2025f..00000000000 --- a/contrib/libcityhash/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -add_library(cityhash - src/city.cc - - include/citycrc.h - include/city.h - src/config.h) - -target_include_directories (cityhash BEFORE PUBLIC include) -target_include_directories (cityhash PRIVATE src) diff --git a/contrib/libcpuid/CMakeLists.txt b/contrib/libcpuid/CMakeLists.txt index 4c257b20de8..cd3e7fa06fe 100644 --- a/contrib/libcpuid/CMakeLists.txt +++ b/contrib/libcpuid/CMakeLists.txt @@ -1,20 +1,20 @@ add_library(cpuid -include/cpuid/asm-bits.c -include/cpuid/cpuid_main.c -include/cpuid/libcpuid_util.c -include/cpuid/rdtsc.c -include/cpuid/recog_amd.c -include/cpuid/recog_intel.c +include/libcpuid/asm-bits.c +include/libcpuid/cpuid_main.c +include/libcpuid/libcpuid_util.c +include/libcpuid/rdtsc.c +include/libcpuid/recog_amd.c +include/libcpuid/recog_intel.c -include/cpuid/asm-bits.h -include/cpuid/config.h -include/cpuid/libcpuid_constants.h -include/cpuid/libcpuid.h -include/cpuid/libcpuid_types.h -include/cpuid/libcpuid_util.h -include/cpuid/rdtsc.h -include/cpuid/recog_amd.h -include/cpuid/recog_intel.h +include/libcpuid/asm-bits.h +include/libcpuid/config.h +include/libcpuid/libcpuid_constants.h +include/libcpuid/libcpuid.h +include/libcpuid/libcpuid_types.h +include/libcpuid/libcpuid_util.h +include/libcpuid/rdtsc.h +include/libcpuid/recog_amd.h +include/libcpuid/recog_intel.h ) -target_include_directories (cpuid PUBLIC include) +target_include_directories (cpuid SYSTEM PUBLIC include) diff --git a/contrib/libcpuid/include/libcpuid/amd_code_t.h b/contrib/libcpuid/include/libcpuid/amd_code_t.h new file mode 100644 index 00000000000..2472a3d61d5 --- /dev/null +++ b/contrib/libcpuid/include/libcpuid/amd_code_t.h @@ -0,0 +1,65 @@ +/* + * Copyright 2016 Veselin Georgiev, + * anrieffNOSPAM @ mgail_DOT.com (convert to gmail) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * This file contains a list of internal codes we use in detection. It is + * of no external use and isn't a complete list of AMD products. + */ + CODE2(OPTERON_GENERIC, 1000), + CODE(OPTERON_800), + CODE(ATHLON_XP), + CODE(ATHLON_XP_M), + CODE(ATHLON_XP_M_LV), + CODE(ATHLON), + CODE(ATHLON_MP), + CODE(MOBILE_ATHLON64), + CODE(ATHLON_FX), + CODE(DURON), + CODE(DURON_MP), + CODE(MOBILE_DURON), + CODE(MOBILE_SEMPRON), + CODE(OPTERON_SINGLE), + CODE(OPTERON_DUALCORE), + CODE(OPTERON_800_DUALCORE), + CODE(MOBILE_TURION), + CODE(ATHLON_64), + CODE(ATHLON_64_FX), + CODE(TURION_64), + CODE(TURION_X2), + CODE(SEMPRON), + CODE(M_SEMPRON), + CODE(SEMPRON_DUALCORE), + CODE(PHENOM), + CODE(PHENOM2), + CODE(ATHLON_64_X2), + CODE(ATHLON_64_X3), + CODE(ATHLON_64_X4), + CODE(FUSION_C), + CODE(FUSION_E), + CODE(FUSION_EA), + CODE(FUSION_Z), + CODE(FUSION_A), + diff --git a/contrib/libcpuid/include/cpuid/asm-bits.c b/contrib/libcpuid/include/libcpuid/asm-bits.c similarity index 95% rename from contrib/libcpuid/include/cpuid/asm-bits.c rename to contrib/libcpuid/include/libcpuid/asm-bits.c index ca936f2abf5..b8e32284f57 100644 --- a/contrib/libcpuid/include/cpuid/asm-bits.c +++ b/contrib/libcpuid/include/libcpuid/asm-bits.c @@ -75,13 +75,13 @@ int cpuid_exists_by_eflags(void) #endif /* PLATFORM_X86 */ } +#ifdef INLINE_ASM_SUPPORTED /* * with MSVC/AMD64, the exec_cpuid() and cpu_rdtsc() functions * are implemented in separate .asm files. Otherwise, use inline assembly */ void exec_cpuid(uint32_t *regs) { -#ifdef INLINE_ASM_SUPPORTED #ifdef COMPILER_GCC # ifdef PLATFORM_X64 __asm __volatile( @@ -166,8 +166,8 @@ void exec_cpuid(uint32_t *regs) # error "Unsupported compiler" # endif /* COMPILER_MICROSOFT */ #endif -#endif /* INLINE_ASSEMBLY_SUPPORTED */ } +#endif /* INLINE_ASSEMBLY_SUPPORTED */ #ifdef INLINE_ASM_SUPPORTED void cpu_rdtsc(uint64_t* result) @@ -214,7 +214,8 @@ void busy_sse_loop(int cycles) " xorps %%xmm6, %%xmm6\n" " xorps %%xmm7, %%xmm7\n" XALIGN - ".bsLoop:\n" + /* ".bsLoop:\n" */ + "1:\n" // 0: " addps %%xmm1, %%xmm0\n" " addps %%xmm2, %%xmm1\n" @@ -505,7 +506,8 @@ void busy_sse_loop(int cycles) " addps %%xmm0, %%xmm7\n" " dec %%eax\n" - " jnz .bsLoop\n" + /* "jnz .bsLoop\n" */ + " jnz 1b\n" ::"a"(cycles) ); #else diff --git a/contrib/libcpuid/include/cpuid/asm-bits.h b/contrib/libcpuid/include/libcpuid/asm-bits.h similarity index 100% rename from contrib/libcpuid/include/cpuid/asm-bits.h rename to contrib/libcpuid/include/libcpuid/asm-bits.h diff --git a/contrib/libcpuid/include/cpuid/config.h b/contrib/libcpuid/include/libcpuid/config.h similarity index 57% rename from contrib/libcpuid/include/cpuid/config.h rename to contrib/libcpuid/include/libcpuid/config.h index 29b513c4ac1..2326cfeede5 100644 --- a/contrib/libcpuid/include/cpuid/config.h +++ b/contrib/libcpuid/include/libcpuid/config.h @@ -1,2 +1,2 @@ /* Version number of package */ -#define VERSION "0.2.1" +#define VERSION "0.4.0" diff --git a/contrib/libcpuid/include/cpuid/cpuid_main.c b/contrib/libcpuid/include/libcpuid/cpuid_main.c similarity index 81% rename from contrib/libcpuid/include/cpuid/cpuid_main.c rename to contrib/libcpuid/include/libcpuid/cpuid_main.c index 737aa706a37..02a7cb7ad50 100644 --- a/contrib/libcpuid/include/cpuid/cpuid_main.c +++ b/contrib/libcpuid/include/libcpuid/cpuid_main.c @@ -24,11 +24,14 @@ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "libcpuid.h" +#include "libcpuid_internal.h" #include "recog_intel.h" #include "recog_amd.h" #include "asm-bits.h" #include "libcpuid_util.h" +//#ifdef HAVE_CONFIG_H // CLICKHOUSE PATCH #include "config.h" +//#endif // CLICKHOUSE PATCH #include #include #include @@ -51,9 +54,9 @@ static void raw_data_t_constructor(struct cpu_raw_data_t* raw) static void cpu_id_t_constructor(struct cpu_id_t* id) { memset(id, 0, sizeof(struct cpu_id_t)); - id->l1_data_cache = id->l1_instruction_cache = id->l2_cache = id->l3_cache = -1; - id->l1_assoc = id->l2_assoc = id->l3_assoc = -1; - id->l1_cacheline = id->l2_cacheline = id->l3_cacheline = -1; + id->l1_data_cache = id->l1_instruction_cache = id->l2_cache = id->l3_cache = id->l4_cache = -1; + id->l1_assoc = id->l2_assoc = id->l3_assoc = id->l4_assoc = -1; + id->l1_cacheline = id->l2_cacheline = id->l3_cacheline = id->l4_cacheline = -1; id->sse_size = -1; } @@ -181,14 +184,26 @@ static void load_features_common(struct cpu_raw_data_t* raw, struct cpu_id_t* da }; const struct feature_map_t matchtable_ecx1[] = { { 0, CPU_FEATURE_PNI }, + { 1, CPU_FEATURE_PCLMUL }, { 3, CPU_FEATURE_MONITOR }, { 9, CPU_FEATURE_SSSE3 }, { 12, CPU_FEATURE_FMA3 }, { 13, CPU_FEATURE_CX16 }, { 19, CPU_FEATURE_SSE4_1 }, - { 21, CPU_FEATURE_X2APIC }, + { 20, CPU_FEATURE_SSE4_2 }, + { 22, CPU_FEATURE_MOVBE }, { 23, CPU_FEATURE_POPCNT }, + { 25, CPU_FEATURE_AES }, + { 26, CPU_FEATURE_XSAVE }, + { 27, CPU_FEATURE_OSXSAVE }, + { 28, CPU_FEATURE_AVX }, { 29, CPU_FEATURE_F16C }, + { 30, CPU_FEATURE_RDRAND }, + }; + const struct feature_map_t matchtable_ebx7[] = { + { 3, CPU_FEATURE_BMI1 }, + { 5, CPU_FEATURE_AVX2 }, + { 8, CPU_FEATURE_BMI2 }, }; const struct feature_map_t matchtable_edx81[] = { { 11, CPU_FEATURE_SYSCALL }, @@ -205,6 +220,9 @@ static void load_features_common(struct cpu_raw_data_t* raw, struct cpu_id_t* da match_features(matchtable_edx1, COUNT_OF(matchtable_edx1), raw->basic_cpuid[1][3], data); match_features(matchtable_ecx1, COUNT_OF(matchtable_ecx1), raw->basic_cpuid[1][2], data); } + if (raw->basic_cpuid[0][0] >= 7) { + match_features(matchtable_ebx7, COUNT_OF(matchtable_ebx7), raw->basic_cpuid[7][1], data); + } if (raw->ext_cpuid[0][0] >= 0x80000001) { match_features(matchtable_edx81, COUNT_OF(matchtable_edx81), raw->ext_cpuid[1][3], data); match_features(matchtable_ecx81, COUNT_OF(matchtable_ecx81), raw->ext_cpuid[1][2], data); @@ -229,10 +247,10 @@ static void load_features_common(struct cpu_raw_data_t* raw, struct cpu_id_t* da } } -static int cpuid_basic_identify(struct cpu_raw_data_t* raw, struct cpu_id_t* data) +static cpu_vendor_t cpuid_vendor_identify(const uint32_t *raw_vendor, char *vendor_str) { - int i, j, basic, xmodel, xfamily, ext; - char brandstr[64] = {0}; + int i; + cpu_vendor_t vendor = VENDOR_UNKNOWN; const struct { cpu_vendor_t vendor; char match[16]; } matchtable[NUM_CPU_VENDORS] = { /* source: http://www.sandpile.org/ia32/cpuid.htm */ @@ -247,18 +265,27 @@ static int cpuid_basic_identify(struct cpu_raw_data_t* raw, struct cpu_id_t* dat { VENDOR_SIS , "SiS SiS SiS " }, { VENDOR_NSC , "Geode by NSC" }, }; - - memcpy(data->vendor_str + 0, &raw->basic_cpuid[0][1], 4); - memcpy(data->vendor_str + 4, &raw->basic_cpuid[0][3], 4); - memcpy(data->vendor_str + 8, &raw->basic_cpuid[0][2], 4); - data->vendor_str[12] = 0; + + memcpy(vendor_str + 0, &raw_vendor[1], 4); + memcpy(vendor_str + 4, &raw_vendor[3], 4); + memcpy(vendor_str + 8, &raw_vendor[2], 4); + vendor_str[12] = 0; + /* Determine vendor: */ - data->vendor = VENDOR_UNKNOWN; for (i = 0; i < NUM_CPU_VENDORS; i++) - if (!strcmp(data->vendor_str, matchtable[i].match)) { - data->vendor = matchtable[i].vendor; + if (!strcmp(vendor_str, matchtable[i].match)) { + vendor = matchtable[i].vendor; break; } + return vendor; +} + +static int cpuid_basic_identify(struct cpu_raw_data_t* raw, struct cpu_id_t* data) +{ + int i, j, basic, xmodel, xfamily, ext; + char brandstr[64] = {0}; + data->vendor = cpuid_vendor_identify(raw->basic_cpuid[0], data->vendor_str); + if (data->vendor == VENDOR_UNKNOWN) return set_error(ERR_CPU_UNKN); basic = raw->basic_cpuid[0][0]; @@ -347,7 +374,7 @@ int cpuid_get_raw_data(struct cpu_raw_data_t* data) cpu_exec_cpuid(i, data->basic_cpuid[i]); for (i = 0; i < 32; i++) cpu_exec_cpuid(0x80000000 + i, data->ext_cpuid[i]); - for (i = 0; i < 4; i++) { + for (i = 0; i < MAX_INTELFN4_LEVEL; i++) { memset(data->intel_fn4[i], 0, sizeof(data->intel_fn4[i])); data->intel_fn4[i][0] = 4; data->intel_fn4[i][2] = i; @@ -359,6 +386,18 @@ int cpuid_get_raw_data(struct cpu_raw_data_t* data) data->intel_fn11[i][2] = i; cpu_exec_cpuid_ext(data->intel_fn11[i]); } + for (i = 0; i < MAX_INTELFN12H_LEVEL; i++) { + memset(data->intel_fn12h[i], 0, sizeof(data->intel_fn12h[i])); + data->intel_fn12h[i][0] = 0x12; + data->intel_fn12h[i][2] = i; + cpu_exec_cpuid_ext(data->intel_fn12h[i]); + } + for (i = 0; i < MAX_INTELFN14H_LEVEL; i++) { + memset(data->intel_fn14h[i], 0, sizeof(data->intel_fn14h[i])); + data->intel_fn14h[i][0] = 0x14; + data->intel_fn14h[i][2] = i; + cpu_exec_cpuid_ext(data->intel_fn14h[i]); + } return set_error(ERR_OK); } @@ -390,6 +429,14 @@ int cpuid_serialize_raw_data(struct cpu_raw_data_t* data, const char* filename) fprintf(f, "intel_fn11[%d]=%08x %08x %08x %08x\n", i, data->intel_fn11[i][0], data->intel_fn11[i][1], data->intel_fn11[i][2], data->intel_fn11[i][3]); + for (i = 0; i < MAX_INTELFN12H_LEVEL; i++) + fprintf(f, "intel_fn12h[%d]=%08x %08x %08x %08x\n", i, + data->intel_fn12h[i][0], data->intel_fn12h[i][1], + data->intel_fn12h[i][2], data->intel_fn12h[i][3]); + for (i = 0; i < MAX_INTELFN14H_LEVEL; i++) + fprintf(f, "intel_fn14h[%d]=%08x %08x %08x %08x\n", i, + data->intel_fn14h[i][0], data->intel_fn14h[i][1], + data->intel_fn14h[i][2], data->intel_fn14h[i][3]); if (strcmp(filename, "")) fclose(f); @@ -434,10 +481,12 @@ int cpuid_deserialize_raw_data(struct cpu_raw_data_t* data, const char* filename recognized = 1; } syntax = 1; - syntax = syntax && parse_token("basic_cpuid", token, value, data->basic_cpuid, 32, &recognized); - syntax = syntax && parse_token("ext_cpuid", token, value, data->ext_cpuid, 32, &recognized); - syntax = syntax && parse_token("intel_fn4", token, value, data->intel_fn4, 4, &recognized); - syntax = syntax && parse_token("intel_fn11", token, value, data->intel_fn11, 4, &recognized); + syntax = syntax && parse_token("basic_cpuid", token, value, data->basic_cpuid, MAX_CPUID_LEVEL, &recognized); + syntax = syntax && parse_token("ext_cpuid", token, value, data->ext_cpuid, MAX_EXT_CPUID_LEVEL, &recognized); + syntax = syntax && parse_token("intel_fn4", token, value, data->intel_fn4, MAX_INTELFN4_LEVEL, &recognized); + syntax = syntax && parse_token("intel_fn11", token, value, data->intel_fn11, MAX_INTELFN11_LEVEL, &recognized); + syntax = syntax && parse_token("intel_fn12h", token, value, data->intel_fn12h, MAX_INTELFN12H_LEVEL, &recognized); + syntax = syntax && parse_token("intel_fn14h", token, value, data->intel_fn14h, MAX_INTELFN14H_LEVEL, &recognized); if (!syntax) { warnf("Error: %s:%d: Syntax error\n", filename, cur_line); fclose(f); @@ -453,7 +502,7 @@ int cpuid_deserialize_raw_data(struct cpu_raw_data_t* data, const char* filename return set_error(ERR_OK); } -int cpu_identify(struct cpu_raw_data_t* raw, struct cpu_id_t* data) +int cpu_ident_internal(struct cpu_raw_data_t* raw, struct cpu_id_t* data, struct internal_id_info_t* internal) { int r; struct cpu_raw_data_t myraw; @@ -467,10 +516,10 @@ int cpu_identify(struct cpu_raw_data_t* raw, struct cpu_id_t* data) return set_error(r); switch (data->vendor) { case VENDOR_INTEL: - r = cpuid_identify_intel(raw, data); + r = cpuid_identify_intel(raw, data, internal); break; case VENDOR_AMD: - r = cpuid_identify_amd(raw, data); + r = cpuid_identify_amd(raw, data, internal); break; default: break; @@ -478,6 +527,12 @@ int cpu_identify(struct cpu_raw_data_t* raw, struct cpu_id_t* data) return set_error(r); } +int cpu_identify(struct cpu_raw_data_t* raw, struct cpu_id_t* data) +{ + struct internal_id_info_t throwaway; + return cpu_ident_internal(raw, data, &throwaway); +} + const char* cpu_feature_str(cpu_feature_t feature) { const struct { cpu_feature_t feature; const char* name; } @@ -577,6 +632,21 @@ const char* cpu_feature_str(cpu_feature_t feature) { CPU_FEATURE_PFI, "pfi" }, { CPU_FEATURE_PA, "pa" }, { CPU_FEATURE_AVX2, "avx2" }, + { CPU_FEATURE_BMI1, "bmi1" }, + { CPU_FEATURE_BMI2, "bmi2" }, + { CPU_FEATURE_HLE, "hle" }, + { CPU_FEATURE_RTM, "rtm" }, + { CPU_FEATURE_AVX512F, "avx512f" }, + { CPU_FEATURE_AVX512DQ, "avx512dq" }, + { CPU_FEATURE_AVX512PF, "avx512pf" }, + { CPU_FEATURE_AVX512ER, "avx512er" }, + { CPU_FEATURE_AVX512CD, "avx512cd" }, + { CPU_FEATURE_SHA_NI, "sha_ni" }, + { CPU_FEATURE_AVX512BW, "avx512bw" }, + { CPU_FEATURE_AVX512VL, "avx512vl" }, + { CPU_FEATURE_SGX, "sgx" }, + { CPU_FEATURE_RDSEED, "rdseed" }, + { CPU_FEATURE_ADX, "adx" }, }; unsigned i, n = COUNT_OF(matchtable); if (n != NUM_CPU_FEATURES) { @@ -600,6 +670,15 @@ const char* cpuid_error(void) { ERR_BADFMT , "Bad file format"}, { ERR_NOT_IMP , "Not implemented"}, { ERR_CPU_UNKN , "Unsupported processor"}, + { ERR_NO_RDMSR , "RDMSR instruction is not supported"}, + { ERR_NO_DRIVER, "RDMSR driver error (generic)"}, + { ERR_NO_PERMS , "No permissions to install RDMSR driver"}, + { ERR_EXTRACT , "Cannot extract RDMSR driver (read only media?)"}, + { ERR_HANDLE , "Bad handle"}, + { ERR_INVMSR , "Invalid MSR"}, + { ERR_INVCNB , "Invalid core number"}, + { ERR_HANDLE_R , "Error on handle read"}, + { ERR_INVRANGE , "Invalid given range"}, }; unsigned i; for (i = 0; i < COUNT_OF(matchtable); i++) @@ -626,6 +705,23 @@ void cpuid_set_verbosiness_level(int level) _current_verboselevel = level; } +cpu_vendor_t cpuid_get_vendor(void) +{ + static cpu_vendor_t vendor = VENDOR_UNKNOWN; + uint32_t raw_vendor[4]; + char vendor_str[VENDOR_STR_MAX]; + + if(vendor == VENDOR_UNKNOWN) { + if (!cpuid_present()) + set_error(ERR_NO_CPUID); + else { + cpu_exec_cpuid(0, raw_vendor); + vendor = cpuid_vendor_identify(raw_vendor, vendor_str); + } + } + return vendor; +} + void cpuid_get_cpu_list(cpu_vendor_t vendor, struct cpu_list_t* list) { switch (vendor) { diff --git a/contrib/libcpuid/include/libcpuid/intel_code_t.h b/contrib/libcpuid/include/libcpuid/intel_code_t.h new file mode 100644 index 00000000000..c50ec9c5a83 --- /dev/null +++ b/contrib/libcpuid/include/libcpuid/intel_code_t.h @@ -0,0 +1,83 @@ +/* + * Copyright 2016 Veselin Georgiev, + * anrieffNOSPAM @ mgail_DOT.com (convert to gmail) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * This file contains a list of internal codes we use in detection. It is + * of no external use and isn't a complete list of intel products. + */ + CODE2(PENTIUM, 2000), + CODE(MOBILE_PENTIUM), + + CODE(XEON), + CODE(XEON_IRWIN), + CODE(XEONMP), + CODE(XEON_POTOMAC), + CODE(XEON_I7), + CODE(XEON_GAINESTOWN), + CODE(XEON_WESTMERE), + + CODE(MOBILE_PENTIUM_M), + CODE(CELERON), + CODE(MOBILE_CELERON), + CODE(NOT_CELERON), + + + CODE(CORE_SOLO), + CODE(MOBILE_CORE_SOLO), + CODE(CORE_DUO), + CODE(MOBILE_CORE_DUO), + + CODE(WOLFDALE), + CODE(MEROM), + CODE(PENRYN), + CODE(QUAD_CORE), + CODE(DUAL_CORE_HT), + CODE(QUAD_CORE_HT), + CODE(MORE_THAN_QUADCORE), + CODE(PENTIUM_D), + + CODE(ATOM_UNKNOWN), + CODE(ATOM_SILVERTHORNE), + CODE(ATOM_DIAMONDVILLE), + CODE(ATOM_PINEVIEW), + CODE(ATOM_CEDARVIEW), + + CODE(CORE_I3), + CODE(CORE_I5), + CODE(CORE_I7), + CODE(CORE_IVY3), /* 22nm Core-iX */ + CODE(CORE_IVY5), + CODE(CORE_IVY7), + CODE(CORE_HASWELL3), /* 22nm Core-iX, Haswell */ + CODE(CORE_HASWELL5), + CODE(CORE_HASWELL7), + CODE(CORE_BROADWELL3), /* 14nm Core-iX, Broadwell */ + CODE(CORE_BROADWELL5), + CODE(CORE_BROADWELL7), + CODE(CORE_SKYLAKE3), /* 14nm Core-iX, Skylake */ + CODE(CORE_SKYLAKE5), + CODE(CORE_SKYLAKE7), + diff --git a/contrib/libcpuid/include/cpuid/libcpuid.h b/contrib/libcpuid/include/libcpuid/libcpuid.h similarity index 75% rename from contrib/libcpuid/include/cpuid/libcpuid.h rename to contrib/libcpuid/include/libcpuid/libcpuid.h index b78b0d6f514..866c0e8441d 100644 --- a/contrib/libcpuid/include/cpuid/libcpuid.h +++ b/contrib/libcpuid/include/libcpuid/libcpuid.h @@ -26,25 +26,37 @@ #ifndef __LIBCPUID_H__ #define __LIBCPUID_H__ /** - * @File libcpuid.h - * @Author Veselin Georgiev - * @Date Oct 2008 - * @Version 0.2.1 + * \file libcpuid.h + * \author Veselin Georgiev + * \date Oct 2008 + * \version 0.4.0 * * Version history: * - * 0.1.0 (2008-10-15): initial adaptation from wxfractgui sources - * 0.1.1 (2009-07-06): Added intel_fn11 fields to cpu_raw_data_t to handle - * new processor topology enumeration required on Core i7 - * 0.1.2 (2009-09-26): Added support for MSR reading through self-extracting - * kernel driver on Win32. - * 0.1.3 (2010-04-20): Added support for greater more accurate CPU clock - * measurements with cpu_clock_by_ic() - * 0.2.0 (2011-10-11): Support for AMD Bulldozer CPUs, 128-bit SSE unit size - * checking. A backwards-incompatible change, since the - * sizeof cpu_id_t is now different. - * 0.2.1 (2012-05-26): Support for Ivy Bridge, and detecting the presence of - * the RdRand instruction. + * * 0.1.0 (2008-10-15): initial adaptation from wxfractgui sources + * * 0.1.1 (2009-07-06): Added intel_fn11 fields to cpu_raw_data_t to handle + * new processor topology enumeration required on Core i7 + * * 0.1.2 (2009-09-26): Added support for MSR reading through self-extracting + * kernel driver on Win32. + * * 0.1.3 (2010-04-20): Added support for greater more accurate CPU clock + * measurements with cpu_clock_by_ic() + * * 0.2.0 (2011-10-11): Support for AMD Bulldozer CPUs, 128-bit SSE unit size + * checking. A backwards-incompatible change, since the + * sizeof cpu_id_t is now different. + * * 0.2.1 (2012-05-26): Support for Ivy Bridge, and detecting the presence of + * the RdRand instruction. + * * 0.2.2 (2015-11-04): Support for newer processors up to Haswell and Vishera. + * Fix clock detection in cpu_clock_by_ic() for Bulldozer. + * More entries supported in cpu_msrinfo(). + * *BSD and Solaris support (unofficial). + * * 0.3.0 (2016-07-09): Support for Skylake; MSR ops in FreeBSD; INFO_VOLTAGE + * for AMD CPUs. Level 4 cache support for Crystalwell + * (a backwards-incompatible change since the sizeof + * cpu_raw_data_t is now different). + * * 0.4.0 (2016-09-30): Better detection of AMD clock multiplier with msrinfo. + * Support for Intel SGX detection + * (a backwards-incompatible change since the sizeof + * cpu_raw_data_t and cpu_id_t is now different). */ /** @mainpage A simple libcpuid introduction @@ -57,6 +69,15 @@ * To fetch the CPUID info needed for CPU identification, use * \ref cpuid_get_raw_data
* To make sense of that data (decode, extract features), use \ref cpu_identify
+ * To detect the CPU speed, use either \ref cpu_clock, \ref cpu_clock_by_os, + * \ref cpu_tsc_mark + \ref cpu_tsc_unmark + \ref cpu_clock_by_mark, + * \ref cpu_clock_measure or \ref cpu_clock_by_ic. + * Read carefully for pros/cons of each method.
+ * + * To read MSRs, use \ref cpu_msr_driver_open to get a handle, and then + * \ref cpu_rdmsr for querying abilities. Some MSR decoding is available on recent + * CPUs, and can be queried through \ref cpu_msrinfo; the various types of queries + * are described in \ref cpu_msrinfo_request_t. *

*/ @@ -116,6 +137,81 @@ struct cpu_raw_data_t { enumeration leaf), this stores the result of CPUID with eax = 11 and ecx = 0, 1, 2... */ uint32_t intel_fn11[MAX_INTELFN11_LEVEL][4]; + + /** when the CPU is intel and supports leaf 12h (SGX enumeration leaf), + * this stores the result of CPUID with eax = 0x12 and + * ecx = 0, 1, 2... */ + uint32_t intel_fn12h[MAX_INTELFN12H_LEVEL][4]; + + /** when the CPU is intel and supports leaf 14h (Intel Processor Trace + * capabilities leaf). + * this stores the result of CPUID with eax = 0x12 and + * ecx = 0, 1, 2... */ + uint32_t intel_fn14h[MAX_INTELFN14H_LEVEL][4]; +}; + +/** + * @brief This contains information about SGX features of the processor + * Example usage: + * @code + * ... + * struct cpu_raw_data_t raw; + * struct cpu_id_t id; + * + * if (cpuid_get_raw_data(&raw) == 0 && cpu_identify(&raw, &id) == 0 && id.sgx.present) { + * printf("SGX is present.\n"); + * printf("SGX1 instructions: %s.\n", id.sgx.flags[INTEL_SGX1] ? "present" : "absent"); + * printf("SGX2 instructions: %s.\n", id.sgx.flags[INTEL_SGX2] ? "present" : "absent"); + * printf("Max 32-bit enclave size: 2^%d bytes.\n", id.sgx.max_enclave_32bit); + * printf("Max 64-bit enclave size: 2^%d bytes.\n", id.sgx.max_enclave_64bit); + * for (int i = 0; i < id.sgx.num_epc_sections; i++) { + * struct cpu_epc_t epc = cpuid_get_epc(i, NULL); + * printf("EPC section #%d: address = %x, size = %d bytes.\n", epc.address, epc.size); + * } + * } else { + * printf("SGX is not present.\n"); + * } + * @endcode + */ +struct cpu_sgx_t { + /** Whether SGX is present (boolean) */ + uint32_t present; + + /** Max enclave size in 32-bit mode. This is a power-of-two value: + * if it is "31", then the max enclave size is 2^31 bytes (2 GiB). + */ + uint8_t max_enclave_32bit; + + /** Max enclave size in 64-bit mode. This is a power-of-two value: + * if it is "36", then the max enclave size is 2^36 bytes (64 GiB). + */ + uint8_t max_enclave_64bit; + + /** + * contains SGX feature flags. See the \ref cpu_sgx_feature_t + * "INTEL_SGX*" macros below. + */ + uint8_t flags[SGX_FLAGS_MAX]; + + /** number of Enclave Page Cache (EPC) sections. Info for each + * section is available through the \ref cpuid_get_epc() function + */ + int num_epc_sections; + + /** bit vector of the supported extended features that can be written + * to the MISC region of the SSA (Save State Area) + */ + uint32_t misc_select; + + /** a bit vector of the attributes that can be set to SECS.ATTRIBUTES + * via ECREATE. Corresponds to bits 0-63 (incl.) of SECS.ATTRIBUTES. + */ + uint64_t secs_attributes; + + /** a bit vector of the bits that can be set in the XSAVE feature + * request mask; Corresponds to bits 64-127 of SECS.ATTRIBUTES. + */ + uint64_t secs_xfrm; }; /** @@ -133,7 +229,8 @@ struct cpu_id_t { /** * contain CPU flags. Used to test for features. See - * the CPU_FEATURE_* macros below. @see Features + * the \ref cpu_feature_t "CPU_FEATURE_*" macros below. + * @see Features */ uint8_t flags[CPU_FLAGS_MAX]; @@ -164,11 +261,17 @@ struct cpu_id_t { /** * The total number of logical processors. + * The same value is availabe through \ref cpuid_get_total_cpus. * * This is num_logical_cpus * {total physical processors in the system} + * (but only on a real system, under a VM this number may be lower). * * If you're writing a multithreaded program and you want to run it on * all CPUs, this is the number of threads you need. + * + * @note in a VM, this will exactly match the number of CPUs set in + * the VM's configuration. + * */ int32_t total_logical_cpus; @@ -194,6 +297,9 @@ struct cpu_id_t { /** L3 cache size in KB. Zero on most systems */ int32_t l3_cache; + + /** L4 cache size in KB. Zero on most systems */ + int32_t l4_cache; /** Cache associativity for the L1 data cache. -1 if undetermined */ int32_t l1_assoc; @@ -203,6 +309,9 @@ struct cpu_id_t { /** Cache associativity for the L3 cache. -1 if undetermined */ int32_t l3_assoc; + + /** Cache associativity for the L4 cache. -1 if undetermined */ + int32_t l4_assoc; /** Cache-line size for L1 data cache. -1 if undetermined */ int32_t l1_cacheline; @@ -213,6 +322,9 @@ struct cpu_id_t { /** Cache-line size for L3 cache. -1 if undetermined */ int32_t l3_cacheline; + /** Cache-line size for L4 cache. -1 if undetermined */ + int32_t l4_cacheline; + /** * The brief and human-friendly CPU codename, which was recognized.
* Examples: @@ -234,9 +346,13 @@ struct cpu_id_t { /** * contain miscellaneous detection information. Used to test about specifics of - * certain detected features. See CPU_HINT_* macros below. @see Hints + * certain detected features. See \ref cpu_hint_t "CPU_HINT_*" macros below. + * @see Hints */ uint8_t detection_hints[CPU_HINTS_MAX]; + + /** contains information about SGX features if the processor, if present */ + struct cpu_sgx_t sgx; }; /** @@ -355,6 +471,21 @@ typedef enum { CPU_FEATURE_PFI, /*!< Processor Feedback Interface support */ CPU_FEATURE_PA, /*!< Processor accumulator */ CPU_FEATURE_AVX2, /*!< AVX2 instructions */ + CPU_FEATURE_BMI1, /*!< BMI1 instructions */ + CPU_FEATURE_BMI2, /*!< BMI2 instructions */ + CPU_FEATURE_HLE, /*!< Hardware Lock Elision prefixes */ + CPU_FEATURE_RTM, /*!< Restricted Transactional Memory instructions */ + CPU_FEATURE_AVX512F, /*!< AVX-512 Foundation */ + CPU_FEATURE_AVX512DQ, /*!< AVX-512 Double/Quad granular insns */ + CPU_FEATURE_AVX512PF, /*!< AVX-512 Prefetch */ + CPU_FEATURE_AVX512ER, /*!< AVX-512 Exponential/Reciprocal */ + CPU_FEATURE_AVX512CD, /*!< AVX-512 Conflict detection */ + CPU_FEATURE_SHA_NI, /*!< SHA-1/SHA-256 instructions */ + CPU_FEATURE_AVX512BW, /*!< AVX-512 Byte/Word granular insns */ + CPU_FEATURE_AVX512VL, /*!< AVX-512 128/256 vector length extensions */ + CPU_FEATURE_SGX, /*!< SGX extensions. Non-autoritative, check cpu_id_t::sgx::present to verify presence */ + CPU_FEATURE_RDSEED, /*!< RDSEED instruction */ + CPU_FEATURE_ADX, /*!< ADX extensions (arbitrary precision) */ /* termination: */ NUM_CPU_FEATURES, } cpu_feature_t; @@ -370,6 +501,36 @@ typedef enum { NUM_CPU_HINTS, } cpu_hint_t; +/** + * @brief SGX features flags + * \see cpu_sgx_t + * + * Usage: + * @code + * ... + * struct cpu_raw_data_t raw; + * struct cpu_id_t id; + * if (cpuid_get_raw_data(&raw) == 0 && cpu_identify(&raw, &id) == 0 && id.sgx.present) { + * if (id.sgx.flags[INTEL_SGX1]) + * // The CPU has SGX1 instructions support... + * ... + * } else { + * // no SGX + * } + * } else { + * // processor cannot be determined. + * } + * @endcode + */ + +typedef enum { + INTEL_SGX1, /*!< SGX1 instructions support */ + INTEL_SGX2, /*!< SGX2 instructions support */ + + /* termination: */ + NUM_SGX_FEATURES, +} cpu_sgx_feature_t; + /** * @brief Describes common library error codes */ @@ -387,7 +548,10 @@ typedef enum { ERR_NO_PERMS = -10, /*!< "No permissions to install RDMSR driver" */ ERR_EXTRACT = -11, /*!< "Cannot extract RDMSR driver (read only media?)" */ ERR_HANDLE = -12, /*!< "Bad handle" */ - ERR_INVMSR = -13, /*!< "Invalid MSR" */ + ERR_INVMSR = -13, /*!< "Invalid MSR" */ + ERR_INVCNB = -14, /*!< "Invalid core number" */ + ERR_HANDLE_R = -15, /*!< "Error on handle read" */ + ERR_INVRANGE = -16, /*!< "Invalid given range" */ } cpu_error_t; /** @@ -400,8 +564,14 @@ struct cpu_mark_t { }; /** - * @brief Returns the total number of CPUs even if CPUID is not present - * @retval Number of CPUs available + * @brief Returns the total number of logical CPU threads (even if CPUID is not present). + * + * Under VM, this number (and total_logical_cpus, since they are fetched with the same code) + * may be nonsensical, i.e. might not equal NumPhysicalCPUs*NumCoresPerCPU*HyperThreading. + * This is because no matter how many logical threads the host machine has, you may limit them + * in the VM to any number you like. **This** is the number returned by cpuid_get_total_cpus(). + * + * @returns Number of logical CPU threads available. Equals the \ref cpu_id_t::total_logical_cpus. */ int cpuid_get_total_cpus(void); @@ -713,6 +883,32 @@ int cpu_clock_by_ic(int millis, int runs); */ int cpu_clock(void); + +/** + * @brief The return value of cpuid_get_epc(). + * @details + * Describes an EPC (Enclave Page Cache) layout (physical address and size). + * A CPU may have one or more EPC areas, and information about each is + * fetched via \ref cpuid_get_epc. + */ +struct cpu_epc_t { + uint64_t start_addr; + uint64_t length; +}; + +/** + * @brief Fetches information about an EPC (Enclave Page Cache) area. + * @param index - zero-based index, valid range [0..cpu_id_t.egx.num_epc_sections) + * @param raw - a pointer to fetched raw CPUID data. Needed only for testing, + * you can safely pass NULL here (if you pass a real structure, + * it will be used for fetching the leaf 12h data if index < 2; + * otherwise the real CPUID instruction will be used). + * @returns the requested data. If the CPU doesn't support SGX, or if + * index >= cpu_id_t.egx.num_epc_sections, both fields of the returned + * structure will be zeros. + */ +struct cpu_epc_t cpuid_get_epc(int index, const struct cpu_raw_data_t* raw); + /** * @brief Returns the libcpuid version * @@ -750,6 +946,14 @@ libcpuid_warn_fn_t cpuid_set_warn_function(libcpuid_warn_fn_t warn_fun); void cpuid_set_verbosiness_level(int level); +/** + * @brief Obtains the CPU vendor from CPUID from the current CPU + * @note The result is cached. + * @returns VENDOR_UNKNOWN if failed, otherwise the CPU vendor type. + * @see cpu_vendor_t + */ +cpu_vendor_t cpuid_get_vendor(void); + /** * @brief a structure that holds a list of processor names */ @@ -788,6 +992,7 @@ void cpuid_get_cpu_list(cpu_vendor_t vendor, struct cpu_list_t* list); */ void cpuid_free_cpu_list(struct cpu_list_t* list); +struct msr_driver_t; /** * @brief Starts/opens a driver, needed to read MSRs (Model Specific Registers) * @@ -799,9 +1004,23 @@ void cpuid_free_cpu_list(struct cpu_list_t* list); * The error message can be obtained by calling \ref cpuid_error. * @see cpu_error_t */ -struct msr_driver_t; struct msr_driver_t* cpu_msr_driver_open(void); +/** + * @brief Similar to \ref cpu_msr_driver_open, but accept one parameter + * + * This function works on certain operating systems (GNU/Linux, FreeBSD) + * + * @param core_num specify the core number for MSR. + * The first core number is 0. + * The last core number is \ref cpuid_get_total_cpus - 1. + * + * @returns a handle to the driver on success, and NULL on error. + * The error message can be obtained by calling \ref cpuid_error. + * @see cpu_error_t + */ +struct msr_driver_t* cpu_msr_driver_open_core(unsigned core_num); + /** * @brief Reads a Model-Specific Register (MSR) * @@ -826,26 +1045,51 @@ struct msr_driver_t* cpu_msr_driver_open(void); * The error message can be obtained by calling \ref cpuid_error. * @see cpu_error_t */ -int cpu_rdmsr(struct msr_driver_t* handle, int msr_index, uint64_t* result); +int cpu_rdmsr(struct msr_driver_t* handle, uint32_t msr_index, uint64_t* result); typedef enum { INFO_MPERF, /*!< Maximum performance frequency clock. This is a counter, which increments as a - proportion of the actual processor speed */ + proportion of the actual processor speed. */ INFO_APERF, /*!< Actual performance frequency clock. This accumulates the core clock counts when the core is active. */ + INFO_MIN_MULTIPLIER, /*!< Minimum CPU:FSB ratio for this CPU, + multiplied by 100. */ INFO_CUR_MULTIPLIER, /*!< Current CPU:FSB ratio, multiplied by 100. e.g., a CPU:FSB value of 18.5 reads as - 1850. */ - INFO_MAX_MULTIPLIER, /*!< Maxumum CPU:FSB ratio for this CPU, - multiplied by 100 */ - INFO_TEMPERATURE, /*!< The current core temperature in Celsius */ + "1850". */ + INFO_MAX_MULTIPLIER, /*!< Maximum CPU:FSB ratio for this CPU, + multiplied by 100. */ + INFO_TEMPERATURE, /*!< The current core temperature in Celsius. */ INFO_THROTTLING, /*!< 1 if the current logical processor is throttling. 0 if it is running normally. */ + INFO_VOLTAGE, /*!< The current core voltage in Volt, + multiplied by 100. */ + INFO_BCLK, /*!< See \ref INFO_BUS_CLOCK. */ + INFO_BUS_CLOCK, /*!< The main bus clock in MHz, + e.g., FSB/QPI/DMI/HT base clock, + multiplied by 100. */ } cpu_msrinfo_request_t; +/** + * @brief Similar to \ref cpu_rdmsr, but extract a range of bits + * + * @param handle - a handle to the MSR reader driver, as created by + * cpu_msr_driver_open + * @param msr_index - the numeric ID of the MSR you want to read + * @param highbit - the high bit in range, must be inferior to 64 + * @param lowbit - the low bit in range, must be equal or superior to 0 + * @param result - a pointer to a 64-bit integer, where the MSR value is stored + * + * @returns zero if successful, and some negative number on error. + * The error message can be obtained by calling \ref cpuid_error. + * @see cpu_error_t + */ +int cpu_rdmsr_range(struct msr_driver_t* handle, uint32_t msr_index, uint8_t highbit, + uint8_t lowbit, uint64_t* result); + /** * @brief Reads extended CPU information from Model-Specific Registers. * @param handle - a handle to an open MSR driver, @see cpu_msr_driver_open diff --git a/contrib/libcpuid/include/cpuid/libcpuid_constants.h b/contrib/libcpuid/include/libcpuid/libcpuid_constants.h similarity index 93% rename from contrib/libcpuid/include/cpuid/libcpuid_constants.h rename to contrib/libcpuid/include/libcpuid/libcpuid_constants.h index 8af4718c906..3ddb6d5e14e 100644 --- a/contrib/libcpuid/include/cpuid/libcpuid_constants.h +++ b/contrib/libcpuid/include/libcpuid/libcpuid_constants.h @@ -37,8 +37,11 @@ #define CPU_FLAGS_MAX 128 #define MAX_CPUID_LEVEL 32 #define MAX_EXT_CPUID_LEVEL 32 -#define MAX_INTELFN4_LEVEL 4 +#define MAX_INTELFN4_LEVEL 8 #define MAX_INTELFN11_LEVEL 4 +#define MAX_INTELFN12H_LEVEL 4 +#define MAX_INTELFN14H_LEVEL 4 #define CPU_HINTS_MAX 16 +#define SGX_FLAGS_MAX 14 #endif /* __LIBCPUID_CONSTANTS_H__ */ diff --git a/contrib/libcpuid/include/libcpuid/libcpuid_internal.h b/contrib/libcpuid/include/libcpuid/libcpuid_internal.h new file mode 100644 index 00000000000..95528896219 --- /dev/null +++ b/contrib/libcpuid/include/libcpuid/libcpuid_internal.h @@ -0,0 +1,63 @@ +/* + * Copyright 2016 Veselin Georgiev, + * anrieffNOSPAM @ mgail_DOT.com (convert to gmail) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef __LIBCPUID_INTERNAL_H__ +#define __LIBCPUID_INTERNAL_H__ +/* + * This file contains internal undocumented declarations and function prototypes + * for the workings of the internal library infrastructure. + */ + +enum _common_codes_t { + NA = 0, + NO_CODE, +}; + +#define CODE(x) x +#define CODE2(x, y) x = y +enum _amd_code_t { + #include "amd_code_t.h" +}; +typedef enum _amd_code_t amd_code_t; + +enum _intel_code_t { + #include "intel_code_t.h" +}; +typedef enum _intel_code_t intel_code_t; +#undef CODE +#undef CODE2 + +struct internal_id_info_t { + union { + amd_code_t amd; + intel_code_t intel; + } code; + int score; // detection (matchtable) score +}; + +int cpu_ident_internal(struct cpu_raw_data_t* raw, struct cpu_id_t* data, + struct internal_id_info_t* internal); + +#endif /* __LIBCPUID_INTERNAL_H__ */ diff --git a/contrib/libcpuid/include/cpuid/libcpuid_types.h b/contrib/libcpuid/include/libcpuid/libcpuid_types.h similarity index 66% rename from contrib/libcpuid/include/cpuid/libcpuid_types.h rename to contrib/libcpuid/include/libcpuid/libcpuid_types.h index 6dd18a97724..f1181edf2ee 100644 --- a/contrib/libcpuid/include/cpuid/libcpuid_types.h +++ b/contrib/libcpuid/include/libcpuid/libcpuid_types.h @@ -32,6 +32,37 @@ #ifndef __LIBCPUID_TYPES_H__ #define __LIBCPUID_TYPES_H__ +//#ifdef HAVE_CONFIG_H // CLICKHOUSE PATCH +# include "config.h" +//#endif // CLICKHOUSE PATCH + +#if 1 // CLICKHOUSE PATCH +//#if defined(HAVE_STDINT_H) // CLICKHOUSE PATCH # include +#else +/* we have to provide our own: */ +# if !defined(HAVE_INT32_T) && !defined(__int32_t_defined) +typedef int int32_t; +# endif + +# if !defined(HAVE_UINT32_T) && !defined(__uint32_t_defined) +typedef unsigned uint32_t; +# endif + +typedef signed char int8_t; +typedef unsigned char uint8_t; +typedef signed short int16_t; +typedef unsigned short uint16_t; +#if (defined _MSC_VER) && (_MSC_VER <= 1300) + /* MSVC 6.0: no long longs ... */ + typedef signed __int64 int64_t; + typedef unsigned __int64 uint64_t; +#else + /* all other sane compilers: */ + typedef signed long long int64_t; + typedef unsigned long long uint64_t; +#endif + +#endif #endif /* __LIBCPUID_TYPES_H__ */ diff --git a/contrib/libcpuid/include/cpuid/libcpuid_util.c b/contrib/libcpuid/include/libcpuid/libcpuid_util.c similarity index 86% rename from contrib/libcpuid/include/cpuid/libcpuid_util.c rename to contrib/libcpuid/include/libcpuid/libcpuid_util.c index 36e83c84b4a..ea6b1b8dfb4 100644 --- a/contrib/libcpuid/include/cpuid/libcpuid_util.c +++ b/contrib/libcpuid/include/libcpuid/libcpuid_util.c @@ -38,7 +38,7 @@ void match_features(const struct feature_map_t* matchtable, int count, uint32_t { int i; for (i = 0; i < count; i++) - if (reg & (1U << matchtable[i].bit)) + if (reg & (1u << matchtable[i].bit)) data->flags[matchtable[i].feature] = 1; } @@ -78,20 +78,20 @@ static int score(const struct match_entry_t* entry, const struct cpu_id_t* data, int brand_code, int model_code) { int res = 0; - if (entry->family == data->family ) res++; - if (entry->model == data->model ) res++; - if (entry->stepping == data->stepping ) res++; - if (entry->ext_family == data->ext_family) res++; - if (entry->ext_model == data->ext_model ) res++; - if (entry->ncores == data->num_cores ) res++; - if (entry->l2cache == data->l2_cache ) res++; - if (entry->l3cache == data->l3_cache ) res++; - if (entry->brand_code == brand_code ) res++; - if (entry->model_code == model_code ) res++; + if (entry->family == data->family ) res += 2; + if (entry->model == data->model ) res += 2; + if (entry->stepping == data->stepping ) res += 2; + if (entry->ext_family == data->ext_family) res += 2; + if (entry->ext_model == data->ext_model ) res += 2; + if (entry->ncores == data->num_cores ) res += 2; + if (entry->l2cache == data->l2_cache ) res += 1; + if (entry->l3cache == data->l3_cache ) res += 1; + if (entry->brand_code == brand_code ) res += 2; + if (entry->model_code == model_code ) res += 2; return res; } -void match_cpu_codename(const struct match_entry_t* matchtable, int count, +int match_cpu_codename(const struct match_entry_t* matchtable, int count, struct cpu_id_t* data, int brand_code, int model_code) { int bestscore = -1; @@ -112,6 +112,7 @@ void match_cpu_codename(const struct match_entry_t* matchtable, int count, } } strcpy(data->cpu_codename, matchtable[bestindex].name); + return bestscore; } void generic_get_cpu_list(const struct match_entry_t* matchtable, int count, @@ -129,7 +130,11 @@ void generic_get_cpu_list(const struct match_entry_t* matchtable, int count, break; } if (!good) continue; +#if defined(_MSC_VER) + list->names[n++] = _strdup(matchtable[i].name); +#else list->names[n++] = strdup(matchtable[i].name); +#endif } list->num_entries = n; } diff --git a/contrib/libcpuid/include/cpuid/libcpuid_util.h b/contrib/libcpuid/include/libcpuid/libcpuid_util.h similarity index 96% rename from contrib/libcpuid/include/cpuid/libcpuid_util.h rename to contrib/libcpuid/include/libcpuid/libcpuid_util.h index 34e1efe960a..22becea26f6 100644 --- a/contrib/libcpuid/include/cpuid/libcpuid_util.h +++ b/contrib/libcpuid/include/libcpuid/libcpuid_util.h @@ -42,7 +42,8 @@ struct match_entry_t { char name[32]; }; -void match_cpu_codename(const struct match_entry_t* matchtable, int count, +// returns the match score: +int match_cpu_codename(const struct match_entry_t* matchtable, int count, struct cpu_id_t* data, int brand_code, int model_code); void warnf(const char* format, ...) diff --git a/contrib/libcpuid/include/libcpuid/msrdriver.c b/contrib/libcpuid/include/libcpuid/msrdriver.c new file mode 100644 index 00000000000..8f9d7ed0ca8 --- /dev/null +++ b/contrib/libcpuid/include/libcpuid/msrdriver.c @@ -0,0 +1,593 @@ +/* + * Copyright 2009 Veselin Georgiev, + * anrieffNOSPAM @ mgail_DOT.com (convert to gmail) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** + * @File msrdriver.c + * @Brief Contains the binary images of the x86 and x64 MSR drivers for Windows + * @Date 2009-09-29 + * + * The driver is courtesy of Nick 'Bombera' Gabareff, and its source is actually + * available, see the contrib/ dir. + * + * However, for simplicity, here we just include the images of the compiled .SYS + * files. + * They are extracted to the filesystem on demand and loaded in the kernel + * by the cpu_msr_driver_open() function + */ +#ifdef _WIN32 +#include "asm-bits.h" +//begin { +int cc_x86driver_code_size = 4608; +uint8_t cc_x86driver_code[4608] = { + 0x4d,0x5a,0x90,0x00,0x03,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xff,0xff,0x00,0x00,0xb8,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0xc8,0x00,0x00,0x00,0x0e,0x1f,0xba,0x0e,0x00,0xb4,0x09,0xcd, + 0x21,0xb8,0x01,0x4c,0xcd,0x21,0x54,0x68,0x69,0x73,0x20,0x70,0x72,0x6f,0x67,0x72,0x61,0x6d, + 0x20,0x63,0x61,0x6e,0x6e,0x6f,0x74,0x20,0x62,0x65,0x20,0x72,0x75,0x6e,0x20,0x69,0x6e,0x20, + 0x44,0x4f,0x53,0x20,0x6d,0x6f,0x64,0x65,0x2e,0x0d,0x0d,0x0a,0x24,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x9f,0x99,0x48,0xdf,0xdb,0xf8,0x26,0x8c,0xdb,0xf8,0x26,0x8c,0xdb,0xf8,0x26,0x8c, + 0xdb,0xf8,0x27,0x8c,0xdd,0xf8,0x26,0x8c,0x21,0xdb,0x3f,0x8c,0xd8,0xf8,0x26,0x8c,0xfc,0x3e, + 0x57,0x8c,0xda,0xf8,0x26,0x8c,0xfc,0x3e,0x5a,0x8c,0xda,0xf8,0x26,0x8c,0xfc,0x3e,0x5e,0x8c, + 0xda,0xf8,0x26,0x8c,0x52,0x69,0x63,0x68,0xdb,0xf8,0x26,0x8c,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x50,0x45,0x00,0x00,0x4c,0x01,0x07,0x00,0x12,0x9b,0x9b,0x4a,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0xe0,0x00,0x02,0x21,0x0b,0x01,0x08,0x00,0x00,0x06,0x00,0x00,0x00,0x0a, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x20,0x00,0x00, + 0x00,0x00,0x01,0x00,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x80,0x00,0x00,0x00,0x04,0x00,0x00, + 0xa9,0xd1,0x00,0x00,0x01,0x00,0x00,0x04,0x00,0x00,0x10,0x00,0x00,0x10,0x00,0x00,0x00,0x00, + 0x10,0x00,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x50,0x00,0x00,0x28,0x00,0x00,0x00,0x00,0x60,0x00,0x00,0xc0,0x03, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x70,0x00,0x00,0x30,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x20,0x00,0x00,0x1c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x2e,0x74, + 0x65,0x78,0x74,0x00,0x00,0x00,0xa3,0x00,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x02,0x00,0x00, + 0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00, + 0x00,0x68,0x2e,0x72,0x64,0x61,0x74,0x61,0x00,0x00,0x62,0x00,0x00,0x00,0x00,0x20,0x00,0x00, + 0x00,0x02,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x40,0x00,0x00,0x48,0x2e,0x64,0x61,0x74,0x61,0x00,0x00,0x00,0x04,0x00,0x00,0x00, + 0x00,0x30,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x40,0x00,0x00,0xc8,0x50,0x41,0x47,0x45,0x30,0x44,0x45,0x46, + 0x8c,0x00,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x08,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x00,0x60,0x49,0x4e,0x49,0x54, + 0x00,0x00,0x00,0x00,0xd4,0x00,0x00,0x00,0x00,0x50,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x0a, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x00,0xe2, + 0x2e,0x72,0x73,0x72,0x63,0x00,0x00,0x00,0xc0,0x03,0x00,0x00,0x00,0x60,0x00,0x00,0x00,0x04, + 0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x40,0x00,0x00,0x42,0x2e,0x72,0x65,0x6c,0x6f,0x63,0x00,0x00,0x68,0x00,0x00,0x00,0x00,0x70, + 0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x40,0x00,0x00,0x42,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x8b,0x4c, + 0x24,0x08,0x83,0x61,0x18,0x00,0x83,0x61,0x1c,0x00,0x32,0xd2,0xff,0x15,0x08,0x20,0x01,0x00, + 0x33,0xc0,0xc2,0x08,0x00,0x56,0x8b,0x74,0x24,0x0c,0x8b,0x46,0x60,0x81,0x78,0x0c,0x0c,0xe0, + 0x22,0x00,0x57,0x75,0x3c,0x83,0x78,0x04,0x08,0x72,0x36,0x83,0x78,0x08,0x04,0x75,0x07,0x8b, + 0x46,0x0c,0x8b,0x08,0xeb,0x05,0xb9,0x9c,0x01,0x00,0x00,0x8b,0x7e,0x0c,0x0f,0x32,0x89,0x07, + 0x89,0x57,0x04,0xc7,0x46,0x1c,0x08,0x00,0x00,0x00,0x33,0xff,0x32,0xd2,0x8b,0xce,0xff,0x15, + 0x08,0x20,0x01,0x00,0x8b,0xc7,0x5f,0x5e,0xc2,0x08,0x00,0x83,0x66,0x1c,0x00,0xbf,0x01,0x00, + 0x00,0xc0,0x89,0x7e,0x18,0xeb,0xe1,0x55,0x8b,0xec,0x51,0x51,0x8b,0x45,0x08,0xff,0x70,0x04, + 0xff,0x15,0x04,0x20,0x01,0x00,0x68,0x3c,0x20,0x01,0x00,0x8d,0x45,0xf8,0x50,0xff,0x15,0x00, + 0x20,0x01,0x00,0x8d,0x45,0xf8,0x50,0xff,0x15,0x14,0x20,0x01,0x00,0xc9,0xc2,0x04,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x5c,0x50,0x00,0x00,0x74,0x50,0x00,0x00,0x86,0x50,0x00,0x00, + 0x9c,0x50,0x00,0x00,0xb4,0x50,0x00,0x00,0x44,0x50,0x00,0x00,0x00,0x00,0x00,0x00,0x5c,0x00, + 0x44,0x00,0x65,0x00,0x76,0x00,0x69,0x00,0x63,0x00,0x65,0x00,0x5c,0x00,0x54,0x00,0x6d,0x00, + 0x70,0x00,0x52,0x00,0x64,0x00,0x72,0x00,0x00,0x00,0x00,0x00,0x5c,0x00,0x44,0x00,0x6f,0x00, + 0x73,0x00,0x44,0x00,0x65,0x00,0x76,0x00,0x69,0x00,0x63,0x00,0x65,0x00,0x73,0x00,0x5c,0x00, + 0x54,0x00,0x6d,0x00,0x70,0x00,0x52,0x00,0x64,0x00,0x72,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x55,0x8b,0xec,0x83, + 0xec,0x14,0x56,0x8b,0x35,0x00,0x20,0x01,0x00,0x57,0x68,0x1c,0x20,0x01,0x00,0x8d,0x45,0xf4, + 0x50,0xff,0xd6,0x8b,0x7d,0x08,0x8d,0x45,0xfc,0x50,0x6a,0x00,0x6a,0x00,0x6a,0x22,0x8d,0x45, + 0xf4,0x50,0x6a,0x04,0x57,0xff,0x15,0x10,0x20,0x01,0x00,0x85,0xc0,0x75,0x4f,0x68,0x3c,0x20, + 0x01,0x00,0x8d,0x45,0xec,0x50,0xff,0xd6,0x8d,0x45,0xf4,0x50,0x8d,0x45,0xec,0x50,0xff,0x15, + 0x0c,0x20,0x01,0x00,0x8b,0xf0,0x85,0xf6,0x74,0x0d,0xff,0x75,0xfc,0xff,0x15,0x04,0x20,0x01, + 0x00,0x8b,0xc6,0xeb,0x23,0x8b,0x45,0xfc,0xa3,0x00,0x30,0x01,0x00,0xb8,0x00,0x10,0x01,0x00, + 0x89,0x47,0x38,0x89,0x47,0x40,0xc7,0x47,0x34,0x75,0x10,0x01,0x00,0xc7,0x47,0x70,0x19,0x10, + 0x01,0x00,0x33,0xc0,0x5f,0x5e,0xc9,0xc2,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x28,0x50,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xc6,0x50, + 0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x5c,0x50,0x00,0x00,0x74,0x50,0x00,0x00,0x86,0x50, + 0x00,0x00,0x9c,0x50,0x00,0x00,0xb4,0x50,0x00,0x00,0x44,0x50,0x00,0x00,0x00,0x00,0x00,0x00, + 0x4b,0x01,0x49,0x6f,0x44,0x65,0x6c,0x65,0x74,0x65,0x53,0x79,0x6d,0x62,0x6f,0x6c,0x69,0x63, + 0x4c,0x69,0x6e,0x6b,0x00,0x00,0x0b,0x04,0x52,0x74,0x6c,0x49,0x6e,0x69,0x74,0x55,0x6e,0x69, + 0x63,0x6f,0x64,0x65,0x53,0x74,0x72,0x69,0x6e,0x67,0x00,0x00,0x49,0x01,0x49,0x6f,0x44,0x65, + 0x6c,0x65,0x74,0x65,0x44,0x65,0x76,0x69,0x63,0x65,0x00,0x00,0xda,0x01,0x49,0x6f,0x66,0x43, + 0x6f,0x6d,0x70,0x6c,0x65,0x74,0x65,0x52,0x65,0x71,0x75,0x65,0x73,0x74,0x00,0x00,0x41,0x01, + 0x49,0x6f,0x43,0x72,0x65,0x61,0x74,0x65,0x53,0x79,0x6d,0x62,0x6f,0x6c,0x69,0x63,0x4c,0x69, + 0x6e,0x6b,0x00,0x00,0x38,0x01,0x49,0x6f,0x43,0x72,0x65,0x61,0x74,0x65,0x44,0x65,0x76,0x69, + 0x63,0x65,0x00,0x00,0x6e,0x74,0x6f,0x73,0x6b,0x72,0x6e,0x6c,0x2e,0x65,0x78,0x65,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x10,0x00,0x00,0x00,0x18,0x00,0x00,0x80, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x01,0x00, + 0x00,0x00,0x30,0x00,0x00,0x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x01,0x00,0x09,0x04,0x00,0x00,0x48,0x00,0x00,0x00,0x60,0x60,0x00,0x00,0x5c,0x03, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x5c,0x03,0x34,0x00,0x00,0x00,0x56,0x00,0x53,0x00,0x5f,0x00,0x56,0x00,0x45,0x00,0x52,0x00, + 0x53,0x00,0x49,0x00,0x4f,0x00,0x4e,0x00,0x5f,0x00,0x49,0x00,0x4e,0x00,0x46,0x00,0x4f,0x00, + 0x00,0x00,0x00,0x00,0xbd,0x04,0xef,0xfe,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00, + 0x00,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x04,0x00,0x04,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0xba,0x02,0x00,0x00,0x01,0x00,0x53,0x00,0x74,0x00,0x72,0x00,0x69,0x00,0x6e,0x00, + 0x67,0x00,0x46,0x00,0x69,0x00,0x6c,0x00,0x65,0x00,0x49,0x00,0x6e,0x00,0x66,0x00,0x6f,0x00, + 0x00,0x00,0x96,0x02,0x00,0x00,0x01,0x00,0x30,0x00,0x34,0x00,0x30,0x00,0x39,0x00,0x30,0x00, + 0x34,0x00,0x62,0x00,0x30,0x00,0x00,0x00,0x58,0x00,0x20,0x00,0x01,0x00,0x43,0x00,0x6f,0x00, + 0x6d,0x00,0x6d,0x00,0x65,0x00,0x6e,0x00,0x74,0x00,0x73,0x00,0x00,0x00,0x4d,0x00,0x53,0x00, + 0x52,0x00,0x20,0x00,0x72,0x00,0x65,0x00,0x61,0x00,0x64,0x00,0x65,0x00,0x72,0x00,0x20,0x00, + 0x33,0x00,0x32,0x00,0x2d,0x00,0x62,0x00,0x69,0x00,0x74,0x00,0x20,0x00,0x6b,0x00,0x65,0x00, + 0x72,0x00,0x6e,0x00,0x65,0x00,0x6c,0x00,0x20,0x00,0x64,0x00,0x72,0x00,0x69,0x00,0x76,0x00, + 0x65,0x00,0x72,0x00,0x00,0x00,0x42,0x00,0x11,0x00,0x01,0x00,0x43,0x00,0x6f,0x00,0x6d,0x00, + 0x70,0x00,0x61,0x00,0x6e,0x00,0x79,0x00,0x4e,0x00,0x61,0x00,0x6d,0x00,0x65,0x00,0x00,0x00, + 0x00,0x00,0x49,0x00,0x72,0x00,0x6f,0x00,0x6e,0x00,0x20,0x00,0x53,0x00,0x74,0x00,0x65,0x00, + 0x65,0x00,0x64,0x00,0x73,0x00,0x20,0x00,0x49,0x00,0x6e,0x00,0x63,0x00,0x2e,0x00,0x00,0x00, + 0x00,0x00,0x60,0x00,0x1c,0x00,0x01,0x00,0x46,0x00,0x69,0x00,0x6c,0x00,0x65,0x00,0x44,0x00, + 0x65,0x00,0x73,0x00,0x63,0x00,0x72,0x00,0x69,0x00,0x70,0x00,0x74,0x00,0x69,0x00,0x6f,0x00, + 0x6e,0x00,0x00,0x00,0x00,0x00,0x54,0x00,0x6d,0x00,0x70,0x00,0x52,0x00,0x64,0x00,0x72,0x00, + 0x20,0x00,0x33,0x00,0x32,0x00,0x2d,0x00,0x62,0x00,0x69,0x00,0x74,0x00,0x20,0x00,0x4b,0x00, + 0x65,0x00,0x72,0x00,0x6e,0x00,0x65,0x00,0x6c,0x00,0x20,0x00,0x4d,0x00,0x6f,0x00,0x64,0x00, + 0x75,0x00,0x6c,0x00,0x65,0x00,0x00,0x00,0x36,0x00,0x0b,0x00,0x01,0x00,0x46,0x00,0x69,0x00, + 0x6c,0x00,0x65,0x00,0x56,0x00,0x65,0x00,0x72,0x00,0x73,0x00,0x69,0x00,0x6f,0x00,0x6e,0x00, + 0x00,0x00,0x00,0x00,0x31,0x00,0x2c,0x00,0x20,0x00,0x30,0x00,0x2c,0x00,0x20,0x00,0x30,0x00, + 0x2c,0x00,0x20,0x00,0x31,0x00,0x00,0x00,0x00,0x00,0x2e,0x00,0x07,0x00,0x01,0x00,0x49,0x00, + 0x6e,0x00,0x74,0x00,0x65,0x00,0x72,0x00,0x6e,0x00,0x61,0x00,0x6c,0x00,0x4e,0x00,0x61,0x00, + 0x6d,0x00,0x65,0x00,0x00,0x00,0x54,0x00,0x6d,0x00,0x70,0x00,0x52,0x00,0x64,0x00,0x72,0x00, + 0x00,0x00,0x00,0x00,0x4a,0x00,0x13,0x00,0x01,0x00,0x4c,0x00,0x65,0x00,0x67,0x00,0x61,0x00, + 0x6c,0x00,0x43,0x00,0x6f,0x00,0x70,0x00,0x79,0x00,0x72,0x00,0x69,0x00,0x67,0x00,0x68,0x00, + 0x74,0x00,0x00,0x00,0x4e,0x00,0x69,0x00,0x63,0x00,0x6b,0x00,0x20,0x00,0x47,0x00,0x61,0x00, + 0x62,0x00,0x61,0x00,0x72,0x00,0x65,0x00,0x76,0x00,0x20,0x00,0x27,0x00,0x32,0x00,0x30,0x00, + 0x30,0x00,0x39,0x00,0x00,0x00,0x00,0x00,0x3e,0x00,0x0b,0x00,0x01,0x00,0x4f,0x00,0x72,0x00, + 0x69,0x00,0x67,0x00,0x69,0x00,0x6e,0x00,0x61,0x00,0x6c,0x00,0x46,0x00,0x69,0x00,0x6c,0x00, + 0x65,0x00,0x6e,0x00,0x61,0x00,0x6d,0x00,0x65,0x00,0x00,0x00,0x54,0x00,0x6d,0x00,0x70,0x00, + 0x52,0x00,0x64,0x00,0x72,0x00,0x2e,0x00,0x73,0x00,0x79,0x00,0x73,0x00,0x00,0x00,0x00,0x00, + 0x54,0x00,0x1a,0x00,0x01,0x00,0x50,0x00,0x72,0x00,0x6f,0x00,0x64,0x00,0x75,0x00,0x63,0x00, + 0x74,0x00,0x4e,0x00,0x61,0x00,0x6d,0x00,0x65,0x00,0x00,0x00,0x00,0x00,0x43,0x00,0x6f,0x00, + 0x72,0x00,0x65,0x00,0x20,0x00,0x32,0x00,0x20,0x00,0x54,0x00,0x65,0x00,0x6d,0x00,0x70,0x00, + 0x65,0x00,0x72,0x00,0x61,0x00,0x74,0x00,0x75,0x00,0x72,0x00,0x65,0x00,0x20,0x00,0x52,0x00, + 0x65,0x00,0x61,0x00,0x64,0x00,0x65,0x00,0x72,0x00,0x00,0x00,0x3a,0x00,0x0b,0x00,0x01,0x00, + 0x50,0x00,0x72,0x00,0x6f,0x00,0x64,0x00,0x75,0x00,0x63,0x00,0x74,0x00,0x56,0x00,0x65,0x00, + 0x72,0x00,0x73,0x00,0x69,0x00,0x6f,0x00,0x6e,0x00,0x00,0x00,0x31,0x00,0x2c,0x00,0x20,0x00, + 0x30,0x00,0x2c,0x00,0x20,0x00,0x30,0x00,0x2c,0x00,0x20,0x00,0x31,0x00,0x00,0x00,0x00,0x00, + 0x44,0x00,0x00,0x00,0x01,0x00,0x56,0x00,0x61,0x00,0x72,0x00,0x46,0x00,0x69,0x00,0x6c,0x00, + 0x65,0x00,0x49,0x00,0x6e,0x00,0x66,0x00,0x6f,0x00,0x00,0x00,0x00,0x00,0x24,0x00,0x04,0x00, + 0x00,0x00,0x54,0x00,0x72,0x00,0x61,0x00,0x6e,0x00,0x73,0x00,0x6c,0x00,0x61,0x00,0x74,0x00, + 0x69,0x00,0x6f,0x00,0x6e,0x00,0x00,0x00,0x00,0x00,0x09,0x04,0xb0,0x04,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x10,0x00,0x00,0x14,0x00,0x00,0x00, + 0x10,0x30,0x5c,0x30,0x82,0x30,0x87,0x30,0x91,0x30,0x9b,0x30,0x00,0x40,0x00,0x00,0x1c,0x00, + 0x00,0x00,0x09,0x30,0x0f,0x30,0x2f,0x30,0x38,0x30,0x4c,0x30,0x5b,0x30,0x67,0x30,0x6c,0x30, + 0x79,0x30,0x80,0x30,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +}; +int cc_x64driver_code_size = 5120; +uint8_t cc_x64driver_code[5120] = { + 0x4d,0x5a,0x90,0x00,0x03,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0xff,0xff,0x00,0x00,0xb8,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0xd0,0x00,0x00,0x00,0x0e,0x1f,0xba,0x0e,0x00,0xb4,0x09,0xcd, + 0x21,0xb8,0x01,0x4c,0xcd,0x21,0x54,0x68,0x69,0x73,0x20,0x70,0x72,0x6f,0x67,0x72,0x61,0x6d, + 0x20,0x63,0x61,0x6e,0x6e,0x6f,0x74,0x20,0x62,0x65,0x20,0x72,0x75,0x6e,0x20,0x69,0x6e,0x20, + 0x44,0x4f,0x53,0x20,0x6d,0x6f,0x64,0x65,0x2e,0x0d,0x0d,0x0a,0x24,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0xb7,0x04,0xa8,0xc2,0xf3,0x65,0xc6,0x91,0xf3,0x65,0xc6,0x91,0xf3,0x65,0xc6,0x91, + 0xf3,0x65,0xc7,0x91,0xf4,0x65,0xc6,0x91,0x85,0xf8,0xbd,0x91,0xf0,0x65,0xc6,0x91,0x85,0xf8, + 0xab,0x91,0xf0,0x65,0xc6,0x91,0x30,0x6a,0x98,0x91,0xf2,0x65,0xc6,0x91,0x85,0xf8,0xbe,0x91, + 0xf2,0x65,0xc6,0x91,0x52,0x69,0x63,0x68,0xf3,0x65,0xc6,0x91,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x50,0x45,0x00,0x00,0x64,0x86,0x07,0x00, + 0x41,0xc8,0x6d,0x49,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xf0,0x00,0x22,0x00,0x0b,0x02, + 0x08,0x00,0x00,0x06,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x00,0x00,0x00,0x10,0x60,0x00,0x00, + 0x00,0x10,0x00,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x02, + 0x00,0x00,0x05,0x00,0x02,0x00,0x05,0x00,0x02,0x00,0x05,0x00,0x02,0x00,0x00,0x00,0x00,0x00, + 0x00,0x80,0x00,0x00,0x00,0x04,0x00,0x00,0x79,0x44,0x00,0x00,0x01,0x00,0x00,0x00,0x00,0x00, + 0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x10,0x00, + 0x00,0x00,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x10,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x6c,0x60,0x00,0x00,0x28,0x00,0x00,0x00, + 0x00,0x70,0x00,0x00,0xc0,0x03,0x00,0x00,0x00,0x40,0x00,0x00,0x48,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x40,0x20,0x00,0x00, + 0x1c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x00,0x38,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x2e,0x74,0x65,0x78,0x74,0x00,0x00,0x00,0x26,0x01,0x00,0x00,0x00,0x10, + 0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x20,0x00,0x00,0x68,0x2e,0x72,0x64,0x61,0x74,0x61,0x00,0x00,0xf0,0x00, + 0x00,0x00,0x00,0x20,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x40,0x00,0x00,0x48,0x2e,0x64,0x61,0x74,0x61,0x00, + 0x00,0x00,0x18,0x01,0x00,0x00,0x00,0x30,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x08,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x40,0x00,0x00,0xc8,0x2e,0x70, + 0x64,0x61,0x74,0x61,0x00,0x00,0x48,0x00,0x00,0x00,0x00,0x40,0x00,0x00,0x00,0x02,0x00,0x00, + 0x00,0x0a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x40,0x00, + 0x00,0x48,0x50,0x41,0x47,0x45,0x30,0x44,0x45,0x46,0x4e,0x01,0x00,0x00,0x00,0x50,0x00,0x00, + 0x00,0x02,0x00,0x00,0x00,0x0c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x20,0x00,0x00,0x60,0x49,0x4e,0x49,0x54,0x00,0x00,0x00,0x00,0x60,0x01,0x00,0x00, + 0x00,0x60,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x0e,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x20,0x00,0x00,0xe2,0x2e,0x72,0x73,0x72,0x63,0x00,0x00,0x00, + 0xc0,0x03,0x00,0x00,0x00,0x70,0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x10,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x40,0x00,0x00,0x42,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x83,0xec,0x28, + 0x33,0xc9,0x48,0x8b,0xc2,0x89,0x4a,0x30,0x48,0x89,0x4a,0x38,0x33,0xd2,0x48,0x8b,0xc8,0xff, + 0x15,0xfd,0x0f,0x00,0x00,0x33,0xc0,0x48,0x83,0xc4,0x28,0xc3,0xcc,0xcc,0xcc,0xcc,0xcc,0xcc, + 0xcc,0xcc,0xcc,0xcc,0xcc,0xcc,0xcc,0xcc,0x48,0x83,0xec,0x28,0x48,0x8b,0x82,0xb8,0x00,0x00, + 0x00,0x4c,0x8b,0xca,0x81,0x78,0x18,0x0c,0xe0,0x22,0x00,0x75,0x43,0x83,0x78,0x08,0x08,0x72, + 0x3d,0x83,0x78,0x10,0x04,0x75,0x08,0x48,0x8b,0x42,0x18,0x8b,0x08,0xeb,0x05,0xb9,0x9c,0x01, + 0x00,0x00,0x4c,0x8b,0x42,0x18,0x0f,0x32,0x48,0xc1,0xe2,0x20,0x49,0x8b,0xc9,0x48,0x0b,0xc2, + 0x33,0xd2,0x49,0x89,0x00,0x49,0xc7,0x41,0x38,0x08,0x00,0x00,0x00,0xff,0x15,0x95,0x0f,0x00, + 0x00,0x33,0xc0,0x48,0x83,0xc4,0x28,0xc3,0xc7,0x42,0x30,0x01,0x00,0x00,0xc0,0x48,0xc7,0x42, + 0x38,0x00,0x00,0x00,0x00,0x49,0x8b,0xc9,0x33,0xd2,0xff,0x15,0x74,0x0f,0x00,0x00,0xb8,0x01, + 0x00,0x00,0xc0,0x48,0x83,0xc4,0x28,0xc3,0xcc,0xcc,0xcc,0xcc,0xcc,0xcc,0xcc,0xcc,0xcc,0xcc, + 0xcc,0xcc,0xcc,0xcc,0xcc,0xcc,0xcc,0xcc,0x48,0x83,0xec,0x38,0x48,0x8b,0x49,0x08,0xff,0x15, + 0x32,0x0f,0x00,0x00,0x48,0x8d,0x15,0x1b,0x00,0x00,0x00,0x48,0x8d,0x4c,0x24,0x20,0xff,0x15, + 0x18,0x0f,0x00,0x00,0x48,0x8d,0x4c,0x24,0x20,0xff,0x15,0x05,0x0f,0x00,0x00,0x48,0x83,0xc4, + 0x38,0xc3,0x5c,0x00,0x44,0x00,0x6f,0x00,0x73,0x00,0x44,0x00,0x65,0x00,0x76,0x00,0x69,0x00, + 0x63,0x00,0x65,0x00,0x73,0x00,0x5c,0x00,0x54,0x00,0x6d,0x00,0x70,0x00,0x52,0x00,0x64,0x00, + 0x72,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0xe6,0x60,0x00,0x00,0x00,0x00,0x00,0x00,0xfe,0x60,0x00,0x00, + 0x00,0x00,0x00,0x00,0x16,0x61,0x00,0x00,0x00,0x00,0x00,0x00,0x28,0x61,0x00,0x00,0x00,0x00, + 0x00,0x00,0x40,0x61,0x00,0x00,0x00,0x00,0x00,0x00,0xd0,0x60,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x41,0xc8,0x6d,0x49,0x00,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x49,0x00,0x00,0x00, + 0x5c,0x20,0x00,0x00,0x5c,0x06,0x00,0x00,0x52,0x53,0x44,0x53,0xd9,0x5e,0xab,0x47,0xc4,0xf2, + 0x4f,0x40,0xaa,0xe9,0x90,0x47,0x67,0x30,0xa5,0xfa,0x03,0x00,0x00,0x00,0x44,0x3a,0x5c,0x74, + 0x6d,0x70,0x5c,0x4b,0x65,0x72,0x6e,0x65,0x6c,0x5c,0x6f,0x62,0x6a,0x66,0x72,0x65,0x5f,0x77, + 0x6e,0x65,0x74,0x5f,0x41,0x4d,0x44,0x36,0x34,0x5c,0x61,0x6d,0x64,0x36,0x34,0x5c,0x54,0x6d, + 0x70,0x52,0x64,0x72,0x2e,0x70,0x64,0x62,0x00,0x00,0x00,0x00,0x01,0x04,0x01,0x00,0x04,0x42, + 0x00,0x00,0x01,0x04,0x01,0x00,0x04,0x42,0x00,0x00,0x01,0x04,0x01,0x00,0x04,0x62,0x00,0x00, + 0x21,0x00,0x00,0x00,0x10,0x50,0x00,0x00,0x74,0x50,0x00,0x00,0xe4,0x20,0x00,0x00,0x21,0x08, + 0x02,0x00,0x08,0x74,0x13,0x00,0x10,0x50,0x00,0x00,0x74,0x50,0x00,0x00,0xe4,0x20,0x00,0x00, + 0x01,0x0c,0x03,0x00,0x0c,0x34,0x12,0x00,0x04,0xe2,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0xcd,0x5d,0x20,0xd2,0x66,0xd4,0xff,0xff,0x32,0xa2,0xdf,0x2d,0x99,0x2b,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x10,0x10,0x00,0x00,0x32,0x10,0x00,0x00,0xa8,0x20,0x00,0x00,0x40,0x10, + 0x00,0x00,0xbe,0x10,0x00,0x00,0xb0,0x20,0x00,0x00,0xd0,0x10,0x00,0x00,0x00,0x11,0x00,0x00, + 0xb8,0x20,0x00,0x00,0x10,0x50,0x00,0x00,0x74,0x50,0x00,0x00,0xe4,0x20,0x00,0x00,0x74,0x50, + 0x00,0x00,0xe8,0x50,0x00,0x00,0xd0,0x20,0x00,0x00,0xe8,0x50,0x00,0x00,0xf5,0x50,0x00,0x00, + 0xc0,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x48,0x83,0xec,0x78,0x48,0x89,0x9c,0x24, + 0x90,0x00,0x00,0x00,0x48,0x8b,0xd9,0x48,0x8d,0x15,0x0a,0x01,0x00,0x00,0x48,0x8d,0x4c,0x24, + 0x48,0xff,0x15,0xd7,0xcf,0xff,0xff,0x41,0xb9,0x22,0x00,0x00,0x00,0x4c,0x8d,0x5c,0x24,0x40, + 0x4c,0x89,0x5c,0x24,0x30,0x4c,0x8d,0x44,0x24,0x48,0x41,0x8d,0x51,0xe6,0x48,0x8b,0xcb,0xc6, + 0x44,0x24,0x28,0x00,0xc7,0x44,0x24,0x20,0x00,0x00,0x00,0x00,0xff,0x15,0xc0,0xcf,0xff,0xff, + 0x85,0xc0,0x0f,0x85,0x80,0x00,0x00,0x00,0x48,0x8d,0x15,0x91,0x00,0x00,0x00,0x48,0x8d,0x4c, + 0x24,0x58,0x48,0x89,0xbc,0x24,0x98,0x00,0x00,0x00,0xff,0x15,0x86,0xcf,0xff,0xff,0x48,0x8d, + 0x54,0x24,0x48,0x48,0x8d,0x4c,0x24,0x58,0xff,0x15,0x86,0xcf,0xff,0xff,0x85,0xc0,0x8b,0xf8, + 0x74,0x0f,0x48,0x8b,0x4c,0x24,0x40,0xff,0x15,0x6d,0xcf,0xff,0xff,0x8b,0xc7,0xeb,0x39,0x48, + 0x8b,0x44,0x24,0x40,0x48,0x89,0x05,0x5d,0xe0,0xff,0xff,0x48,0x8d,0x05,0x16,0xc0,0xff,0xff, + 0x48,0x89,0x43,0x68,0x48,0x8d,0x05,0x4b,0xbf,0xff,0xff,0x48,0x89,0x43,0x70,0x48,0x89,0x83, + 0x80,0x00,0x00,0x00,0x48,0x8d,0x05,0x69,0xbf,0xff,0xff,0x48,0x89,0x83,0xe0,0x00,0x00,0x00, + 0x33,0xc0,0x48,0x8b,0xbc,0x24,0x98,0x00,0x00,0x00,0x48,0x8b,0x9c,0x24,0x90,0x00,0x00,0x00, + 0x48,0x83,0xc4,0x78,0xc3,0xcc,0xcc,0xcc,0xcc,0xcc,0xcc,0xcc,0xcc,0xcc,0xcc,0xcc,0x5c,0x00, + 0x44,0x00,0x6f,0x00,0x73,0x00,0x44,0x00,0x65,0x00,0x76,0x00,0x69,0x00,0x63,0x00,0x65,0x00, + 0x73,0x00,0x5c,0x00,0x54,0x00,0x6d,0x00,0x70,0x00,0x52,0x00,0x64,0x00,0x72,0x00,0x00,0x00, + 0xcc,0xcc,0xcc,0xcc,0xcc,0xcc,0xcc,0xcc,0xcc,0xcc,0x5c,0x00,0x44,0x00,0x65,0x00,0x76,0x00, + 0x69,0x00,0x63,0x00,0x65,0x00,0x5c,0x00,0x54,0x00,0x6d,0x00,0x70,0x00,0x52,0x00,0x64,0x00, + 0x72,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x48,0x8b,0x05,0xf1,0xd0,0xff,0xff,0x49,0xb9,0x32,0xa2,0xdf,0x2d,0x99,0x2b,0x00,0x00,0x48, + 0x85,0xc0,0x74,0x05,0x49,0x3b,0xc1,0x75,0x2f,0x4c,0x8d,0x05,0xd6,0xd0,0xff,0xff,0x48,0xb8, + 0x20,0x03,0x00,0x00,0x80,0xf7,0xff,0xff,0x48,0x8b,0x00,0x49,0x33,0xc0,0x49,0xb8,0xff,0xff, + 0xff,0xff,0xff,0xff,0x00,0x00,0x49,0x23,0xc0,0x49,0x0f,0x44,0xc1,0x48,0x89,0x05,0xae,0xd0, + 0xff,0xff,0x48,0xf7,0xd0,0x48,0x89,0x05,0x9c,0xd0,0xff,0xff,0xe9,0xa7,0xef,0xff,0xff,0xcc, + 0xcc,0xcc,0x98,0x60,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x52,0x61,0x00,0x00, + 0x00,0x20,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xe6,0x60,0x00,0x00,0x00,0x00,0x00,0x00, + 0xfe,0x60,0x00,0x00,0x00,0x00,0x00,0x00,0x16,0x61,0x00,0x00,0x00,0x00,0x00,0x00,0x28,0x61, + 0x00,0x00,0x00,0x00,0x00,0x00,0x40,0x61,0x00,0x00,0x00,0x00,0x00,0x00,0xd0,0x60,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xf6,0x01,0x49,0x6f,0x66,0x43, + 0x6f,0x6d,0x70,0x6c,0x65,0x74,0x65,0x52,0x65,0x71,0x75,0x65,0x73,0x74,0x00,0x00,0x61,0x01, + 0x49,0x6f,0x44,0x65,0x6c,0x65,0x74,0x65,0x53,0x79,0x6d,0x62,0x6f,0x6c,0x69,0x63,0x4c,0x69, + 0x6e,0x6b,0x00,0x00,0x3e,0x04,0x52,0x74,0x6c,0x49,0x6e,0x69,0x74,0x55,0x6e,0x69,0x63,0x6f, + 0x64,0x65,0x53,0x74,0x72,0x69,0x6e,0x67,0x00,0x00,0x5f,0x01,0x49,0x6f,0x44,0x65,0x6c,0x65, + 0x74,0x65,0x44,0x65,0x76,0x69,0x63,0x65,0x00,0x00,0x55,0x01,0x49,0x6f,0x43,0x72,0x65,0x61, + 0x74,0x65,0x53,0x79,0x6d,0x62,0x6f,0x6c,0x69,0x63,0x4c,0x69,0x6e,0x6b,0x00,0x00,0x4c,0x01, + 0x49,0x6f,0x43,0x72,0x65,0x61,0x74,0x65,0x44,0x65,0x76,0x69,0x63,0x65,0x00,0x00,0x6e,0x74, + 0x6f,0x73,0x6b,0x72,0x6e,0x6c,0x2e,0x65,0x78,0x65,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x10,0x00,0x00,0x00,0x18,0x00,0x00,0x80,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0x00,0x00, + 0x30,0x00,0x00,0x80,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x01,0x00,0x09,0x04,0x00,0x00,0x48,0x00,0x00,0x00,0x60,0x70,0x00,0x00,0x60,0x03,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x60,0x03, + 0x34,0x00,0x00,0x00,0x56,0x00,0x53,0x00,0x5f,0x00,0x56,0x00,0x45,0x00,0x52,0x00,0x53,0x00, + 0x49,0x00,0x4f,0x00,0x4e,0x00,0x5f,0x00,0x49,0x00,0x4e,0x00,0x46,0x00,0x4f,0x00,0x00,0x00, + 0x00,0x00,0xbd,0x04,0xef,0xfe,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x00,0x00,0x00, + 0x00,0x00,0x01,0x00,0x01,0x00,0x00,0x00,0x17,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x04,0x00, + 0x04,0x00,0x03,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0xbe,0x02,0x00,0x00,0x01,0x00,0x53,0x00,0x74,0x00,0x72,0x00,0x69,0x00,0x6e,0x00,0x67,0x00, + 0x46,0x00,0x69,0x00,0x6c,0x00,0x65,0x00,0x49,0x00,0x6e,0x00,0x66,0x00,0x6f,0x00,0x00,0x00, + 0x9a,0x02,0x00,0x00,0x01,0x00,0x30,0x00,0x34,0x00,0x30,0x00,0x39,0x00,0x30,0x00,0x34,0x00, + 0x62,0x00,0x30,0x00,0x00,0x00,0x58,0x00,0x20,0x00,0x01,0x00,0x43,0x00,0x6f,0x00,0x6d,0x00, + 0x6d,0x00,0x65,0x00,0x6e,0x00,0x74,0x00,0x73,0x00,0x00,0x00,0x4d,0x00,0x53,0x00,0x52,0x00, + 0x20,0x00,0x72,0x00,0x65,0x00,0x61,0x00,0x64,0x00,0x65,0x00,0x72,0x00,0x20,0x00,0x36,0x00, + 0x34,0x00,0x2d,0x00,0x62,0x00,0x69,0x00,0x74,0x00,0x20,0x00,0x6b,0x00,0x65,0x00,0x72,0x00, + 0x6e,0x00,0x65,0x00,0x6c,0x00,0x20,0x00,0x64,0x00,0x72,0x00,0x69,0x00,0x76,0x00,0x65,0x00, + 0x72,0x00,0x00,0x00,0x42,0x00,0x11,0x00,0x01,0x00,0x43,0x00,0x6f,0x00,0x6d,0x00,0x70,0x00, + 0x61,0x00,0x6e,0x00,0x79,0x00,0x4e,0x00,0x61,0x00,0x6d,0x00,0x65,0x00,0x00,0x00,0x00,0x00, + 0x49,0x00,0x72,0x00,0x6f,0x00,0x6e,0x00,0x20,0x00,0x53,0x00,0x74,0x00,0x65,0x00,0x65,0x00, + 0x64,0x00,0x73,0x00,0x20,0x00,0x49,0x00,0x6e,0x00,0x63,0x00,0x2e,0x00,0x00,0x00,0x00,0x00, + 0x60,0x00,0x1c,0x00,0x01,0x00,0x46,0x00,0x69,0x00,0x6c,0x00,0x65,0x00,0x44,0x00,0x65,0x00, + 0x73,0x00,0x63,0x00,0x72,0x00,0x69,0x00,0x70,0x00,0x74,0x00,0x69,0x00,0x6f,0x00,0x6e,0x00, + 0x00,0x00,0x00,0x00,0x54,0x00,0x6d,0x00,0x70,0x00,0x52,0x00,0x64,0x00,0x72,0x00,0x20,0x00, + 0x36,0x00,0x34,0x00,0x2d,0x00,0x62,0x00,0x69,0x00,0x74,0x00,0x20,0x00,0x4b,0x00,0x65,0x00, + 0x72,0x00,0x6e,0x00,0x65,0x00,0x6c,0x00,0x20,0x00,0x4d,0x00,0x6f,0x00,0x64,0x00,0x75,0x00, + 0x6c,0x00,0x65,0x00,0x00,0x00,0x36,0x00,0x0b,0x00,0x01,0x00,0x46,0x00,0x69,0x00,0x6c,0x00, + 0x65,0x00,0x56,0x00,0x65,0x00,0x72,0x00,0x73,0x00,0x69,0x00,0x6f,0x00,0x6e,0x00,0x00,0x00, + 0x00,0x00,0x31,0x00,0x2c,0x00,0x20,0x00,0x30,0x00,0x2c,0x00,0x20,0x00,0x30,0x00,0x2c,0x00, + 0x20,0x00,0x31,0x00,0x00,0x00,0x00,0x00,0x2e,0x00,0x07,0x00,0x01,0x00,0x49,0x00,0x6e,0x00, + 0x74,0x00,0x65,0x00,0x72,0x00,0x6e,0x00,0x61,0x00,0x6c,0x00,0x4e,0x00,0x61,0x00,0x6d,0x00, + 0x65,0x00,0x00,0x00,0x54,0x00,0x6d,0x00,0x70,0x00,0x52,0x00,0x64,0x00,0x72,0x00,0x00,0x00, + 0x00,0x00,0x4a,0x00,0x13,0x00,0x01,0x00,0x4c,0x00,0x65,0x00,0x67,0x00,0x61,0x00,0x6c,0x00, + 0x43,0x00,0x6f,0x00,0x70,0x00,0x79,0x00,0x72,0x00,0x69,0x00,0x67,0x00,0x68,0x00,0x74,0x00, + 0x00,0x00,0x4e,0x00,0x69,0x00,0x63,0x00,0x6b,0x00,0x20,0x00,0x47,0x00,0x61,0x00,0x62,0x00, + 0x61,0x00,0x72,0x00,0x65,0x00,0x76,0x00,0x20,0x00,0x27,0x00,0x32,0x00,0x30,0x00,0x30,0x00, + 0x39,0x00,0x00,0x00,0x00,0x00,0x42,0x00,0x0d,0x00,0x01,0x00,0x4f,0x00,0x72,0x00,0x69,0x00, + 0x67,0x00,0x69,0x00,0x6e,0x00,0x61,0x00,0x6c,0x00,0x46,0x00,0x69,0x00,0x6c,0x00,0x65,0x00, + 0x6e,0x00,0x61,0x00,0x6d,0x00,0x65,0x00,0x00,0x00,0x54,0x00,0x6d,0x00,0x70,0x00,0x52,0x00, + 0x64,0x00,0x72,0x00,0x36,0x00,0x34,0x00,0x2e,0x00,0x73,0x00,0x79,0x00,0x73,0x00,0x00,0x00, + 0x00,0x00,0x54,0x00,0x1a,0x00,0x01,0x00,0x50,0x00,0x72,0x00,0x6f,0x00,0x64,0x00,0x75,0x00, + 0x63,0x00,0x74,0x00,0x4e,0x00,0x61,0x00,0x6d,0x00,0x65,0x00,0x00,0x00,0x00,0x00,0x43,0x00, + 0x6f,0x00,0x72,0x00,0x65,0x00,0x20,0x00,0x32,0x00,0x20,0x00,0x54,0x00,0x65,0x00,0x6d,0x00, + 0x70,0x00,0x65,0x00,0x72,0x00,0x61,0x00,0x74,0x00,0x75,0x00,0x72,0x00,0x65,0x00,0x20,0x00, + 0x52,0x00,0x65,0x00,0x61,0x00,0x64,0x00,0x65,0x00,0x72,0x00,0x00,0x00,0x3a,0x00,0x0b,0x00, + 0x01,0x00,0x50,0x00,0x72,0x00,0x6f,0x00,0x64,0x00,0x75,0x00,0x63,0x00,0x74,0x00,0x56,0x00, + 0x65,0x00,0x72,0x00,0x73,0x00,0x69,0x00,0x6f,0x00,0x6e,0x00,0x00,0x00,0x31,0x00,0x2c,0x00, + 0x20,0x00,0x30,0x00,0x2c,0x00,0x20,0x00,0x30,0x00,0x2c,0x00,0x20,0x00,0x31,0x00,0x00,0x00, + 0x00,0x00,0x44,0x00,0x00,0x00,0x01,0x00,0x56,0x00,0x61,0x00,0x72,0x00,0x46,0x00,0x69,0x00, + 0x6c,0x00,0x65,0x00,0x49,0x00,0x6e,0x00,0x66,0x00,0x6f,0x00,0x00,0x00,0x00,0x00,0x24,0x00, + 0x04,0x00,0x00,0x00,0x54,0x00,0x72,0x00,0x61,0x00,0x6e,0x00,0x73,0x00,0x6c,0x00,0x61,0x00, + 0x74,0x00,0x69,0x00,0x6f,0x00,0x6e,0x00,0x00,0x00,0x00,0x00,0x09,0x04,0xb0,0x04,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, + 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, +}; +//} end +#endif // _WIN32 + +int msrdriver_dummy; // a dummy to avoid a linker warning on OS X. diff --git a/contrib/libcpuid/include/libcpuid/rdmsr.c b/contrib/libcpuid/include/libcpuid/rdmsr.c new file mode 100644 index 00000000000..a27e939bba0 --- /dev/null +++ b/contrib/libcpuid/include/libcpuid/rdmsr.c @@ -0,0 +1,922 @@ +/* + * Copyright 2009 Veselin Georgiev, + * anrieffNOSPAM @ mgail_DOT.com (convert to gmail) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#define _XOPEN_SOURCE 500 +#include +#include +#include "libcpuid.h" +#include "asm-bits.h" +#include "libcpuid_util.h" +#include "libcpuid_internal.h" +#include "rdtsc.h" + +#if defined (__linux__) || defined (__gnu_linux__) +/* Assuming linux with /dev/cpu/x/msr: */ +#include +#include +#include +#include +#include +#include +struct msr_driver_t { int fd; }; +static int rdmsr_supported(void); +static int load_driver(char *msr_path) +{ + const int file_exists = !access(msr_path, F_OK); + const int file_readable = !access(msr_path, R_OK); + + if (file_exists && file_readable) + return 1; + else if (file_exists && !file_readable) + return 0; + else if (getuid() != 0) + return 0; + else + return !system("modprobe msr 2> /dev/null"); +} + +struct msr_driver_t* cpu_msr_driver_open(void) +{ + return cpu_msr_driver_open_core(0); +} + +struct msr_driver_t* cpu_msr_driver_open_core(unsigned core_num) +{ + char msr[32]; + struct msr_driver_t* handle; + if (core_num >= cpuid_get_total_cpus()) { + set_error(ERR_INVCNB); + return NULL; + } + if (!rdmsr_supported()) { + set_error(ERR_NO_RDMSR); + return NULL; + } + sprintf(msr, "/dev/cpu/%u/msr", core_num); + if(!load_driver(msr)) { + set_error(ERR_NO_DRIVER); + return NULL; + } + int fd = open(msr, O_RDONLY); + if (fd < 0) { + if (errno == EIO) { + set_error(ERR_NO_RDMSR); + return NULL; + } + set_error(ERR_NO_DRIVER); + return NULL; + } + handle = (struct msr_driver_t*) malloc(sizeof(struct msr_driver_t)); + handle->fd = fd; + return handle; +} + +int cpu_rdmsr(struct msr_driver_t* driver, uint32_t msr_index, uint64_t* result) +{ + ssize_t ret; + + if (!driver || driver->fd < 0) + return set_error(ERR_HANDLE); + ret = pread(driver->fd, result, 8, msr_index); + if (ret != 8) + return set_error(ERR_INVMSR); + return 0; +} + +int cpu_msr_driver_close(struct msr_driver_t* drv) +{ + if (drv && drv->fd >= 0) { + close(drv->fd); + free(drv); + } + return 0; +} + +/* #endif defined (__linux__) || defined (__gnu_linux__) */ + +#elif defined (__FreeBSD__) || defined (__DragonFly__) +/* Assuming FreeBSD with /dev/cpuctlX */ +#include +#include +#include +#include +#include + +struct msr_driver_t { int fd; }; +static int rdmsr_supported(void); +static int load_driver(char *msr_path) +{ + const int file_exists = !access(msr_path, F_OK); + const int file_readable = !access(msr_path, R_OK); + + if (file_exists && file_readable) + return 1; + else if (file_exists && !file_readable) + return 0; + else if (getuid() != 0) + return 0; + else + return !system("kldload -n cpuctl 2> /dev/null"); +} + +struct msr_driver_t* cpu_msr_driver_open(void) +{ + return cpu_msr_driver_open_core(0); +} + +struct msr_driver_t* cpu_msr_driver_open_core(unsigned core_num) +{ + char msr[32]; + struct msr_driver_t* handle; + if (core_num >= cpuid_get_total_cpus()) { + set_error(ERR_INVCNB); + return NULL; + } + if (!rdmsr_supported()) { + set_error(ERR_NO_RDMSR); + return NULL; + } + sprintf(msr, "/dev/cpuctl%u", core_num); + if(!load_driver(msr)) { + set_error(ERR_NO_DRIVER); + return NULL; + } + int fd = open(msr, O_RDONLY); + if (fd < 0) { + if (errno == EIO) { + set_error(ERR_NO_RDMSR); + return NULL; + } + set_error(ERR_NO_DRIVER); + return NULL; + } + handle = (struct msr_driver_t*) malloc(sizeof(struct msr_driver_t)); + handle->fd = fd; + return handle; +} + +int cpu_rdmsr(struct msr_driver_t* driver, uint32_t msr_index, uint64_t* result) +{ + cpuctl_msr_args_t args; + args.msr = msr_index; + + if (!driver || driver->fd < 0) + return set_error(ERR_HANDLE); + + if(ioctl(driver->fd, CPUCTL_RDMSR, &args)) + return set_error(ERR_INVMSR); + + *result = args.data; + return 0; +} + +int cpu_msr_driver_close(struct msr_driver_t* drv) +{ + if (drv && drv->fd >= 0) { + close(drv->fd); + free(drv); + } + return 0; +} + +/* #endif defined (__FreeBSD__) || defined (__DragonFly__) */ + +#elif defined (_WIN32) +#include +#include +#include + +extern uint8_t cc_x86driver_code[]; +extern int cc_x86driver_code_size; +extern uint8_t cc_x64driver_code[]; +extern int cc_x64driver_code_size; + +struct msr_driver_t { + char driver_path[MAX_PATH + 1]; + SC_HANDLE scManager; + volatile SC_HANDLE scDriver; + HANDLE hhDriver; + OVERLAPPED ovl; + int errorcode; +}; + +static int rdmsr_supported(void); +static int extract_driver(struct msr_driver_t* driver); +static int load_driver(struct msr_driver_t* driver); + +struct msr_driver_t* cpu_msr_driver_open(void) +{ + struct msr_driver_t* drv; + int status; + if (!rdmsr_supported()) { + set_error(ERR_NO_RDMSR); + return NULL; + } + + drv = (struct msr_driver_t*) malloc(sizeof(struct msr_driver_t)); + if (!drv) { + set_error(ERR_NO_MEM); + return NULL; + } + memset(drv, 0, sizeof(struct msr_driver_t)); + + if (!extract_driver(drv)) { + free(drv); + set_error(ERR_EXTRACT); + return NULL; + } + + status = load_driver(drv); + if (!DeleteFile(drv->driver_path)) + debugf(1, "Deleting temporary driver file failed.\n"); + if (!status) { + set_error(drv->errorcode ? drv->errorcode : ERR_NO_DRIVER); + free(drv); + return NULL; + } + return drv; +} + +struct msr_driver_t* cpu_msr_driver_open_core(unsigned core_num) +{ + warnf("cpu_msr_driver_open_core(): parameter ignored (function is the same as cpu_msr_driver_open)\n"); + return cpu_msr_driver_open(); +} + +typedef BOOL (WINAPI *LPFN_ISWOW64PROCESS) (HANDLE, PBOOL); +static BOOL is_running_x64(void) +{ + BOOL bIsWow64 = FALSE; + + LPFN_ISWOW64PROCESS fnIsWow64Process = (LPFN_ISWOW64PROCESS)GetProcAddress(GetModuleHandle(__TEXT("kernel32")), "IsWow64Process"); + if(NULL != fnIsWow64Process) + fnIsWow64Process(GetCurrentProcess(), &bIsWow64); + return bIsWow64; +} + + +static int extract_driver(struct msr_driver_t* driver) +{ + FILE *f; + if (!GetTempPath(sizeof(driver->driver_path), driver->driver_path)) return 0; + strcat(driver->driver_path, "TmpRdr.sys"); + + f = fopen(driver->driver_path, "wb"); + if (!f) return 0; + if (is_running_x64()) + fwrite(cc_x64driver_code, 1, cc_x64driver_code_size, f); + else + fwrite(cc_x86driver_code, 1, cc_x86driver_code_size, f); + fclose(f); + return 1; +} + +static BOOL wait_for_service_state(SC_HANDLE hService, DWORD dwDesiredState, SERVICE_STATUS *lpsrvStatus){ + BOOL fOK = FALSE; + DWORD dwWaitHint; + + if(hService != NULL){ + while(TRUE){ + fOK = QueryServiceStatus(hService, lpsrvStatus); + if(!fOK) + break; + if(lpsrvStatus->dwCurrentState == dwDesiredState) + break; + + dwWaitHint = lpsrvStatus->dwWaitHint / 10; // Poll 1/10 of the wait hint + if (dwWaitHint < 1000) + dwWaitHint = 1000; // At most once per second + if (dwWaitHint > 10000) + dwWaitHint = 10000; // At least every 10 seconds + Sleep(dwWaitHint); + } + } + + return fOK; +} + +static int load_driver(struct msr_driver_t* drv) +{ + LPTSTR lpszInfo = __TEXT("RDMSR Executor Driver"); + USHORT uLen = 0; + SERVICE_STATUS srvStatus = {0}; + BOOL fRunning = FALSE; + DWORD dwLastError; + LPTSTR lpszDriverServiceName = __TEXT("TmpRdr"); + TCHAR lpszDriverName[] = __TEXT("\\\\.\\Global\\TmpRdr"); + + if((LPVOID)(drv->scManager = OpenSCManager(NULL, NULL, SC_MANAGER_ALL_ACCESS)) != NULL) { + drv->scDriver = CreateService(drv->scManager, lpszDriverServiceName, lpszInfo, SERVICE_ALL_ACCESS, + SERVICE_KERNEL_DRIVER, SERVICE_DEMAND_START, SERVICE_ERROR_NORMAL, + drv->driver_path, NULL, NULL, NULL, NULL, NULL); + if(drv->scDriver == NULL){ + switch(dwLastError = GetLastError()){ + case ERROR_SERVICE_EXISTS: + case ERROR_SERVICE_MARKED_FOR_DELETE:{ + LPQUERY_SERVICE_CONFIG lpqsc; + DWORD dwBytesNeeded; + + drv->scDriver = OpenService(drv->scManager, lpszDriverServiceName, SERVICE_ALL_ACCESS); + if(drv->scDriver == NULL){ + debugf(1, "Error opening service: %d\n", GetLastError()); + break; + } + + QueryServiceConfig(drv->scDriver, NULL, 0, &dwBytesNeeded); + if((dwLastError = GetLastError()) == ERROR_INSUFFICIENT_BUFFER){ + lpqsc = calloc(1, dwBytesNeeded); + if(!QueryServiceConfig(drv->scDriver, lpqsc, dwBytesNeeded, &dwBytesNeeded)){ + free(lpqsc); + debugf(1, "Error query service config(adjusted buffer): %d\n", GetLastError()); + goto clean_up; + } + else{ + free(lpqsc); + } + } + else{ + debugf(1, "Error query service config: %d\n", dwLastError); + goto clean_up; + } + + break; + } + case ERROR_ACCESS_DENIED: + drv->errorcode = ERR_NO_PERMS; + break; + default: + debugf(1, "Create driver service failed: %d\n", dwLastError); + break; + } + } + if(drv->scDriver != NULL){ + if(StartService(drv->scDriver, 0, NULL)){ + if(!wait_for_service_state(drv->scDriver, SERVICE_RUNNING, &srvStatus)){ + debugf(1, "Driver load failed.\n"); + DeleteService(drv->scDriver); + CloseServiceHandle(drv->scManager); + drv->scDriver = NULL; + goto clean_up; + } else { + fRunning = TRUE; + } + } else{ + if((dwLastError = GetLastError()) == ERROR_SERVICE_ALREADY_RUNNING) + fRunning = TRUE; + else{ + debugf(1, "Driver start failed.\n"); + DeleteService(drv->scDriver); + CloseServiceHandle(drv->scManager); + drv->scDriver = NULL; + goto clean_up; + } + + } + if(fRunning) + debugf(1, "Driver already running.\n"); + else + debugf(1, "Driver loaded.\n"); + CloseServiceHandle(drv->scManager); + drv->hhDriver = CreateFile(lpszDriverName, GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_FLAG_OVERLAPPED, 0); + drv->ovl.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + return 1; + } + } else { + debugf(1, "Open SCM failed: %d\n", GetLastError()); + } + +clean_up: + if(drv->scManager != NULL){ + CloseServiceHandle(drv->scManager); + drv->scManager = 0; // pointless + } + if(drv->scDriver != NULL){ + if(!DeleteService(drv->scDriver)) + debugf(1, "Delete driver service failed: %d\n", GetLastError()); + CloseServiceHandle(drv->scDriver); + drv->scDriver = 0; + } + + return 0; +} + +#define FILE_DEVICE_UNKNOWN 0x00000022 +#define IOCTL_UNKNOWN_BASE FILE_DEVICE_UNKNOWN +#define IOCTL_PROCVIEW_RDMSR CTL_CODE(IOCTL_UNKNOWN_BASE, 0x0803, METHOD_BUFFERED, FILE_READ_ACCESS | FILE_WRITE_ACCESS) + +int cpu_rdmsr(struct msr_driver_t* driver, uint32_t msr_index, uint64_t* result) +{ + DWORD dwBytesReturned; + __int64 msrdata; + SERVICE_STATUS srvStatus = {0}; + + if (!driver) + return set_error(ERR_HANDLE); + DeviceIoControl(driver->hhDriver, IOCTL_PROCVIEW_RDMSR, &msr_index, sizeof(int), &msrdata, sizeof(__int64), &dwBytesReturned, &driver->ovl); + GetOverlappedResult(driver->hhDriver, &driver->ovl, &dwBytesReturned, TRUE); + *result = msrdata; + return 0; +} + +int cpu_msr_driver_close(struct msr_driver_t* drv) +{ + SERVICE_STATUS srvStatus = {0}; + if (drv == NULL) return 0; + if(drv->scDriver != NULL){ + if (drv->hhDriver) CancelIo(drv->hhDriver); + if(drv->ovl.hEvent != NULL) + CloseHandle(drv->ovl.hEvent); + if (drv->hhDriver) CloseHandle(drv->hhDriver); + drv->hhDriver = NULL; + drv->ovl.hEvent = NULL; + if (ControlService(drv->scDriver, SERVICE_CONTROL_STOP, &srvStatus)){ + if (wait_for_service_state(drv->scDriver, SERVICE_STOPPED, &srvStatus)){ + DeleteService(drv->scDriver); + } + } + } + return 0; +} + +/* endif defined (_WIN32) */ + +#else /* Unsupported OS */ +/* On others OS (i.e., Darwin), we still do not support RDMSR, so supply dummy struct + and functions */ + +#define RDMSR_UNSUPPORTED_OS + +struct msr_driver_t { int dummy; }; +struct msr_driver_t* cpu_msr_driver_open(void) +{ + set_error(ERR_NOT_IMP); + return NULL; +} + +struct msr_driver_t* cpu_msr_driver_open_core(unsigned core_num) +{ + set_error(ERR_NOT_IMP); + return NULL; +} + +int cpu_rdmsr(struct msr_driver_t* driver, uint32_t msr_index, uint64_t* result) +{ + return set_error(ERR_NOT_IMP); +} + +int cpu_msr_driver_close(struct msr_driver_t* driver) +{ + return set_error(ERR_NOT_IMP); +} + +int cpu_rdmsr_range(struct msr_driver_t* handle, uint32_t msr_index, uint8_t highbit, + uint8_t lowbit, uint64_t* result) +{ + return set_error(ERR_NOT_IMP); +} + +int cpu_msrinfo(struct msr_driver_t* driver, cpu_msrinfo_request_t which) +{ + return set_error(ERR_NOT_IMP); +} + +#endif /* Unsupported OS */ + +#ifndef RDMSR_UNSUPPORTED_OS + +/* Useful links for hackers: +- AMD MSRs: + AMD BIOS and Kernel Developer’s Guide (BKDG) + * AMD Family 10h Processors + http://support.amd.com/TechDocs/31116.pdf + * AMD Family 11h Processors + http://support.amd.com/TechDocs/41256.pdf + * AMD Family 12h Processors + http://support.amd.com/TechDocs/41131.pdf + * AMD Family 14h Processors + http://support.amd.com/TechDocs/43170_14h_Mod_00h-0Fh_BKDG.pdf + * AMD Family 15h Processors + http://support.amd.com/TechDocs/42301_15h_Mod_00h-0Fh_BKDG.pdf + http://support.amd.com/TechDocs/42300_15h_Mod_10h-1Fh_BKDG.pdf + http://support.amd.com/TechDocs/49125_15h_Models_30h-3Fh_BKDG.pdf + http://support.amd.com/TechDocs/50742_15h_Models_60h-6Fh_BKDG.pdf + http://support.amd.com/TechDocs/49125_15h_Models_30h-3Fh_BKDG.pdf + * AMD Family 16h Processors + http://support.amd.com/TechDocs/48751_16h_bkdg.pdf + http://support.amd.com/TechDocs/52740_16h_Models_30h-3Fh_BKDG.pdf + +- Intel MSRs: + Intel® 64 and IA-32 Architectures Software Developer’s Manual + * Volume 3 (3A, 3B, 3C & 3D): System Programming Guide + http://www.intel.com/content/dam/www/public/us/en/documents/manuals/64-ia-32-architectures-software-developer-system-programming-manual-325384.pdf +*/ + +/* AMD MSRs addresses */ +#define MSR_PSTATE_L 0xC0010061 +#define MSR_PSTATE_S 0xC0010063 +#define MSR_PSTATE_0 0xC0010064 +#define MSR_PSTATE_7 0xC001006B + +/* Intel MSRs addresses */ +#define IA32_MPERF 0xE7 +#define IA32_APERF 0xE8 +#define IA32_PERF_STATUS 0x198 +#define IA32_THERM_STATUS 0x19C +#define MSR_EBL_CR_POWERON 0x2A +#define MSR_TURBO_RATIO_LIMIT 0x1AD +#define MSR_TEMPERATURE_TARGET 0x1A2 +#define MSR_PERF_STATUS 0x198 +#define MSR_PLATFORM_INFO 0xCE + + +static int rdmsr_supported(void) +{ + struct cpu_id_t* id = get_cached_cpuid(); + return id->flags[CPU_FEATURE_MSR]; +} + +static int perfmsr_measure(struct msr_driver_t* handle, int msr) +{ + int err; + uint64_t a, b; + uint64_t x, y; + err = cpu_rdmsr(handle, msr, &x); + if (err) return CPU_INVALID_VALUE; + sys_precise_clock(&a); + busy_loop_delay(10); + cpu_rdmsr(handle, msr, &y); + sys_precise_clock(&b); + if (a >= b || x > y) return CPU_INVALID_VALUE; + return (int) ((y - x) / (b - a)); +} + +static int get_amd_multipliers(struct msr_driver_t* handle, struct cpu_id_t *id, + struct internal_id_info_t *internal, + uint32_t pstate, uint64_t *multiplier) +{ + int err; + static int clock = 0; + uint64_t CpuFid, CpuDid, CpuDidLSD; + double divisor; + + if (pstate < MSR_PSTATE_0 || MSR_PSTATE_7 < pstate) + return 1; + + switch (id->ext_family) { + case 0x11: + /* BKDG 11h, page 236 + MSRC001_00[6B:64][8:6] is CpuDid + MSRC001_00[6B:64][5:0] is CpuFid + CPU COF is ((100 MHz * (CpuFid + 08h)) / (2^CpuDid)) */ + err = cpu_rdmsr_range(handle, pstate, 8, 6, &CpuDid); + err += cpu_rdmsr_range(handle, pstate, 5, 0, &CpuFid); + *multiplier = (uint64_t) ((CpuFid + 0x8) / (1ull << CpuDid)); + break; + case 0x12: + /* BKDG 12h, page 469 + MSRC001_00[6B:64][8:4] is CpuFid + MSRC001_00[6B:64][3:0] is CpuDid + CPU COF is (100MHz * (CpuFid + 10h) / (divisor specified by CpuDid)) */ + err = cpu_rdmsr_range(handle, pstate, 8, 4, &CpuFid); + err += cpu_rdmsr_range(handle, pstate, 3, 0, &CpuDid); + if (CpuDid == 0x0) + divisor = 1; + else if (CpuDid == 0x1) + divisor = 1.5; + else if (CpuDid == 0x2) + divisor = 2; + else if (CpuDid == 0x3) + divisor = 3; + else if (CpuDid == 0x4) + divisor = 4; + else if (CpuDid == 0x5) + divisor = 6; + else if (CpuDid == 0x6) + divisor = 8; + else if (CpuDid == 0x7) + divisor = 12; + else if (CpuDid == 0x8) + divisor = 16; + else + divisor = 0; + + if (divisor > 0) + *multiplier = (uint64_t) ((CpuFid + 0x10) / divisor); + else + err++; + break; + case 0x14: + /* BKDG 14h, page 430 + MSRC001_00[6B:64][8:4] is CpuDidMSD + MSRC001_00[6B:64][3:0] is CpuDidLSD + PLL COF is (100 MHz * (D18F3xD4[MainPllOpFreqId] + 10h)) + Divisor is (CpuDidMSD + (CpuDidLSD * 0.25) + 1) + CPU COF is (main PLL frequency specified by D18F3xD4[MainPllOpFreqId]) / (core clock divisor specified by CpuDidMSD and CpuDidLSD) */ + err = cpu_rdmsr_range(handle, pstate, 8, 4, &CpuDid); + err += cpu_rdmsr_range(handle, pstate, 3, 0, &CpuDidLSD); + if (clock == 0) + clock = cpu_clock_measure(100, 1) + 5; // Fake round + *multiplier = (uint64_t) ((clock / 100 + 0x10) / (CpuDid + CpuDidLSD * 0.25 + 1)); + break; + case 0x10: + /* BKDG 10h, page 429 + MSRC001_00[6B:64][8:6] is CpuDid + MSRC001_00[6B:64][5:0] is CpuFid + CPU COF is (100 MHz * (CpuFid + 10h) / (2^CpuDid)) */ + case 0x15: + /* BKDG 15h, page 570/580/635/692 (00h-0Fh/10h-1Fh/30h-3Fh/60h-6Fh) + MSRC001_00[6B:64][8:6] is CpuDid + MSRC001_00[6B:64][5:0] is CpuFid + CoreCOF is (100 * (MSRC001_00[6B:64][CpuFid] + 10h) / (2^MSRC001_00[6B:64][CpuDid])) */ + case 0x16: + /* BKDG 16h, page 549/611 (00h-0Fh/30h-3Fh) + MSRC001_00[6B:64][8:6] is CpuDid + MSRC001_00[6B:64][5:0] is CpuFid + CoreCOF is (100 * (MSRC001_00[6B:64][CpuFid] + 10h) / (2^MSRC001_00[6B:64][CpuDid])) */ + err = cpu_rdmsr_range(handle, pstate, 8, 6, &CpuDid); + err += cpu_rdmsr_range(handle, pstate, 5, 0, &CpuFid); + *multiplier = (uint64_t) ((CpuFid + 0x10) / (1ull << CpuDid)); + break; + default: + err = 1; + break; + } + + return err; +} + +static double get_info_min_multiplier(struct msr_driver_t* handle, struct cpu_id_t *id, + struct internal_id_info_t *internal) +{ + int err; + uint64_t reg; + + if(id->vendor == VENDOR_INTEL) { + /* Refer links above + Table 35-12. MSRs in Next Generation Intel Atom Processors Based on the Goldmont Microarchitecture + Table 35-13. MSRs in Processors Based on Intel® Microarchitecture Code Name Nehalem + Table 35-18. MSRs Supported by Intel® Processors based on Intel® microarchitecture code name Sandy Bridge (Contd.) + Table 35-23. Additional MSRs Supported by 3rd Generation Intel® Core™ Processors (based on Intel® microarchitecture code name Ivy Bridge) + Table 35-24. MSRs Supported by Intel® Xeon® Processors E5 v2 Product Family (based on Ivy Bridge-E microarchitecture) + Table 35-27. Additional MSRs Supported by Processors based on the Haswell or Haswell-E microarchitectures + Table 35-34. Additional MSRs Common to Intel® Xeon® Processor D and Intel Xeon Processors E5 v4 Family Based on the Broadwell Microarchitecture + Table 35-40. Selected MSRs Supported by Next Generation Intel® Xeon Phi™ Processors with DisplayFamily_DisplayModel Signature 06_57H + MSR_PLATFORM_INFO[47:40] is Maximum Efficiency Ratio + Maximum Efficiency Ratio is the minimum ratio that the processor can operates */ + err = cpu_rdmsr_range(handle, MSR_PLATFORM_INFO, 47, 40, ®); + if (!err) return (double) reg; + } + else if(id->vendor == VENDOR_AMD) { + /* Refer links above + MSRC001_0061[6:4] is PstateMaxVal + PstateMaxVal is the lowest-performance non-boosted P-state */ + err = cpu_rdmsr_range(handle, MSR_PSTATE_L, 6, 4, ®); + err += get_amd_multipliers(handle, id, internal, MSR_PSTATE_0 + (uint32_t) reg, ®); + if (!err) return (double) reg; + } + + return (double) CPU_INVALID_VALUE / 100; +} + +static double get_info_cur_multiplier(struct msr_driver_t* handle, struct cpu_id_t *id, + struct internal_id_info_t *internal) +{ + int err; + uint64_t reg; + + if(id->vendor == VENDOR_INTEL && internal->code.intel == PENTIUM) { + err = cpu_rdmsr(handle, MSR_EBL_CR_POWERON, ®); + if (!err) return (double) ((reg>>22) & 0x1f); + } + else if(id->vendor == VENDOR_INTEL && internal->code.intel != PENTIUM) { + /* Refer links above + Table 35-2. IA-32 Architectural MSRs (Contd.) + IA32_PERF_STATUS[15:0] is Current performance State Value + [7:0] is 0x0, [15:8] looks like current ratio */ + err = cpu_rdmsr_range(handle, IA32_PERF_STATUS, 15, 8, ®); + if (!err) return (double) reg; + } + else if(id->vendor == VENDOR_AMD) { + /* Refer links above + MSRC001_0063[2:0] is CurPstate */ + err = cpu_rdmsr_range(handle, MSR_PSTATE_S, 2, 0, ®); + err += get_amd_multipliers(handle, id, internal, MSR_PSTATE_0 + (uint32_t) reg, ®); + if (!err) return (double) reg; + } + + return (double) CPU_INVALID_VALUE / 100; +} + +static double get_info_max_multiplier(struct msr_driver_t* handle, struct cpu_id_t *id, + struct internal_id_info_t *internal) +{ + int err; + uint64_t reg; + + if(id->vendor == VENDOR_INTEL && internal->code.intel == PENTIUM) { + err = cpu_rdmsr(handle, IA32_PERF_STATUS, ®); + if (!err) return (double) ((reg >> 40) & 0x1f); + } + else if(id->vendor == VENDOR_INTEL && internal->code.intel != PENTIUM) { + /* Refer links above + Table 35-10. Specific MSRs Supported by Intel® Atom™ Processor C2000 Series with CPUID Signature 06_4DH + Table 35-12. MSRs in Next Generation Intel Atom Processors Based on the Goldmont Microarchitecture (Contd.) + Table 35-13. MSRs in Processors Based on Intel® Microarchitecture Code Name Nehalem (Contd.) + Table 35-14. Additional MSRs in Intel® Xeon® Processor 5500 and 3400 Series + Table 35-16. Additional MSRs Supported by Intel Processors (Based on Intel® Microarchitecture Code Name Westmere) + Table 35-19. MSRs Supported by 2nd Generation Intel® Core™ Processors (Intel® microarchitecture code name Sandy Bridge) + Table 35-21. Selected MSRs Supported by Intel® Xeon® Processors E5 Family (based on Sandy Bridge microarchitecture) + Table 35-28. MSRs Supported by 4th Generation Intel® Core™ Processors (Haswell microarchitecture) (Contd.) + Table 35-30. Additional MSRs Supported by Intel® Xeon® Processor E5 v3 Family + Table 35-33. Additional MSRs Supported by Intel® Core™ M Processors and 5th Generation Intel® Core™ Processors + Table 35-34. Additional MSRs Common to Intel® Xeon® Processor D and Intel Xeon Processors E5 v4 Family Based on the Broadwell Microarchitecture + Table 35-37. Additional MSRs Supported by 6th Generation Intel® Core™ Processors Based on Skylake Microarchitecture + Table 35-40. Selected MSRs Supported by Next Generation Intel® Xeon Phi™ Processors with DisplayFamily_DisplayModel Signature 06_57H + MSR_TURBO_RATIO_LIMIT[7:0] is Maximum Ratio Limit for 1C */ + err = cpu_rdmsr_range(handle, MSR_TURBO_RATIO_LIMIT, 7, 0, ®); + if (!err) return (double) reg; + } + else if(id->vendor == VENDOR_AMD) { + /* Refer links above + MSRC001_0064 is Pb0 + Pb0 is the highest-performance boosted P-state */ + err = get_amd_multipliers(handle, id, internal, MSR_PSTATE_0, ®); + if (!err) return (double) reg; + } + + return (double) CPU_INVALID_VALUE / 100; +} + +static int get_info_temperature(struct msr_driver_t* handle, struct cpu_id_t *id, + struct internal_id_info_t *internal) +{ + int err; + uint64_t DigitalReadout, ReadingValid, TemperatureTarget; + + if(id->vendor == VENDOR_INTEL) { + /* Refer links above + Table 35-2. IA-32 Architectural MSRs + IA32_THERM_STATUS[22:16] is Digital Readout + IA32_THERM_STATUS[31] is Reading Valid + + Table 35-6. MSRs Common to the Silvermont Microarchitecture and Newer Microarchitectures for Intel® Atom + Table 35-13. MSRs in Processors Based on Intel® Microarchitecture Code Name Nehalem (Contd.) + Table 35-18. MSRs Supported by Intel® Processors based on Intel® microarchitecture code name Sandy Bridge (Contd.) + Table 35-24. MSRs Supported by Intel® Xeon® Processors E5 v2 Product Family (based on Ivy Bridge-E microarchitecture) (Contd.) + Table 35-34. Additional MSRs Common to Intel® Xeon® Processor D and Intel Xeon Processors E5 v4 Family Based on the Broadwell Microarchitecture + Table 35-40. Selected MSRs Supported by Next Generation Intel® Xeon Phi™ Processors with DisplayFamily_DisplayModel Signature 06_57H + MSR_TEMPERATURE_TARGET[23:16] is Temperature Target */ + err = cpu_rdmsr_range(handle, IA32_THERM_STATUS, 22, 16, &DigitalReadout); + err += cpu_rdmsr_range(handle, IA32_THERM_STATUS, 31, 31, &ReadingValid); + err += cpu_rdmsr_range(handle, MSR_TEMPERATURE_TARGET, 23, 16, &TemperatureTarget); + if(!err && ReadingValid) return (int) (TemperatureTarget - DigitalReadout); + } + + return CPU_INVALID_VALUE; +} + +static double get_info_voltage(struct msr_driver_t* handle, struct cpu_id_t *id, + struct internal_id_info_t *internal) +{ + int err; + uint64_t reg, CpuVid; + + if(id->vendor == VENDOR_INTEL) { + /* Refer links above + Table 35-18. MSRs Supported by Intel® Processors based on Intel® microarchitecture code name Sandy Bridge (Contd.) + MSR_PERF_STATUS[47:32] is Core Voltage + P-state core voltage can be computed by MSR_PERF_STATUS[37:32] * (float) 1/(2^13). */ + err = cpu_rdmsr_range(handle, MSR_PERF_STATUS, 47, 32, ®); + if (!err) return (double) reg / (1 << 13); + } + else if(id->vendor == VENDOR_AMD) { + /* Refer links above + MSRC001_00[6B:64][15:9] is CpuVid + MSRC001_0063[2:0] is P-state Status + 2.4.1.6.3 Serial VID (SVI) Encodings: voltage = 1.550V - 0.0125V * SviVid[6:0] */ + err = cpu_rdmsr_range(handle, MSR_PSTATE_S, 2, 0, ®); + err += cpu_rdmsr_range(handle, MSR_PSTATE_0 + (uint32_t) reg, 15, 9, &CpuVid); + if (!err && MSR_PSTATE_0 + (uint32_t) reg <= MSR_PSTATE_7) return 1.550 - 0.0125 * CpuVid; + } + + return (double) CPU_INVALID_VALUE / 100; +} + +static double get_info_bus_clock(struct msr_driver_t* handle, struct cpu_id_t *id, + struct internal_id_info_t *internal) +{ + int err; + static int clock = 0; + uint64_t reg; + + if(clock == 0) + clock = cpu_clock_measure(100, 1); + + if(id->vendor == VENDOR_INTEL) { + /* Refer links above + Table 35-12. MSRs in Next Generation Intel Atom Processors Based on the Goldmont Microarchitecture + Table 35-13. MSRs in Processors Based on Intel® Microarchitecture Code Name Nehalem + Table 35-18. MSRs Supported by Intel® Processors based on Intel® microarchitecture code name Sandy Bridge (Contd.) + Table 35-23. Additional MSRs Supported by 3rd Generation Intel® Core™ Processors (based on Intel® microarchitecture code name Ivy Bridge) + Table 35-24. MSRs Supported by Intel® Xeon® Processors E5 v2 Product Family (based on Ivy Bridge-E microarchitecture) + Table 35-27. Additional MSRs Supported by Processors based on the Haswell or Haswell-E microarchitectures + Table 35-40. Selected MSRs Supported by Next Generation Intel® Xeon Phi™ Processors with DisplayFamily_DisplayModel Signature 06_57H + MSR_PLATFORM_INFO[15:8] is Maximum Non-Turbo Ratio */ + err = cpu_rdmsr_range(handle, MSR_PLATFORM_INFO, 15, 8, ®); + if (!err) return (double) clock / reg; + } + else if(id->vendor == VENDOR_AMD) { + /* Refer links above + MSRC001_0061[2:0] is CurPstateLimit + CurPstateLimit is the highest-performance non-boosted P-state */ + err = cpu_rdmsr_range(handle, MSR_PSTATE_L, 2, 0, ®); + err += get_amd_multipliers(handle, id, internal, MSR_PSTATE_0 + (uint32_t) reg, ®); + if (!err) return (double) clock / reg; + } + + return (double) CPU_INVALID_VALUE / 100; +} + +int cpu_rdmsr_range(struct msr_driver_t* handle, uint32_t msr_index, uint8_t highbit, + uint8_t lowbit, uint64_t* result) +{ + int err; + const uint8_t bits = highbit - lowbit + 1; + + if(highbit > 63 || lowbit > highbit) + return set_error(ERR_INVRANGE); + + err = cpu_rdmsr(handle, msr_index, result); + + if(!err && bits < 64) { + /* Show only part of register */ + *result >>= lowbit; + *result &= (1ULL << bits) - 1; + } + + return err; +} + +int cpu_msrinfo(struct msr_driver_t* handle, cpu_msrinfo_request_t which) +{ + struct cpu_raw_data_t raw; + static struct cpu_id_t id; + static struct internal_id_info_t internal; + internal.score = -1; + + if (handle == NULL) + return set_error(ERR_HANDLE); + + if (internal.score == -1) { + cpuid_get_raw_data(&raw); + cpu_ident_internal(&raw, &id, &internal); + } + + switch (which) { + case INFO_MPERF: + return perfmsr_measure(handle, IA32_MPERF); + case INFO_APERF: + return perfmsr_measure(handle, IA32_APERF); + case INFO_MIN_MULTIPLIER: + return (int) (get_info_min_multiplier(handle, &id, &internal) * 100); + case INFO_CUR_MULTIPLIER: + return (int) (get_info_cur_multiplier(handle, &id, &internal) * 100); + case INFO_MAX_MULTIPLIER: + return (int) (get_info_max_multiplier(handle, &id, &internal) * 100); + case INFO_TEMPERATURE: + return get_info_temperature(handle, &id, &internal); + case INFO_THROTTLING: + return CPU_INVALID_VALUE; + case INFO_VOLTAGE: + return (int) (get_info_voltage(handle, &id, &internal) * 100); + case INFO_BCLK: + case INFO_BUS_CLOCK: + return (int) (get_info_bus_clock(handle, &id, &internal) * 100); + default: + return CPU_INVALID_VALUE; + } +} + +#endif // RDMSR_UNSUPPORTED_OS diff --git a/contrib/libcpuid/include/cpuid/rdtsc.c b/contrib/libcpuid/include/libcpuid/rdtsc.c similarity index 86% rename from contrib/libcpuid/include/cpuid/rdtsc.c rename to contrib/libcpuid/include/libcpuid/rdtsc.c index 5930681561a..df4543946f5 100644 --- a/contrib/libcpuid/include/cpuid/rdtsc.c +++ b/contrib/libcpuid/include/libcpuid/rdtsc.c @@ -226,6 +226,45 @@ int cpu_clock_measure(int millis, int quad_check) return (results[bi] + results[bj] + _zero) / 2; } + +static void adjust_march_ic_multiplier(const struct cpu_id_t* id, int* numerator, int* denom) +{ + /* + * for cpu_clock_by_ic: we need to know how many clocks does a typical ADDPS instruction + * take, when issued in rapid succesion without dependencies. The whole idea of + * cpu_clock_by_ic was that this is easy to determine, at least it was back in 2010. Now + * it's getting progressively more hairy, but here are the current measurements: + * + * 1. For CPUs with 64-bit SSE units, ADDPS issue rate is 0.5 IPC (one insn in 2 clocks) + * 2. For CPUs with 128-bit SSE units, issue rate is exactly 1.0 IPC + * 3. For Bulldozer and later, it is 1.4 IPC (we multiply by 5/7) + * 4. For Skylake and later, it is 1.6 IPC (we multiply by 5/8) + */ + // + if (id->sse_size < 128) { + debugf(1, "SSE execution path is 64-bit\n"); + // on a CPU with half SSE unit length, SSE instructions execute at 0.5 IPC; + // the resulting value must be multiplied by 2: + *numerator = 2; + } else { + debugf(1, "SSE execution path is 128-bit\n"); + } + // + // Bulldozer or later: assume 1.4 IPC + if (id->vendor == VENDOR_AMD && id->ext_family >= 21) { + debugf(1, "cpu_clock_by_ic: Bulldozer (or later) detected, dividing result by 1.4\n"); + *numerator = 5; + *denom = 7; // multiply by 5/7, to divide by 1.4 + } + // + // Skylake or later: assume 1.6 IPC + if (id->vendor == VENDOR_INTEL && id->ext_model >= 94) { + debugf(1, "cpu_clock_by_ic: Skylake (or later) detected, dividing result by 1.6\n"); + *numerator = 5; + *denom = 8; // to divide by 1.6, multiply by 5/8 + } +} + int cpu_clock_by_ic(int millis, int runs) { int max_value = 0, cur_value, i, ri, cycles_inner, cycles_outer, c; @@ -237,21 +276,7 @@ int cpu_clock_by_ic(int millis, int runs) // if there aren't SSE instructions - we can't run the test at all if (!id || !id->flags[CPU_FEATURE_SSE]) return -1; // - if (id->sse_size < 128) { - debugf(1, "SSE execution path is 64-bit\n"); - // on a CPU with half SSE unit length, SSE instructions execute at 0.5 IPC; - // the resulting value must be multiplied by 2: - multiplier_numerator = 2; - } else { - debugf(1, "SSE execution path is 128-bit\n"); - } - // - // on a Bulldozer or later CPU, SSE instructions execute at 1.4 IPC, handle that as well: - if (id->vendor == VENDOR_AMD && id->ext_family >= 21) { - debugf(1, "cpu_clock_by_ic: Bulldozer (or later) detected, dividing result by 1.4\n"); - multiplier_numerator = 5; - multiplier_denom = 7; // multiply by 5/7, to divide by 1.4 - } + adjust_march_ic_multiplier(id, &multiplier_numerator, &multiplier_denom); // tl = millis * 125; // (*1000 / 8) cycles_inner = 128; diff --git a/contrib/libcpuid/include/cpuid/rdtsc.h b/contrib/libcpuid/include/libcpuid/rdtsc.h similarity index 100% rename from contrib/libcpuid/include/cpuid/rdtsc.h rename to contrib/libcpuid/include/libcpuid/rdtsc.h diff --git a/contrib/libcpuid/include/cpuid/recog_amd.c b/contrib/libcpuid/include/libcpuid/recog_amd.c similarity index 85% rename from contrib/libcpuid/include/cpuid/recog_amd.c rename to contrib/libcpuid/include/libcpuid/recog_amd.c index c5390b9fd24..2e6c8a9ead8 100644 --- a/contrib/libcpuid/include/cpuid/recog_amd.c +++ b/contrib/libcpuid/include/libcpuid/recog_amd.c @@ -28,47 +28,16 @@ #include #include #include "libcpuid.h" -#include "recog_amd.h" #include "libcpuid_util.h" +#include "libcpuid_internal.h" +#include "recog_amd.h" -enum _amd_code_t { - NA, - NO_CODE, - OPTERON_GENERIC, - OPTERON_800, - ATHLON_XP, - ATHLON_XP_M, - ATHLON_XP_M_LV, - ATHLON, - ATHLON_MP, - MOBILE_ATHLON64, - ATHLON_FX, - DURON, - DURON_MP, - MOBILE_DURON, - MOBILE_SEMPRON, - OPTERON_SINGLE, - OPTERON_DUALCORE, - OPTERON_800_DUALCORE, - MOBILE_TURION, - ATHLON_64, - ATHLON_64_FX, - TURION_64, - TURION_X2, - SEMPRON, - M_SEMPRON, - SEMPRON_DUALCORE, - PHENOM, - PHENOM2, - ATHLON_64_X2, - ATHLON_64_X3, - ATHLON_64_X4, - FUSION_C, - FUSION_E, - FUSION_EA, - FUSION_Z, +const struct amd_code_str { amd_code_t code; char *str; } amd_code_str[] = { + #define CODE(x) { x, #x } + #define CODE2(x, y) CODE(x) + #include "amd_code_t.h" + #undef CODE }; -typedef enum _amd_code_t amd_code_t; const struct match_entry_t cpudb_amd[] = { { -1, -1, -1, -1, -1, 1, -1, -1, NO_CODE , 0, "Unknown AMD CPU" }, @@ -146,6 +115,7 @@ const struct match_entry_t cpudb_amd[] = { { 15, -1, -1, 15, -1, 1, 1024, -1, ATHLON_64 , 0, "Athlon 64 (1024K)" }, { 15, -1, -1, 15, -1, 1, -1, -1, ATHLON_FX , 0, "Athlon FX" }, { 15, -1, -1, 15, -1, 1, -1, -1, ATHLON_64_FX , 0, "Athlon 64 FX" }, + { 15, 3, -1, 15, 35, 2, -1, -1, ATHLON_64_FX , 0, "Athlon 64 FX X2 (Toledo)" }, { 15, -1, -1, 15, -1, 2, 512, -1, ATHLON_64_X2 , 0, "Athlon 64 X2 (512K)" }, { 15, -1, -1, 15, -1, 2, 1024, -1, ATHLON_64_X2 , 0, "Athlon 64 X2 (1024K)" }, { 15, -1, -1, 15, -1, 1, 512, -1, TURION_64 , 0, "Turion 64 (512K)" }, @@ -237,31 +207,65 @@ const struct match_entry_t cpudb_amd[] = { { 15, 4, -1, 16, 10, 4, 512, -1, PHENOM2 , 0, "Phenom II X4 (Zosma)" }, { 15, 4, -1, 16, 10, 6, 512, -1, PHENOM2 , 0, "Phenom II X6 (Thuban)" }, - { 15, 4, -1, 16, -1, 2, 1024, -1, ATHLON_64_X2 , 0, "Athlon II X2 (Regor)" }, - { 15, 4, -1, 16, -1, 2, 512, -1, ATHLON_64_X2 , 0, "Athlon II X2 (Regor)" }, + { 15, 6, -1, 16, 6, 2, 512, -1, ATHLON , 0, "Athlon II (Champlain)" }, + { 15, 6, -1, 16, 6, 2, 512, -1, ATHLON_64_X2 , 0, "Athlon II X2 (Regor)" }, + { 15, 6, -1, 16, 6, 2, 1024, -1, ATHLON_64_X2 , 0, "Athlon II X2 (Regor)" }, { 15, 5, -1, 16, 5, 3, 512, -1, ATHLON_64_X3 , 0, "Athlon II X3 (Rana)" }, { 15, 5, -1, 16, 5, 4, 512, -1, ATHLON_64_X4 , 0, "Athlon II X4 (Propus)" }, - /* 2011 CPUs with AMD fusion: */ - { 15, -1, -1, 20, 1, 1, 512, -1, FUSION_C , 0, "Brazos Ontario" }, - { 15, -1, -1, 20, 1, 2, 512, -1, FUSION_C , 0, "Brazos Ontario (Dual-core)" }, - { 15, -1, -1, 20, 1, 1, 512, -1, FUSION_E , 0, "Brazos Zacate" }, - { 15, -1, -1, 20, 1, 2, 512, -1, FUSION_E , 0, "Brazos Zacate (Dual-core)" }, - { 15, -1, -1, 20, 1, 1, 512, -1, FUSION_Z , 0, "Brazos Desna" }, - { 15, -1, -1, 18, 1, 2, 512, -1, FUSION_EA , 0, "Llano X2" }, - { 15, -1, -1, 18, 1, 2, 1024, -1, FUSION_EA , 0, "Llano X2" }, - { 15, -1, -1, 18, 1, 3, 1024, -1, FUSION_EA , 0, "Llano X3" }, - { 15, -1, -1, 18, 1, 4, 1024, -1, FUSION_EA , 0, "Llano X4" }, + + /* 2011 CPUs: K10 architecture: Llano */ + { 15, 1, -1, 18, 1, 2, 512, -1, FUSION_EA , 0, "Llano X2" }, + { 15, 1, -1, 18, 1, 2, 1024, -1, FUSION_EA , 0, "Llano X2" }, + { 15, 1, -1, 18, 1, 3, 1024, -1, FUSION_EA , 0, "Llano X3" }, + { 15, 1, -1, 18, 1, 4, 1024, -1, FUSION_EA , 0, "Llano X4" }, + /* 2011 CPUs: Bobcat architecture: Ontario, Zacate, Desna, Hondo */ + { 15, 2, -1, 20, -1, 1, 512, -1, FUSION_C , 0, "Brazos Ontario" }, + { 15, 2, -1, 20, -1, 2, 512, -1, FUSION_C , 0, "Brazos Ontario (Dual-core)" }, + { 15, 1, -1, 20, -1, 1, 512, -1, FUSION_E , 0, "Brazos Zacate" }, + { 15, 1, -1, 20, -1, 2, 512, -1, FUSION_E , 0, "Brazos Zacate (Dual-core)" }, + { 15, 2, -1, 20, -1, 2, 512, -1, FUSION_Z , 0, "Brazos Desna (Dual-core)" }, + /* 2012 CPUs: Piledriver architecture: Trinity and Richland */ + { 15, 0, -1, 21, 10, 2, 1024, -1, FUSION_A , 0, "Trinity X2" }, + { 15, 0, -1, 21, 16, 2, 1024, -1, FUSION_A , 0, "Trinity X2" }, + { 15, 0, -1, 21, 10, 4, 1024, -1, FUSION_A , 0, "Trinity X4" }, + { 15, 0, -1, 21, 16, 4, 1024, -1, FUSION_A , 0, "Trinity X4" }, + { 15, 3, -1, 21, 13, 2, 1024, -1, FUSION_A , 0, "Richland X2" }, + { 15, 3, -1, 21, 13, 4, 1024, -1, FUSION_A , 0, "Richland X4" }, + /* 2013 CPUs: Jaguar architecture: Kabini and Temash */ + { 15, 0, -1, 22, 0, 2, 1024, -1, FUSION_A , 0, "Kabini X2" }, + { 15, 0, -1, 22, 0, 4, 1024, -1, FUSION_A , 0, "Kabini X4" }, + /* 2014 CPUs: Steamroller architecture: Kaveri */ + { 15, 0, -1, 21, 30, 2, 1024, -1, FUSION_A , 0, "Kaveri X2" }, + { 15, 0, -1, 21, 30, 4, 1024, -1, FUSION_A , 0, "Kaveri X4" }, + /* 2014 CPUs: Puma architecture: Beema and Mullins */ + { 15, 0, -1, 22, 30, 2, 1024, -1, FUSION_E , 0, "Mullins X2" }, + { 15, 0, -1, 22, 30, 4, 1024, -1, FUSION_A , 0, "Mullins X4" }, + /* 2015 CPUs: Excavator architecture: Carrizo */ + { 15, 1, -1, 21, 60, 2, 1024, -1, FUSION_A , 0, "Carrizo X2" }, + { 15, 1, -1, 21, 60, 4, 1024, -1, FUSION_A , 0, "Carrizo X4" }, + /* 2015 CPUs: Steamroller architecture: Godavari */ + //TODO + /* 2016 CPUs: Excavator architecture: Bristol Ridge */ + //TODO /* Newer Opterons: */ - { 15, 9, -1, 16, 9, 8, -1, -1, OPTERON_GENERIC , 0, "Magny-Cours Opteron" }, + { 15, 9, -1, 22, 9, 8, -1, -1, OPTERON_GENERIC , 0, "Magny-Cours Opteron" }, /* Bulldozer CPUs: */ + { 15, -1, -1, 21, 0, 4, 2048, -1, NO_CODE , 0, "Bulldozer X2" }, { 15, -1, -1, 21, 1, 4, 2048, -1, NO_CODE , 0, "Bulldozer X2" }, { 15, -1, -1, 21, 1, 6, 2048, -1, NO_CODE , 0, "Bulldozer X3" }, { 15, -1, -1, 21, 1, 8, 2048, -1, NO_CODE , 0, "Bulldozer X4" }, + /* Piledriver CPUs: */ { 15, -1, -1, 21, 2, 4, 2048, -1, NO_CODE , 0, "Vishera X2" }, { 15, -1, -1, 21, 2, 6, 2048, -1, NO_CODE , 0, "Vishera X3" }, { 15, -1, -1, 21, 2, 8, 2048, -1, NO_CODE , 0, "Vishera X4" }, + /* Steamroller CPUs: */ + //TODO + /* Excavator CPUs: */ + //TODO + /* Zen CPUs: */ + //TODO }; @@ -287,6 +291,7 @@ static void load_amd_features(struct cpu_raw_data_t* raw, struct cpu_id_t* data) { 12, CPU_FEATURE_SKINIT }, { 13, CPU_FEATURE_WDT }, { 16, CPU_FEATURE_FMA4 }, + { 21, CPU_FEATURE_TBM }, }; const struct feature_map_t matchtable_edx87[] = { { 0, CPU_FEATURE_TS }, @@ -307,7 +312,7 @@ static void load_amd_features(struct cpu_raw_data_t* raw, struct cpu_id_t* data) match_features(matchtable_edx81, COUNT_OF(matchtable_edx81), raw->ext_cpuid[1][3], data); match_features(matchtable_ecx81, COUNT_OF(matchtable_ecx81), raw->ext_cpuid[1][2], data); } - if (raw->ext_cpuid[0][0] >= 0x80000001) + if (raw->ext_cpuid[0][0] >= 0x80000007) match_features(matchtable_edx87, COUNT_OF(matchtable_edx87), raw->ext_cpuid[7][3], data); if (raw->ext_cpuid[0][0] >= 0x8000001a) { /* We have the extended info about SSE unit size */ @@ -320,7 +325,7 @@ static void decode_amd_cache_info(struct cpu_raw_data_t* raw, struct cpu_id_t* d { int l3_result; const int assoc_table[16] = { - 0, 1, 2, 0, 4, 0, 8, 0, 16, 0, 32, 48, 64, 92, 128, 255 + 0, 1, 2, 0, 4, 0, 8, 0, 16, 0, 32, 48, 64, 96, 128, 255 }; unsigned n = raw->ext_cpuid[0][0]; @@ -442,24 +447,36 @@ static amd_code_t decode_amd_codename_part1(const char *bs) if (match_pattern(bs, "Z-##")) return FUSION_Z; if (match_pattern(bs, "E#-####") || match_pattern(bs, "A#-####")) return FUSION_EA; - return NO_CODE; + return (amd_code_t) NO_CODE; } -static void decode_amd_codename(struct cpu_raw_data_t* raw, struct cpu_id_t* data) +static void decode_amd_codename(struct cpu_raw_data_t* raw, struct cpu_id_t* data, struct internal_id_info_t* internal) { amd_code_t code = decode_amd_codename_part1(data->brand_str); - + int i = 0; + char* code_str = NULL; + for (i = 0; i < COUNT_OF(amd_code_str); i++) { + if (code == amd_code_str[i].code) { + code_str = amd_code_str[i].str; + break; + } + } if (code == ATHLON_64_X2 && data->l2_cache < 512) code = SEMPRON_DUALCORE; - match_cpu_codename(cpudb_amd, COUNT_OF(cpudb_amd), data, code, 0); + if (code_str) + debugf(2, "Detected AMD brand code: %d (%s)\n", code, code_str); + else + debugf(2, "Detected AMD brand code: %d\n", code); + internal->code.amd = code; + internal->score = match_cpu_codename(cpudb_amd, COUNT_OF(cpudb_amd), data, code, 0); } -int cpuid_identify_amd(struct cpu_raw_data_t* raw, struct cpu_id_t* data) +int cpuid_identify_amd(struct cpu_raw_data_t* raw, struct cpu_id_t* data, struct internal_id_info_t* internal) { load_amd_features(raw, data); decode_amd_cache_info(raw, data); decode_amd_number_of_cores(raw, data); - decode_amd_codename(raw, data); + decode_amd_codename(raw, data, internal); return 0; } diff --git a/contrib/libcpuid/include/cpuid/recog_amd.h b/contrib/libcpuid/include/libcpuid/recog_amd.h similarity index 96% rename from contrib/libcpuid/include/cpuid/recog_amd.h rename to contrib/libcpuid/include/libcpuid/recog_amd.h index e0f3b61f727..34e89598397 100644 --- a/contrib/libcpuid/include/cpuid/recog_amd.h +++ b/contrib/libcpuid/include/libcpuid/recog_amd.h @@ -26,7 +26,7 @@ #ifndef __RECOG_AMD_H__ #define __RECOG_AMD_H__ -int cpuid_identify_amd(struct cpu_raw_data_t* raw, struct cpu_id_t* data); +int cpuid_identify_amd(struct cpu_raw_data_t* raw, struct cpu_id_t* data, struct internal_id_info_t* internal); void cpuid_get_list_amd(struct cpu_list_t* list); #endif /* __RECOG_AMD_H__ */ diff --git a/contrib/libcpuid/include/cpuid/recog_intel.c b/contrib/libcpuid/include/libcpuid/recog_intel.c similarity index 75% rename from contrib/libcpuid/include/cpuid/recog_intel.c rename to contrib/libcpuid/include/libcpuid/recog_intel.c index 625b2777f51..2ffc41c8a15 100644 --- a/contrib/libcpuid/include/cpuid/recog_intel.c +++ b/contrib/libcpuid/include/libcpuid/recog_intel.c @@ -26,61 +26,16 @@ #include #include #include "libcpuid.h" -#include "recog_intel.h" #include "libcpuid_util.h" +#include "libcpuid_internal.h" +#include "recog_intel.h" - -enum _intel_code_t { - NA, - NO_CODE, - PENTIUM = 10, - MOBILE_PENTIUM, - - XEON = 20, - XEON_IRWIN, - XEONMP, - XEON_POTOMAC, - XEON_I7, - XEON_GAINESTOWN, - XEON_WESTMERE, - - MOBILE_PENTIUM_M = 30, - CELERON, - MOBILE_CELERON, - NOT_CELERON, - - - CORE_SOLO = 40, - MOBILE_CORE_SOLO, - CORE_DUO, - MOBILE_CORE_DUO, - - WOLFDALE = 50, - MEROM, - PENRYN, - QUAD_CORE, - DUAL_CORE_HT, - QUAD_CORE_HT, - MORE_THAN_QUADCORE, - PENTIUM_D, - - ATOM = 60, - ATOM_SILVERTHORNE, - ATOM_DIAMONDVILLE, - ATOM_PINEVIEW, - ATOM_CEDARVIEW, - - CORE_I3 = 70, - CORE_I5, - CORE_I7, - CORE_IVY3, /* 22nm Core-iX */ - CORE_IVY5, - CORE_IVY7, - CORE_HASWELL3, /* 22nm Core-iX, Haswell */ - CORE_HASWELL5, - CORE_HASWELL7, +const struct intel_bcode_str { intel_code_t code; char *str; } intel_bcode_str[] = { + #define CODE(x) { x, #x } + #define CODE2(x, y) CODE(x) + #include "intel_code_t.h" + #undef CODE }; -typedef enum _intel_code_t intel_code_t; enum _intel_model_t { UNKNOWN = -1, @@ -134,12 +89,12 @@ const struct match_entry_t cpudb_intel[] = { { 6, 5, -1, -1, -1, 1, -1, -1, MOBILE_PENTIUM , 0, "Mobile Pentium II (Tonga)"}, { 6, 6, -1, -1, -1, 1, -1, -1, NO_CODE , 0, "Pentium II (Dixon)" }, - { 6, 3, -1, -1, -1, 1, -1, -1, XEON , 0, "P-II Xeon" }, - { 6, 5, -1, -1, -1, 1, -1, -1, XEON , 0, "P-II Xeon" }, - { 6, 6, -1, -1, -1, 1, -1, -1, XEON , 0, "P-II Xeon" }, + { 6, 3, -1, -1, -1, 1, -1, -1, XEON , 0, "P-II Xeon (Klamath)" }, + { 6, 5, -1, -1, -1, 1, -1, -1, XEON , 0, "P-II Xeon (Drake)" }, + { 6, 6, -1, -1, -1, 1, -1, -1, XEON , 0, "P-II Xeon (Dixon)" }, - { 6, 5, -1, -1, -1, 1, -1, -1, CELERON , 0, "P-II Celeron (no L2)" }, - { 6, 6, -1, -1, -1, 1, -1, -1, CELERON , 0, "P-II Celeron (128K)" }, + { 6, 5, -1, -1, -1, 1, -1, -1, CELERON , 0, "P-II Celeron (Covingtons" }, + { 6, 6, -1, -1, -1, 1, -1, -1, CELERON , 0, "P-II Celeron (Mendocino)" }, /* -------------------------------------------------- */ @@ -148,15 +103,15 @@ const struct match_entry_t cpudb_intel[] = { { 6, 10, -1, -1, -1, 1, -1, -1, NO_CODE , 0, "Pentium III (Coppermine)"}, { 6, 11, -1, -1, -1, 1, -1, -1, NO_CODE , 0, "Pentium III (Tualatin)" }, - { 6, 7, -1, -1, -1, 1, -1, -1, XEON , 0, "P-III Xeon" }, - { 6, 8, -1, -1, -1, 1, -1, -1, XEON , 0, "P-III Xeon" }, - { 6, 10, -1, -1, -1, 1, -1, -1, XEON , 0, "P-III Xeon" }, - { 6, 11, -1, -1, -1, 1, -1, -1, XEON , 0, "P-III Xeon" }, + { 6, 7, -1, -1, -1, 1, -1, -1, XEON , 0, "P-III Xeon (Tanner)" }, + { 6, 8, -1, -1, -1, 1, -1, -1, XEON , 0, "P-III Xeon (Cascades)" }, + { 6, 10, -1, -1, -1, 1, -1, -1, XEON , 0, "P-III Xeon (Cascades)" }, + { 6, 11, -1, -1, -1, 1, -1, -1, XEON , 0, "P-III Xeon (Tualatin)" }, - { 6, 7, -1, -1, -1, 1, -1, -1, CELERON , 0, "P-III Celeron" }, - { 6, 8, -1, -1, -1, 1, -1, -1, CELERON , 0, "P-III Celeron" }, - { 6, 10, -1, -1, -1, 1, -1, -1, CELERON , 0, "P-III Celeron" }, - { 6, 11, -1, -1, -1, 1, -1, -1, CELERON , 0, "P-III Celeron" }, + { 6, 7, -1, -1, -1, 1, -1, -1, CELERON , 0, "P-III Celeron (Katmai)" }, + { 6, 8, -1, -1, -1, 1, -1, -1, CELERON , 0, "P-III Celeron (Coppermine)" }, + { 6, 10, -1, -1, -1, 1, -1, -1, CELERON , 0, "P-III Celeron (Coppermine)" }, + { 6, 11, -1, -1, -1, 1, -1, -1, CELERON , 0, "P-III Celeron (Tualatin)" }, /* Netburst based (Pentium 4 and later) classic P4s */ @@ -190,17 +145,17 @@ const struct match_entry_t cpudb_intel[] = { { 15, 6, -1, 15, -1, 1, -1, -1, XEON , 0, "Xeon (Dempsey)" }, /* Pentium Ds */ - { 15, 4, 4, 15, -1, 1, -1, -1, NO_CODE , 0, "Pentium D" }, - { 15, 4, -1, 15, -1, 1, -1, -1, PENTIUM_D , 0, "Pentium D" }, - { 15, 4, 7, 15, -1, 1, -1, -1, NO_CODE , 0, "Pentium D" }, - { 15, 6, -1, 15, -1, 1, -1, -1, PENTIUM_D , 0, "Pentium D" }, + { 15, 4, 4, 15, -1, 1, -1, -1, NO_CODE , 0, "Pentium D (SmithField)" }, + { 15, 4, -1, 15, -1, 1, -1, -1, PENTIUM_D , 0, "Pentium D (SmithField)" }, + { 15, 4, 7, 15, -1, 1, -1, -1, NO_CODE , 0, "Pentium D (SmithField)" }, + { 15, 6, -1, 15, -1, 1, -1, -1, PENTIUM_D , 0, "Pentium D (Presler)" }, /* Celeron and Celeron Ds */ - { 15, 1, -1, 15, -1, 1, -1, -1, CELERON , 0, "P-4 Celeron (128K)" }, - { 15, 2, -1, 15, -1, 1, -1, -1, CELERON , 0, "P-4 Celeron (128K)" }, - { 15, 3, -1, 15, -1, 1, -1, -1, CELERON , 0, "Celeron D" }, - { 15, 4, -1, 15, -1, 1, -1, -1, CELERON , 0, "Celeron D" }, - { 15, 6, -1, 15, -1, 1, -1, -1, CELERON , 0, "Celeron D" }, + { 15, 1, -1, 15, -1, 1, -1, -1, CELERON , 0, "P-4 Celeron (Willamette)" }, + { 15, 2, -1, 15, -1, 1, -1, -1, CELERON , 0, "P-4 Celeron (Northwood)" }, + { 15, 3, -1, 15, -1, 1, -1, -1, CELERON , 0, "P-4 Celeron D (Prescott)" }, + { 15, 4, -1, 15, -1, 1, -1, -1, CELERON , 0, "P-4 Celeron D (Prescott)" }, + { 15, 6, -1, 15, -1, 1, -1, -1, CELERON , 0, "P-4 Celeron D (Cedar Mill)" }, /* -------------------------------------------------- */ /* Intel Core microarchitecture - P6-based */ @@ -214,7 +169,7 @@ const struct match_entry_t cpudb_intel[] = { { 6, 13, -1, -1, -1, 1, -1, -1, MOBILE_PENTIUM_M , 0, "Pentium M (Dothan)" }, { 6, 13, -1, -1, -1, 1, -1, -1, CELERON , 0, "Celeron M" }, - { 6, 12, -1, -1, -1, -1, -1, -1, ATOM , 0, "Unknown Atom" }, + { 6, 12, -1, -1, -1, -1, -1, -1, ATOM_UNKNOWN , 0, "Unknown Atom" }, { 6, 12, -1, -1, -1, -1, -1, -1, ATOM_DIAMONDVILLE , 0, "Atom (Diamondville)" }, { 6, 12, -1, -1, -1, -1, -1, -1, ATOM_SILVERTHORNE , 0, "Atom (Silverthorne)" }, { 6, 12, -1, -1, -1, -1, -1, -1, ATOM_CEDARVIEW , 0, "Atom (Cedarview)" }, @@ -260,57 +215,13 @@ const struct match_entry_t cpudb_intel[] = { { 6, 7, -1, -1, 23, 2, 3072, -1, WOLFDALE , 0, "Wolfdale (Core 2 Duo) 3M" }, { 6, 7, -1, -1, 23, 2, 6144, -1, WOLFDALE , 0, "Wolfdale (Core 2 Duo) 6M" }, { 6, 7, -1, -1, 23, 1, -1, -1, MOBILE_CORE_DUO , 0, "Penryn (Core 2 Duo)" }, + { 6, 7, -1, -1, 23, 2, 1024, -1, PENRYN , 0, "Penryn (Core 2 Duo)" }, { 6, 7, -1, -1, 23, 2, 3072, -1, PENRYN , 0, "Penryn (Core 2 Duo) 3M" }, { 6, 7, -1, -1, 23, 2, 6144, -1, PENRYN , 0, "Penryn (Core 2 Duo) 6M" }, { 6, 7, -1, -1, 23, 4, 2048, -1, QUAD_CORE , 0, "Yorkfield (Core 2 Quad) 2M"}, { 6, 7, -1, -1, 23, 4, 3072, -1, QUAD_CORE , 0, "Yorkfield (Core 2 Quad) 3M"}, { 6, 7, -1, -1, 23, 4, 6144, -1, QUAD_CORE , 0, "Yorkfield (Core 2 Quad) 6M"}, - { 6, 5, -1, -1, 37, 2, -1, -1, NO_CODE , 0, "Unknown Core i3/i5 CPU" }, - { 6, 5, -1, -1, 37, 2, -1, 4096, CORE_I7 , 0, "Arrandale (Core i7)" }, - { 6, 5, -1, -1, 37, 2, -1, 3072, CORE_I5 , 0, "Arrandale (Core i5)" }, - { 6, 5, -1, -1, 37, 2, -1, 4096, CORE_I5 , 0, "Clarkdale (Core i5)" }, - { 6, 5, -1, -1, 37, 4, -1, 8192, CORE_I5 , 0, "Lynnfield (Core i5)" }, - { 6, 5, -1, -1, 37, 2, -1, 3072, CORE_I3 , 0, "Arrandale (Core i3)" }, - { 6, 5, -1, -1, 37, 2, -1, 4096, CORE_I3 , 0, "Clarkdale (Core i3)" }, - - { 6, 10, -1, -1, 42, -1, -1, -1, NO_CODE , 0, "Unknown Sandy Bridge" }, - { 6, 10, -1, -1, 42, -1, -1, -1, CORE_I7 , 0, "Sandy Bridge i7" }, - { 6, 10, -1, -1, 42, 4, -1, -1, CORE_I7 , 0, "Sandy Bridge (Core i7)" }, - { 6, 10, -1, -1, 42, 4, -1, -1, CORE_I5 , 0, "Sandy Bridge (Core i5)" }, - { 6, 10, -1, -1, 42, 2, -1, -1, CORE_I3 , 0, "Sandy Bridge (Core i3)" }, - { 6, 10, -1, -1, 42, 1, -1, -1, CELERON , 0, "Celeron (Sandy Bridge)" }, - { 6, 10, -1, -1, 42, 2, -1, -1, CELERON , 0, "Celeron (Sandy Bridge)" }, - { 6, 10, -1, -1, 42, 2, -1, -1, PENTIUM , 0, "Pentium (Sandy Bridge)" }, - - { 6, 10, -1, -1, 26, 1, -1, -1, CORE_I7 , 0, "Intel Core i7" }, - { 6, 10, -1, -1, 26, 4, -1, -1, CORE_I7 , 0, "Bloomfield (Core i7)" }, - { 6, 10, -1, -1, 30, 4, -1, -1, CORE_I7 , 0, "Lynnfield (Core i7)" }, - { 6, 10, -1, -1, 26, 4, -1, -1, XEON_I7 , 0, "Xeon (Bloomfield)" }, - - { 6, 10, -1, -1, 26, 4, -1, -1, XEON_GAINESTOWN , 0, "Xeon (Gainestown)" }, - { 6, 10, -1, -1, 26, 4, -1, 4096, XEON_GAINESTOWN , 0, "Xeon (Gainestown) 4M" }, - { 6, 10, -1, -1, 26, 4, -1, 8192, XEON_GAINESTOWN , 0, "Xeon (Gainestown) 8M" }, - - { 6, 12, -1, -1, 44, -1, -1, -1, XEON_WESTMERE , 0, "Xeon (Westmere-based)" }, - { 6, 12, -1, -1, 44, 4, -1, 12288, CORE_I7 , 0, "Gulftown (Core i7)" }, - { 6, 12, -1, -1, 44, -1, -1, 12288, XEON_WESTMERE , 0, "Xeon (Gulftown)" }, - - { 6, 13, -1, -1, 45, -1, -1, -1, XEON , 0, "Xeon (Sandy Bridge)" }, - - { 6, 13, -1, -1, 45, -1, -1, -1, CORE_I7 , 0, "Sandy Bridge-E (Core i7)" }, - { 6, 13, -1, -1, 45, -1, -1, -1, CORE_I5 , 0, "Sandy Bridge-E (Core i5)" }, - { 6, 13, -1, -1, 45, -1, -1, -1, CORE_I3 , 0, "Sandy Bridge-E (Core i3)" }, - - { 6, 10, -1, -1, 58, 4, -1, -1, CORE_IVY7 , 0, "Ivy Bridge (Core i7)" }, - { 6, 10, -1, -1, 58, 4, -1, -1, CORE_IVY5 , 0, "Ivy Bridge (Core i5)" }, - { 6, 10, -1, -1, 58, 2, -1, -1, CORE_IVY3 , 0, "Ivy Bridge (Core i3)" }, - - { 6, 12, -1, -1, 60, 4, -1, -1, CORE_HASWELL7 , 0, "Haswell (Core i7)" }, - { 6, 12, -1, -1, 60, 4, -1, -1, CORE_HASWELL5 , 0, "Haswell (Core i5)" }, - { 6, 12, -1, -1, 60, 2, -1, -1, CORE_HASWELL3 , 0, "Haswell (Core i3)" }, - - /* Core microarchitecture-based Xeons: */ { 6, 14, -1, -1, 14, 1, -1, -1, XEON , 0, "Xeon LV" }, { 6, 15, -1, -1, 15, 2, 4096, -1, XEON , _5100, "Xeon (Woodcrest)" }, @@ -324,6 +235,85 @@ const struct match_entry_t cpudb_intel[] = { { 6, 7, -1, -1, 23, 4, 3072, -1, XEON , X3300, "Xeon (Yorkfield/3M)" }, { 6, 7, -1, -1, 23, 4, 6144, -1, XEON , X3300, "Xeon (Yorkfield/6M)" }, + /* Nehalem CPUs (45nm): */ + { 6, 10, -1, -1, 26, 4, -1, -1, XEON_GAINESTOWN , 0, "Gainestown (Xeon)" }, + { 6, 10, -1, -1, 26, 4, -1, 4096, XEON_GAINESTOWN , 0, "Gainestown 4M (Xeon)" }, + { 6, 10, -1, -1, 26, 4, -1, 8192, XEON_GAINESTOWN , 0, "Gainestown 8M (Xeon)" }, + { 6, 10, -1, -1, 26, 4, -1, -1, XEON_I7 , 0, "Bloomfield (Xeon)" }, + { 6, 10, -1, -1, 26, 4, -1, -1, CORE_I7 , 0, "Bloomfield (Core i7)" }, + { 6, 10, -1, -1, 30, 4, -1, -1, CORE_I7 , 0, "Lynnfield (Core i7)" }, + { 6, 5, -1, -1, 37, 4, -1, 8192, CORE_I5 , 0, "Lynnfield (Core i5)" }, + + /* Westmere CPUs (32nm): */ + { 6, 5, -1, -1, 37, 2, -1, -1, NO_CODE , 0, "Unknown Core i3/i5" }, + { 6, 12, -1, -1, 44, -1, -1, -1, XEON_WESTMERE , 0, "Westmere (Xeon)" }, + { 6, 12, -1, -1, 44, -1, -1, 12288, XEON_WESTMERE , 0, "Gulftown (Xeon)" }, + { 6, 12, -1, -1, 44, 4, -1, 12288, CORE_I7 , 0, "Gulftown (Core i7)" }, + { 6, 5, -1, -1, 37, 2, -1, 4096, CORE_I5 , 0, "Clarkdale (Core i5)" }, + { 6, 5, -1, -1, 37, 2, -1, 4096, CORE_I3 , 0, "Clarkdale (Core i3)" }, + { 6, 5, -1, -1, 37, 2, -1, -1, PENTIUM , 0, "Arrandale" }, + { 6, 5, -1, -1, 37, 2, -1, 4096, CORE_I7 , 0, "Arrandale (Core i7)" }, + { 6, 5, -1, -1, 37, 2, -1, 3072, CORE_I5 , 0, "Arrandale (Core i5)" }, + { 6, 5, -1, -1, 37, 2, -1, 3072, CORE_I3 , 0, "Arrandale (Core i3)" }, + + /* Sandy Bridge CPUs (32nm): */ + { 6, 10, -1, -1, 42, -1, -1, -1, NO_CODE , 0, "Unknown Sandy Bridge" }, + { 6, 10, -1, -1, 42, -1, -1, -1, XEON , 0, "Sandy Bridge (Xeon)" }, + { 6, 10, -1, -1, 42, -1, -1, -1, CORE_I7 , 0, "Sandy Bridge (Core i7)" }, + { 6, 10, -1, -1, 42, 4, -1, -1, CORE_I7 , 0, "Sandy Bridge (Core i7)" }, + { 6, 10, -1, -1, 42, 4, -1, -1, CORE_I5 , 0, "Sandy Bridge (Core i5)" }, + { 6, 10, -1, -1, 42, 2, -1, -1, CORE_I3 , 0, "Sandy Bridge (Core i3)" }, + { 6, 10, -1, -1, 42, 2, -1, -1, PENTIUM , 0, "Sandy Bridge (Pentium)" }, + { 6, 10, -1, -1, 42, 1, -1, -1, CELERON , 0, "Sandy Bridge (Celeron)" }, + { 6, 10, -1, -1, 42, 2, -1, -1, CELERON , 0, "Sandy Bridge (Celeron)" }, + { 6, 13, -1, -1, 45, -1, -1, -1, NO_CODE , 0, "Sandy Bridge-E" }, + { 6, 13, -1, -1, 45, -1, -1, -1, XEON , 0, "Sandy Bridge-E (Xeon)" }, + + /* Ivy Bridge CPUs (22nm): */ + { 6, 10, -1, -1, 58, -1, -1, -1, XEON , 0, "Ivy Bridge (Xeon)" }, + { 6, 10, -1, -1, 58, 4, -1, -1, CORE_IVY7 , 0, "Ivy Bridge (Core i7)" }, + { 6, 10, -1, -1, 58, 4, -1, -1, CORE_IVY5 , 0, "Ivy Bridge (Core i5)" }, + { 6, 10, -1, -1, 58, 2, -1, -1, CORE_IVY3 , 0, "Ivy Bridge (Core i3)" }, + { 6, 10, -1, -1, 58, 2, -1, -1, PENTIUM , 0, "Ivy Bridge (Pentium)" }, + { 6, 10, -1, -1, 58, 1, -1, -1, CELERON , 0, "Ivy Bridge (Celeron)" }, + { 6, 10, -1, -1, 58, 2, -1, -1, CELERON , 0, "Ivy Bridge (Celeron)" }, + { 6, 14, -1, -1, 62, -1, -1, -1, NO_CODE , 0, "Ivy Bridge-E" }, + + /* Haswell CPUs (22nm): */ + { 6, 12, -1, -1, 60, -1, -1, -1, XEON , 0, "Haswell (Xeon)" }, + { 6, 12, -1, -1, 60, 4, -1, -1, CORE_HASWELL7 , 0, "Haswell (Core i7)" }, + { 6, 5, -1, -1, 69, 4, -1, -1, CORE_HASWELL7 , 0, "Haswell (Core i7)" }, + { 6, 12, -1, -1, 60, 4, -1, -1, CORE_HASWELL5 , 0, "Haswell (Core i5)" }, + { 6, 5, -1, -1, 69, 4, -1, -1, CORE_HASWELL5 , 0, "Haswell (Core i5)" }, + { 6, 12, -1, -1, 60, 2, -1, -1, CORE_HASWELL3 , 0, "Haswell (Core i3)" }, + { 6, 5, -1, -1, 69, 2, -1, -1, CORE_HASWELL3 , 0, "Haswell (Core i3)" }, + { 6, 12, -1, -1, 60, 2, -1, -1, PENTIUM , 0, "Haswell (Pentium)" }, + { 6, 12, -1, -1, 60, 2, -1, -1, CELERON , 0, "Haswell (Celeron)" }, + { 6, 12, -1, -1, 60, 1, -1, -1, CELERON , 0, "Haswell (Celeron)" }, + { 6, 15, -1, -1, 63, -1, -1, -1, NO_CODE , 0, "Haswell-E" }, + + /* Broadwell CPUs (14nm): */ + { 6, 7, -1, -1, 71, 4, -1, -1, CORE_BROADWELL7 , 0, "Broadwell (Core i7)" }, + { 6, 7, -1, -1, 71, 4, -1, -1, CORE_BROADWELL5 , 0, "Broadwell (Core i5)" }, + { 6, 13, -1, -1, 61, 4, -1, -1, CORE_BROADWELL7 , 0, "Broadwell-U (Core i7)" }, + { 6, 13, -1, -1, 61, 2, -1, -1, CORE_BROADWELL7 , 0, "Broadwell-U (Core i7)" }, + { 6, 13, -1, -1, 61, 2, -1, -1, CORE_BROADWELL5 , 0, "Broadwell-U (Core i5)" }, + { 6, 13, -1, -1, 61, 2, -1, -1, CORE_BROADWELL3 , 0, "Broadwell-U (Core i3)" }, + { 6, 13, -1, -1, 61, 2, -1, -1, PENTIUM , 0, "Broadwell-U (Pentium)" }, + { 6, 13, -1, -1, 61, 2, -1, -1, CELERON , 0, "Broadwell-U (Celeron)" }, + { 6, 13, -1, -1, 61, 2, -1, -1, NA , 0, "Broadwell-U (Core M)" }, + { 6, 15, -1, -1, 79, 2, -1, -1, CORE_BROADWELL3 , 0, "Broadwell-E (Core i3)" }, + { 6, 15, -1, -1, 79, 2, -1, -1, CORE_BROADWELL5 , 0, "Broadwell-E (Core i5)" }, + { 6, 15, -1, -1, 79, 4, -1, -1, CORE_BROADWELL5 , 0, "Broadwell-E (Core i5)" }, + { 6, 15, -1, -1, 79, 2, -1, -1, CORE_BROADWELL7 , 0, "Broadwell-E (Core i7)" }, + { 6, 15, -1, -1, 79, 4, -1, -1, CORE_BROADWELL7 , 0, "Broadwell-E (Core i7)" }, + + /* Skylake CPUs (14nm): */ + { 6, 14, -1, -1, 94, 4, -1, -1, CORE_BROADWELL7 , 0, "Skylake (Core i7)" }, + { 6, 14, -1, -1, 94, 4, -1, -1, CORE_BROADWELL5 , 0, "Skylake (Core i5)" }, + { 6, 14, -1, -1, 94, 4, -1, -1, CORE_BROADWELL3 , 0, "Skylake (Core i3)" }, + { 6, 14, -1, -1, 94, 4, -1, -1, PENTIUM , 0, "Skylake (Pentium)" }, + /* Itaniums */ { 7, -1, -1, -1, -1, 1, -1, -1, NO_CODE , 0, "Itanium" }, { 15, -1, -1, 16, -1, 1, -1, -1, NO_CODE , 0, "Itanium 2" }, @@ -343,7 +333,6 @@ static void load_intel_features(struct cpu_raw_data_t* raw, struct cpu_id_t* dat { 31, CPU_FEATURE_PBE }, }; const struct feature_map_t matchtable_ecx1[] = { - { 1, CPU_FEATURE_PCLMUL }, { 2, CPU_FEATURE_DTS64 }, { 4, CPU_FEATURE_DS_CPL }, { 5, CPU_FEATURE_VMX }, @@ -354,37 +343,45 @@ static void load_intel_features(struct cpu_raw_data_t* raw, struct cpu_id_t* dat { 14, CPU_FEATURE_XTPR }, { 15, CPU_FEATURE_PDCM }, { 18, CPU_FEATURE_DCA }, - { 20, CPU_FEATURE_SSE4_2 }, - { 22, CPU_FEATURE_MOVBE }, - { 25, CPU_FEATURE_AES }, - { 26, CPU_FEATURE_XSAVE }, - { 27, CPU_FEATURE_OSXSAVE }, - { 28, CPU_FEATURE_AVX }, - { 30, CPU_FEATURE_RDRAND }, - }; - const struct feature_map_t matchtable_ebx7[] = { - { 5, CPU_FEATURE_AVX2 }, + { 21, CPU_FEATURE_X2APIC }, }; const struct feature_map_t matchtable_edx81[] = { { 20, CPU_FEATURE_XD }, }; + const struct feature_map_t matchtable_ebx7[] = { + { 2, CPU_FEATURE_SGX }, + { 4, CPU_FEATURE_HLE }, + { 11, CPU_FEATURE_RTM }, + { 16, CPU_FEATURE_AVX512F }, + { 17, CPU_FEATURE_AVX512DQ }, + { 18, CPU_FEATURE_RDSEED }, + { 19, CPU_FEATURE_ADX }, + { 26, CPU_FEATURE_AVX512PF }, + { 27, CPU_FEATURE_AVX512ER }, + { 28, CPU_FEATURE_AVX512CD }, + { 29, CPU_FEATURE_SHA_NI }, + { 30, CPU_FEATURE_AVX512BW }, + { 31, CPU_FEATURE_AVX512VL }, + }; if (raw->basic_cpuid[0][0] >= 1) { match_features(matchtable_edx1, COUNT_OF(matchtable_edx1), raw->basic_cpuid[1][3], data); match_features(matchtable_ecx1, COUNT_OF(matchtable_ecx1), raw->basic_cpuid[1][2], data); } - if (raw->basic_cpuid[0][0] >= 7) { - match_features(matchtable_ebx7, COUNT_OF(matchtable_ebx7), raw->basic_cpuid[7][1], data); - } if (raw->ext_cpuid[0][0] >= 1) { match_features(matchtable_edx81, COUNT_OF(matchtable_edx81), raw->ext_cpuid[1][3], data); } + // detect TSX/AVX512: + if (raw->basic_cpuid[0][0] >= 7) { + match_features(matchtable_ebx7, COUNT_OF(matchtable_ebx7), raw->basic_cpuid[7][1], data); + } } enum _cache_type_t { L1I, L1D, L2, - L3 + L3, + L4 }; typedef enum _cache_type_t cache_type_t; @@ -409,6 +406,12 @@ static void check_case(uint8_t on, cache_type_t cache, int size, int assoc, int data->l3_cache = size; data->l3_assoc = assoc; data->l3_cacheline = linesize; + break; + case L4: + data->l4_cache = size; + data->l4_assoc = assoc; + data->l4_cacheline = linesize; + break; default: break; } @@ -529,6 +532,8 @@ static void decode_intel_deterministic_cache_info(struct cpu_raw_data_t* raw, type = L2; else if (level == 3 && typenumber == 3) type = L3; + else if (level == 4 && typenumber == 3) + type = L4; else { warnf("deterministic_cache: unknown level/typenumber combo (%d/%d), cannot\n", level, typenumber); warnf("deterministic_cache: recognize cache type\n"); @@ -561,8 +566,12 @@ static int decode_intel_extended_topology(struct cpu_raw_data_t* raw, } } if (num_smt == -1 || num_core == -1) return 0; - data->num_cores = num_core / num_smt; data->num_logical_cpus = num_core; + data->num_cores = num_core / num_smt; + // make sure num_cores is at least 1. In VMs, the CPUID instruction + // is rigged and may give nonsensical results, but we should at least + // avoid outputs like data->num_cores == 0. + if (data->num_cores <= 0) data->num_cores = 1; return 1; } @@ -587,7 +596,9 @@ static void decode_intel_number_of_cores(struct cpu_raw_data_t* raw, data->num_logical_cpus = logical_cpus; } else { data->num_cores = 1; - data->num_logical_cpus = (logical_cpus >= 2 ? logical_cpus : 2); + data->num_logical_cpus = (logical_cpus >= 1 ? logical_cpus : 1); + if (data->num_logical_cpus == 1) + data->flags[CPU_FEATURE_HT] = 0; } } else { data->num_cores = data->num_logical_cpus = 1; @@ -596,7 +607,7 @@ static void decode_intel_number_of_cores(struct cpu_raw_data_t* raw, static intel_code_t get_brand_code(struct cpu_id_t* data) { - intel_code_t code = NO_CODE; + intel_code_t code = (intel_code_t) NO_CODE; int i, need_matchtable = 1, core_ix_base = 0; const char* bs = data->brand_str; const char* s; @@ -607,15 +618,16 @@ static intel_code_t get_brand_code(struct cpu_id_t* data) { CELERON, "Celeron" }, { MOBILE_PENTIUM_M, "Pentium(R) M" }, { CORE_SOLO, "Pentium(R) Dual CPU" }, + { CORE_SOLO, "Pentium(R) Dual-Core" }, { PENTIUM_D, "Pentium(R) D" }, { PENTIUM, "Pentium" }, { CORE_SOLO, "Genuine Intel(R) CPU" }, { CORE_SOLO, "Intel(R) Core(TM)" }, { ATOM_DIAMONDVILLE, "Atom(TM) CPU [N ][23]## " }, { ATOM_SILVERTHORNE, "Atom(TM) CPU Z" }, - { ATOM_PINEVIEW, "Atom(TM) CPU D" }, - { ATOM_CEDARVIEW, "Atom(TM) CPU N####" }, - { ATOM, "Atom(TM) CPU" }, + { ATOM_PINEVIEW, "Atom(TM) CPU [ND][45]## " }, + { ATOM_CEDARVIEW, "Atom(TM) CPU [ND]#### " }, + { ATOM_UNKNOWN, "Atom(TM) CPU" }, }; if (strstr(bs, "Mobile")) { @@ -637,6 +649,9 @@ static intel_code_t get_brand_code(struct cpu_id_t* data) /* if it has FMA, then it is at least Haswell */ if (data->flags[CPU_FEATURE_FMA3]) core_ix_base = CORE_HASWELL3; + /* if it has RTM, then it is at least a Broadwell-E or Skylake */ + if (data->flags[CPU_FEATURE_RDSEED]) + core_ix_base = CORE_BROADWELL3; switch (bs[i + 9]) { case '3': code = core_ix_base + 0; break; @@ -659,7 +674,8 @@ static intel_code_t get_brand_code(struct cpu_id_t* data) code = XEON_GAINESTOWN; else if (match_pattern(bs, "[ELXW]56##")) code = XEON_WESTMERE; - else if (data->l3_cache > 0) + else if (data->l3_cache > 0 && data->family == 16) + /* restrict by family, since later Xeons also have L3 ... */ code = XEON_IRWIN; } if (code == XEONMP && data->l3_cache > 0) @@ -770,8 +786,75 @@ static intel_model_t get_model_code(struct cpu_id_t* data) #undef HAVE } -int cpuid_identify_intel(struct cpu_raw_data_t* raw, struct cpu_id_t* data) +static void decode_intel_sgx_features(const struct cpu_raw_data_t* raw, struct cpu_id_t* data) { + struct cpu_epc_t epc; + int i; + + if (raw->basic_cpuid[0][0] < 0x12) return; // no 12h leaf + if (raw->basic_cpuid[0x12][0] == 0) return; // no sub-leafs available, probably it's disabled by BIOS + + // decode sub-leaf 0: + if (raw->basic_cpuid[0x12][0] & 1) data->sgx.flags[INTEL_SGX1] = 1; + if (raw->basic_cpuid[0x12][0] & 2) data->sgx.flags[INTEL_SGX2] = 1; + if (data->sgx.flags[INTEL_SGX1] || data->sgx.flags[INTEL_SGX2]) + data->sgx.present = 1; + data->sgx.misc_select = raw->basic_cpuid[0x12][1]; + data->sgx.max_enclave_32bit = (raw->basic_cpuid[0x12][3] ) & 0xff; + data->sgx.max_enclave_64bit = (raw->basic_cpuid[0x12][3] >> 8) & 0xff; + + // decode sub-leaf 1: + data->sgx.secs_attributes = raw->intel_fn12h[1][0] | (((uint64_t) raw->intel_fn12h[1][1]) << 32); + data->sgx.secs_xfrm = raw->intel_fn12h[1][2] | (((uint64_t) raw->intel_fn12h[1][3]) << 32); + + // decode higher-order subleafs, whenever present: + data->sgx.num_epc_sections = -1; + for (i = 0; i < 1000000; i++) { + epc = cpuid_get_epc(i, raw); + if (epc.length == 0) { + debugf(2, "SGX: epc section request for %d returned null, no more EPC sections.\n", i); + data->sgx.num_epc_sections = i; + break; + } + } + if (data->sgx.num_epc_sections == -1) { + debugf(1, "SGX: warning: seems to be infinitude of EPC sections.\n"); + data->sgx.num_epc_sections = 1000000; + } +} + +struct cpu_epc_t cpuid_get_epc(int index, const struct cpu_raw_data_t* raw) +{ + uint32_t regs[4]; + struct cpu_epc_t retval = {0, 0}; + if (raw && index < MAX_INTELFN12H_LEVEL - 2) { + // this was queried already, use the data: + memcpy(regs, raw->intel_fn12h[2 + index], sizeof(regs)); + } else { + // query this ourselves: + regs[0] = 0x12; + regs[2] = 2 + index; + regs[1] = regs[3] = 0; + cpu_exec_cpuid_ext(regs); + } + + // decode values: + if ((regs[0] & 0xf) == 0x1) { + retval.start_addr |= (regs[0] & 0xfffff000); // bits [12, 32) -> bits [12, 32) + retval.start_addr |= ((uint64_t) (regs[1] & 0x000fffff)) << 32; // bits [0, 20) -> bits [32, 52) + retval.length |= (regs[2] & 0xfffff000); // bits [12, 32) -> bits [12, 32) + retval.length |= ((uint64_t) (regs[3] & 0x000fffff)) << 32; // bits [0, 20) -> bits [32, 52) + } + return retval; +} + +int cpuid_identify_intel(struct cpu_raw_data_t* raw, struct cpu_id_t* data, struct internal_id_info_t* internal) +{ + intel_code_t brand_code; + intel_model_t model_code; + int i; + char* brand_code_str = NULL; + load_intel_features(raw, data); if (raw->basic_cpuid[0][0] >= 4) { /* Deterministic way is preferred, being more generic */ @@ -780,8 +863,31 @@ int cpuid_identify_intel(struct cpu_raw_data_t* raw, struct cpu_id_t* data) decode_intel_oldstyle_cache_info(raw, data); } decode_intel_number_of_cores(raw, data); - match_cpu_codename(cpudb_intel, COUNT_OF(cpudb_intel), data, - get_brand_code(data), get_model_code(data)); + + brand_code = get_brand_code(data); + model_code = get_model_code(data); + for (i = 0; i < COUNT_OF(intel_bcode_str); i++) { + if (brand_code == intel_bcode_str[i].code) { + brand_code_str = intel_bcode_str[i].str; + break; + } + } + if (brand_code_str) + debugf(2, "Detected Intel brand code: %d (%s)\n", brand_code, brand_code_str); + else + debugf(2, "Detected Intel brand code: %d\n", brand_code); + debugf(2, "Detected Intel model code: %d\n", model_code); + + internal->code.intel = brand_code; + + if (data->flags[CPU_FEATURE_SGX]) { + debugf(2, "SGX seems to be present, decoding...\n"); + // if SGX is indicated by the CPU, verify its presence: + decode_intel_sgx_features(raw, data); + } + + internal->score = match_cpu_codename(cpudb_intel, COUNT_OF(cpudb_intel), data, + brand_code, model_code); return 0; } diff --git a/contrib/libcpuid/include/cpuid/recog_intel.h b/contrib/libcpuid/include/libcpuid/recog_intel.h similarity index 96% rename from contrib/libcpuid/include/cpuid/recog_intel.h rename to contrib/libcpuid/include/libcpuid/recog_intel.h index 67279373271..b99c783bf43 100644 --- a/contrib/libcpuid/include/cpuid/recog_intel.h +++ b/contrib/libcpuid/include/libcpuid/recog_intel.h @@ -26,7 +26,7 @@ #ifndef __RECOG_INTEL_H__ #define __RECOG_INTEL_H__ -int cpuid_identify_intel(struct cpu_raw_data_t* raw, struct cpu_id_t* data); +int cpuid_identify_intel(struct cpu_raw_data_t* raw, struct cpu_id_t* data, struct internal_id_info_t* internal); void cpuid_get_list_intel(struct cpu_list_t* list); #endif /*__RECOG_INTEL_H__*/ diff --git a/contrib/libfarmhash/CMakeLists.txt b/contrib/libfarmhash/CMakeLists.txt index 16471da3f2f..20bba58cde7 100644 --- a/contrib/libfarmhash/CMakeLists.txt +++ b/contrib/libfarmhash/CMakeLists.txt @@ -6,4 +6,4 @@ if (MSVC) target_compile_definitions (farmhash PRIVATE FARMHASH_NO_BUILTIN_EXPECT=1) endif () -target_include_directories (farmhash PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) +target_include_directories (farmhash PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/contrib/libmetrohash/CMakeLists.txt b/contrib/libmetrohash/CMakeLists.txt index 6947b92e054..2bd5628d0f8 100644 --- a/contrib/libmetrohash/CMakeLists.txt +++ b/contrib/libmetrohash/CMakeLists.txt @@ -10,4 +10,4 @@ add_library(metrohash src/metrohash128.cpp ${SOURCES_SSE42_ONLY}) -target_include_directories(metrohash PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +target_include_directories(metrohash PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src) diff --git a/contrib/libpcg-random/include/pcg_extras.hpp b/contrib/libpcg-random/include/pcg_extras.hpp index 929c756b151..08e8dbacd84 100644 --- a/contrib/libpcg-random/include/pcg_extras.hpp +++ b/contrib/libpcg-random/include/pcg_extras.hpp @@ -81,7 +81,7 @@ #define PCG_128BIT_CONSTANT(high,low) \ ((pcg128_t(high) << 64) + low) #else - #include "pcg_uint128.hpp" + #include "pcg_uint128.hpp" // Y_IGNORE namespace pcg_extras { typedef pcg_extras::uint_x4 pcg128_t; } diff --git a/contrib/librdkafka b/contrib/librdkafka index c3d50eb6137..7478b5ef16a 160000 --- a/contrib/librdkafka +++ b/contrib/librdkafka @@ -1 +1 @@ -Subproject commit c3d50eb613704fb9c8ab3bce95a88275cb5875b7 +Subproject commit 7478b5ef16aadd6543fe38bc6a2deb895c70da98 diff --git a/contrib/librdkafka-cmake/CMakeLists.txt b/contrib/librdkafka-cmake/CMakeLists.txt new file mode 100644 index 00000000000..90421cfb31d --- /dev/null +++ b/contrib/librdkafka-cmake/CMakeLists.txt @@ -0,0 +1,60 @@ +set(RDKAFKA_SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/librdkafka/src) + +set(SRCS +${RDKAFKA_SOURCE_DIR}/crc32c.c +${RDKAFKA_SOURCE_DIR}/rdaddr.c +${RDKAFKA_SOURCE_DIR}/rdavl.c +${RDKAFKA_SOURCE_DIR}/rdbuf.c +${RDKAFKA_SOURCE_DIR}/rdcrc32.c +${RDKAFKA_SOURCE_DIR}/rdkafka.c +${RDKAFKA_SOURCE_DIR}/rdkafka_assignor.c +${RDKAFKA_SOURCE_DIR}/rdkafka_broker.c +${RDKAFKA_SOURCE_DIR}/rdkafka_buf.c +${RDKAFKA_SOURCE_DIR}/rdkafka_cgrp.c +${RDKAFKA_SOURCE_DIR}/rdkafka_conf.c +${RDKAFKA_SOURCE_DIR}/rdkafka_event.c +${RDKAFKA_SOURCE_DIR}/rdkafka_feature.c +${RDKAFKA_SOURCE_DIR}/rdkafka_lz4.c +${RDKAFKA_SOURCE_DIR}/rdkafka_metadata.c +${RDKAFKA_SOURCE_DIR}/rdkafka_metadata_cache.c +${RDKAFKA_SOURCE_DIR}/rdkafka_msg.c +${RDKAFKA_SOURCE_DIR}/rdkafka_msgset_reader.c +${RDKAFKA_SOURCE_DIR}/rdkafka_msgset_writer.c +${RDKAFKA_SOURCE_DIR}/rdkafka_offset.c +${RDKAFKA_SOURCE_DIR}/rdkafka_op.c +${RDKAFKA_SOURCE_DIR}/rdkafka_partition.c +${RDKAFKA_SOURCE_DIR}/rdkafka_pattern.c +${RDKAFKA_SOURCE_DIR}/rdkafka_queue.c +${RDKAFKA_SOURCE_DIR}/rdkafka_range_assignor.c +${RDKAFKA_SOURCE_DIR}/rdkafka_request.c +${RDKAFKA_SOURCE_DIR}/rdkafka_roundrobin_assignor.c +${RDKAFKA_SOURCE_DIR}/rdkafka_sasl.c +${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_plain.c +${RDKAFKA_SOURCE_DIR}/rdkafka_subscription.c +${RDKAFKA_SOURCE_DIR}/rdkafka_timer.c +${RDKAFKA_SOURCE_DIR}/rdkafka_topic.c +${RDKAFKA_SOURCE_DIR}/rdkafka_transport.c +${RDKAFKA_SOURCE_DIR}/rdkafka_interceptor.c +${RDKAFKA_SOURCE_DIR}/rdkafka_header.c +${RDKAFKA_SOURCE_DIR}/rdlist.c +${RDKAFKA_SOURCE_DIR}/rdlog.c +${RDKAFKA_SOURCE_DIR}/rdmurmur2.c +${RDKAFKA_SOURCE_DIR}/rdports.c +${RDKAFKA_SOURCE_DIR}/rdrand.c +${RDKAFKA_SOURCE_DIR}/rdregex.c +${RDKAFKA_SOURCE_DIR}/rdstring.c +${RDKAFKA_SOURCE_DIR}/rdunittest.c +${RDKAFKA_SOURCE_DIR}/rdvarint.c +${RDKAFKA_SOURCE_DIR}/snappy.c +${RDKAFKA_SOURCE_DIR}/tinycthread.c +${RDKAFKA_SOURCE_DIR}/xxhash.c +${RDKAFKA_SOURCE_DIR}/lz4.c +${RDKAFKA_SOURCE_DIR}/lz4frame.c +${RDKAFKA_SOURCE_DIR}/lz4hc.c +${RDKAFKA_SOURCE_DIR}/rdgz.c +) + +add_library(rdkafka ${LINK_MODE} ${SRCS}) +target_include_directories(rdkafka PRIVATE include) +target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR}) +target_link_libraries(rdkafka PUBLIC ${ZLIB_LIBRARIES} ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY}) diff --git a/contrib/librdkafka-cmake/config.h b/contrib/librdkafka-cmake/config.h new file mode 100644 index 00000000000..68e93a10ff1 --- /dev/null +++ b/contrib/librdkafka-cmake/config.h @@ -0,0 +1,74 @@ +// Automatically generated by ./configure +#ifndef _CONFIG_H_ +#define _CONFIG_H_ +#define ARCH "x86_64" +#define CPU "generic" +#define WITHOUT_OPTIMIZATION 0 +#define ENABLE_DEVEL 0 +#define ENABLE_VALGRIND 0 +#define ENABLE_REFCNT_DEBUG 0 +#define ENABLE_SHAREDPTR_DEBUG 0 +#define ENABLE_LZ4_EXT 1 +#define ENABLE_SSL 1 +//#define ENABLE_SASL 1 +#define MKL_APP_NAME "librdkafka" +#define MKL_APP_DESC_ONELINE "The Apache Kafka C/C++ library" +// distro +//#define SOLIB_EXT ".so" +// gcc +//#define WITH_GCC 1 +// gxx +//#define WITH_GXX 1 +// pkgconfig +//#define WITH_PKGCONFIG 1 +// install +//#define WITH_INSTALL 1 +// PIC +//#define HAVE_PIC 1 +// gnulib +//#define WITH_GNULD 1 +// __atomic_32 +#define HAVE_ATOMICS_32 1 +// __atomic_32 +#define HAVE_ATOMICS_32_ATOMIC 1 +// atomic_32 +#define ATOMIC_OP32(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST) +// __atomic_64 +#define HAVE_ATOMICS_64 1 +// __atomic_64 +#define HAVE_ATOMICS_64_ATOMIC 1 +// atomic_64 +#define ATOMIC_OP64(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST) +// atomic_64 +#define ATOMIC_OP(OP1,OP2,PTR,VAL) __atomic_ ## OP1 ## _ ## OP2(PTR, VAL, __ATOMIC_SEQ_CST) +// parseversion +#define RDKAFKA_VERSION_STR "0.11.4" +// parseversion +#define MKL_APP_VERSION "0.11.4" +// libdl +//#define WITH_LIBDL 1 +// WITH_PLUGINS +//#define WITH_PLUGINS 1 +// zlib +#define WITH_ZLIB 1 +// WITH_SNAPPY +#define WITH_SNAPPY 1 +// WITH_SOCKEM +#define WITH_SOCKEM 1 +// libssl +#define WITH_SSL 1 +// WITH_SASL_SCRAM +//#define WITH_SASL_SCRAM 1 +// crc32chw +#define WITH_CRC32C_HW 1 +// regex +#define HAVE_REGEX 1 +// strndup +#define HAVE_STRNDUP 1 +// strerror_r +#define HAVE_STRERROR_R 1 +// pthread_setname_gnu +#define HAVE_PTHREAD_SETNAME_GNU 1 +// python +//#define HAVE_PYTHON 1 +#endif /* _CONFIG_H_ */ diff --git a/contrib/librdkafka-cmake/include/README b/contrib/librdkafka-cmake/include/README new file mode 100644 index 00000000000..58fa024e68a --- /dev/null +++ b/contrib/librdkafka-cmake/include/README @@ -0,0 +1 @@ +This directory is needed because rdkafka files have #include "../config.h" diff --git a/contrib/libsparsehash/README b/contrib/libsparsehash/README index 8c806b42a38..527cfa1d304 100644 --- a/contrib/libsparsehash/README +++ b/contrib/libsparsehash/README @@ -1,7 +1,7 @@ This directory contains several hash-map implementations, similar in API to SGI's hash_map class, but with different performance characteristics. sparse_hash_map uses very little space overhead, 1-2 -bits per entry. dense_hash_map is very fast, particulary on lookup. +bits per entry. dense_hash_map is very fast, particularly on lookup. (sparse_hash_set and dense_hash_set are the set versions of these routines.) On the other hand, these classes have requirements that may not make them appropriate for all applications. diff --git a/contrib/libsparsehash/sparsehash/sparsetable b/contrib/libsparsehash/sparsehash/sparsetable index efbeaac0a69..d162623a5f5 100644 --- a/contrib/libsparsehash/sparsehash/sparsetable +++ b/contrib/libsparsehash/sparsehash/sparsetable @@ -1088,7 +1088,7 @@ class sparsegroup { // This is equivalent to memmove(), but faster on my Intel P4, // at least with gcc4.1 -O2 / glibc 2.3.6. for (size_type i = settings.num_buckets; i > offset; --i) - memcpy(group + i, group + i-1, sizeof(*group)); + memcpy(static_cast(group + i), group + i-1, sizeof(*group)); } // Create space at group[offset], without special assumptions about value_type @@ -1154,7 +1154,7 @@ class sparsegroup { // at lesat with gcc4.1 -O2 / glibc 2.3.6. assert(settings.num_buckets > 0); for (size_type i = offset; i < settings.num_buckets-1; ++i) - memcpy(group + i, group + i+1, sizeof(*group)); // hopefully inlined! + memcpy(static_cast(group + i), group + i+1, sizeof(*group)); // hopefully inlined! group = settings.realloc_or_die(group, settings.num_buckets-1); } diff --git a/contrib/libtcmalloc/CMakeLists.txt b/contrib/libtcmalloc/CMakeLists.txt index 5bf61f98bea..d7f52e1d384 100644 --- a/contrib/libtcmalloc/CMakeLists.txt +++ b/contrib/libtcmalloc/CMakeLists.txt @@ -1,4 +1,3 @@ - message (STATUS "Building: tcmalloc_minimal_internal") add_library (tcmalloc_minimal_internal diff --git a/contrib/llvm b/contrib/llvm new file mode 160000 index 00000000000..163def21781 --- /dev/null +++ b/contrib/llvm @@ -0,0 +1 @@ +Subproject commit 163def217817c90fb982a6daf384744d8472b92b diff --git a/contrib/mariadb-connector-c b/contrib/mariadb-connector-c new file mode 160000 index 00000000000..a0fd36cc5a5 --- /dev/null +++ b/contrib/mariadb-connector-c @@ -0,0 +1 @@ +Subproject commit a0fd36cc5a5313414a5a2ebe9322577a29b4782a diff --git a/contrib/mariadb-connector-c-cmake/CMakeLists.txt b/contrib/mariadb-connector-c-cmake/CMakeLists.txt new file mode 100644 index 00000000000..4c1184b3edb --- /dev/null +++ b/contrib/mariadb-connector-c-cmake/CMakeLists.txt @@ -0,0 +1,66 @@ +set(MARIADB_CLIENT_SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/mariadb-connector-c) +set(MARIADB_CLIENT_BINARY_DIR ${CMAKE_BINARY_DIR}/contrib/mariadb-connector-c) + +set(SRCS +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/bmove_upp.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/get_password.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_alloc.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_array.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_charset.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_compress.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_context.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_default.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_dtoa.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_errmsg.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_hash.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_init.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_io.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_list.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_ll2str.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_loaddata.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_net.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_password.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_pvio.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/mariadb_async.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/mariadb_charset.c +#${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/mariadb_dyncol.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/mariadb_lib.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/mariadb_stmt.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_sha1.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_stmt_codec.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_string.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_time.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/ma_tls.c +#${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/secure/gnutls.c +#${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/secure/ma_schannel.c +${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/secure/openssl.c +#${MARIADB_CLIENT_SOURCE_DIR}/libmariadb/secure/schannel.c +#${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/auth_gssapi_client.c +#${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/dialog.c +#${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/gssapi_client.c +#${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/gssapi_errmsg.c +${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/mariadb_cleartext.c +${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/my_auth.c +${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/old_password.c +${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/sha256_pw.c +#${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/sspi_client.c +#${MARIADB_CLIENT_SOURCE_DIR}/plugins/auth/sspi_errmsg.c +${MARIADB_CLIENT_SOURCE_DIR}/plugins/connection/aurora.c +${MARIADB_CLIENT_SOURCE_DIR}/plugins/connection/replication.c +#${MARIADB_CLIENT_SOURCE_DIR}/plugins/io/remote_io.c +#${MARIADB_CLIENT_SOURCE_DIR}/plugins/pvio/pvio_npipe.c +#${MARIADB_CLIENT_SOURCE_DIR}/plugins/pvio/pvio_shmem.c +${MARIADB_CLIENT_SOURCE_DIR}/plugins/pvio/pvio_socket.c +#${MARIADB_CLIENT_SOURCE_DIR}/plugins/trace/trace_example.c +${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/libmariadb/ma_client_plugin.c +) + +add_library(mysqlclient STATIC ${SRCS}) + +target_link_libraries(mysqlclient ${OPENSSL_LIBRARIES}) + +target_include_directories(mysqlclient PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/include) +target_include_directories(mysqlclient PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/common/include) +target_include_directories(mysqlclient PUBLIC ${MARIADB_CLIENT_SOURCE_DIR}/include) + +target_compile_definitions(mysqlclient PRIVATE -D THREAD -D HAVE_OPENSSL -D HAVE_TLS) diff --git a/contrib/mariadb-connector-c-cmake/common/include/mysql/mysql.h b/contrib/mariadb-connector-c-cmake/common/include/mysql/mysql.h new file mode 100644 index 00000000000..741c7ba03c9 --- /dev/null +++ b/contrib/mariadb-connector-c-cmake/common/include/mysql/mysql.h @@ -0,0 +1 @@ +#include diff --git a/contrib/mariadb-connector-c-cmake/common/include/mysql/mysqld_error.h b/contrib/mariadb-connector-c-cmake/common/include/mysql/mysqld_error.h new file mode 100644 index 00000000000..95d26eef163 --- /dev/null +++ b/contrib/mariadb-connector-c-cmake/common/include/mysql/mysqld_error.h @@ -0,0 +1 @@ +#include diff --git a/contrib/mariadb-connector-c-cmake/linux_x86_64/include/config.h b/contrib/mariadb-connector-c-cmake/linux_x86_64/include/config.h new file mode 100644 index 00000000000..90c42c97df6 --- /dev/null +++ b/contrib/mariadb-connector-c-cmake/linux_x86_64/include/config.h @@ -0,0 +1,269 @@ + +/* + * Include file constants (processed in LibmysqlIncludeFiles.txt 1 + */ +#define HAVE_ALLOCA_H 1 +/* #undef HAVE_BIGENDIAN */ +#define HAVE_SETLOCALE 1 +#define HAVE_NL_LANGINFO 1 +#define HAVE_ARPA_INET_H 1 +#define HAVE_CRYPT_H 1 +#define HAVE_DIRENT_H 1 +#define HAVE_DLFCN_H 1 +#define HAVE_EXECINFO_H 1 +#define HAVE_FCNTL_H 1 +#define HAVE_FENV_H 1 +#define HAVE_FLOAT_H 1 +/* #undef HAVE_FPU_CONTROL_H */ +#define HAVE_GRP_H 1 +/* #undef HAVE_IEEEFP_H */ +#define HAVE_LIMITS_H 1 +#define HAVE_MALLOC_H 1 +#define HAVE_MEMORY_H 1 +#define HAVE_NETINET_IN_H 1 +#define HAVE_PATHS_H 1 +#define HAVE_PWD_H 1 +#define HAVE_SCHED_H 1 +/* #undef HAVE_SELECT_H */ +#define HAVE_STDDEF_H 1 +#define HAVE_STDINT_H 1 +#define HAVE_STDLIB_H 1 +#define HAVE_STRING_H 1 +#define HAVE_STRINGS_H 1 +/* #undef HAVE_SYNCH_H */ +/* #undef HAVE_SYS_FPU_H */ +#define HAVE_SYS_IOCTL_H 1 +#define HAVE_SYS_IPC_H 1 +#define HAVE_SYS_MMAN_H 1 +#define HAVE_SYS_PRCTL_H 1 +#define HAVE_SYS_SELECT_H 1 +#define HAVE_SYS_SHM_H 1 +#define HAVE_SYS_SOCKET_H 1 +#define HAVE_SYS_STAT_H 1 +/* #undef HAVE_SYS_STREAM_H */ +#define HAVE_SYS_TIMEB_H 1 +#define HAVE_SYS_TYPES_H 1 +#define HAVE_SYS_UN_H 1 +/* #undef HAVE_SYSENT_H */ +#define HAVE_TERMIO_H 1 +#define HAVE_TERMIOS_H 1 +#define HAVE_UNISTD_H 1 +#define HAVE_UTIME_H 1 +#define HAVE_UCONTEXT_H 1 + +/* + * function definitions - processed in LibmysqlFunctions.txt + */ +#define HAVE_ACCESS 1 +/* #undef HAVE_AIOWAIT */ +#define HAVE_ALARM 1 +/* #undef HAVE_ALLOCA */ +#define HAVE_BCMP 1 +/* #undef HAVE_BFILL */ +/* #undef HAVE_BMOVE */ +#define HAVE_BZERO 1 +#define HAVE_CLOCK_GETTIME 1 +/* #undef HAVE_COMPRESS */ +/* #undef HAVE_CRYPT */ +#define HAVE_DLERROR 1 +#define HAVE_DLOPEN 1 +#define HAVE_FCHMOD 1 +#define HAVE_FCNTL 1 +/* #undef HAVE_FCONVERT */ +#define HAVE_FDATASYNC 1 +#define HAVE_FESETROUND 1 +#define HAVE_FINITE 1 +#define HAVE_FSEEKO 1 +#define HAVE_FSYNC 1 +#define HAVE_GETADDRINFO 1 +#define HAVE_GETCWD 1 +#define HAVE_GETHOSTBYADDR_R 1 +#define HAVE_GETHOSTBYNAME_R 1 +/* #undef HAVE_GETHRTIME */ +#define HAVE_GETNAMEINFO 1 +#define HAVE_GETPAGESIZE 1 +#define HAVE_GETPASS 1 +/* #undef HAVE_GETPASSPHRASE */ +#define HAVE_GETPWNAM 1 +#define HAVE_GETPWUID 1 +#define HAVE_GETRLIMIT 1 +#define HAVE_GETRUSAGE 1 +#define HAVE_GETWD 1 +#define HAVE_GMTIME_R 1 +#define HAVE_INITGROUPS 1 +#define HAVE_LDIV 1 +#define HAVE_LOCALTIME_R 1 +#define HAVE_LOG2 1 +#define HAVE_LONGJMP 1 +#define HAVE_LSTAT 1 +#define HAVE_MADVISE 1 +#define HAVE_MALLINFO 1 +#define HAVE_MEMALIGN 1 +#define HAVE_MEMCPY 1 +#define HAVE_MEMMOVE 1 +#define HAVE_MKSTEMP 1 +#define HAVE_MLOCK 1 +#define HAVE_MLOCKALL 1 +#define HAVE_MMAP 1 +#define HAVE_MMAP64 1 +#define HAVE_PERROR 1 +#define HAVE_POLL 1 +#define HAVE_PREAD 1 +/* #undef HAVE_PTHREAD_ATTR_CREATE */ +#define HAVE_PTHREAD_ATTR_GETSTACKSIZE 1 +/* #undef HAVE_PTHREAD_ATTR_SETPRIO */ +#define HAVE_PTHREAD_ATTR_SETSCHEDPARAM 1 +#define HAVE_PTHREAD_ATTR_SETSCOPE 1 +#define HAVE_PTHREAD_ATTR_SETSTACKSIZE 1 +/* #undef HAVE_PTHREAD_CONDATTR_CREATE */ +/* #undef HAVE_PTHREAD_INIT */ +#define HAVE_PTHREAD_KEY_DELETE 1 +#define HAVE_PTHREAD_KILL 1 +#define HAVE_PTHREAD_RWLOCK_RDLOCK 1 +/* #undef HAVE_PTHREAD_SETPRIO_NP */ +#define HAVE_PTHREAD_SETSCHEDPARAM 1 +#define HAVE_PTHREAD_SIGMASK 1 +/* #undef HAVE_PTHREAD_THREADMASK */ +/* #undef HAVE_PTHREAD_YIELD_NP */ +#define HAVE_READDIR_R 1 +#define HAVE_READLINK 1 +#define HAVE_REALPATH 1 +#define HAVE_RENAME 1 +#define HAVE_SCHED_YIELD 1 +#define HAVE_SELECT 1 +/* #undef HAVE_SETFD */ +/* #undef HAVE_SETFILEPOINTER */ +#define HAVE_SIGNAL 1 +#define HAVE_SIGACTION 1 +/* #undef HAVE_SIGTHREADMASK */ +#define HAVE_SIGWAIT 1 +#define HAVE_SLEEP 1 +#define HAVE_SNPRINTF 1 +/* #undef HAVE_SQLITE */ +#define HAVE_STPCPY 1 +#define HAVE_STRERROR 1 +/* #undef HAVE_STRLCPY */ +#define HAVE_STRNLEN 1 +#define HAVE_STRPBRK 1 +#define HAVE_STRSEP 1 +#define HAVE_STRSTR 1 +#define HAVE_STRTOK_R 1 +#define HAVE_STRTOL 1 +#define HAVE_STRTOLL 1 +#define HAVE_STRTOUL 1 +#define HAVE_STRTOULL 1 +/* #undef HAVE_TELL */ +/* #undef HAVE_THR_SETCONCURRENCY */ +/* #undef HAVE_THR_YIELD */ +#define HAVE_VASPRINTF 1 +#define HAVE_VSNPRINTF 1 + +/* + * types and sizes + */ +/* Types we may use */ +#define SIZEOF_CHAR 1 +#if defined(SIZEOF_CHAR) +# define HAVE_CHAR 1 +#endif + +#define SIZEOF_CHARP 8 +#if defined(SIZEOF_CHARP) +# define HAVE_CHARP 1 +#endif + +#define SIZEOF_SHORT 2 +#if defined(SIZEOF_SHORT) +# define HAVE_SHORT 1 +#endif + +#define SIZEOF_INT 4 +#if defined(SIZEOF_INT) +# define HAVE_INT 1 +#endif + +#define SIZEOF_LONG 8 +#if defined(SIZEOF_LONG) +# define HAVE_LONG 1 +#endif + +#define SIZEOF_LONG_LONG 8 +#if defined(SIZEOF_LONG_LONG) +# define HAVE_LONG_LONG 1 +#endif + + +#define SIZEOF_SIGSET_T 128 +#if defined(SIZEOF_SIGSET_T) +# define HAVE_SIGSET_T 1 +#endif + +#define SIZEOF_SIZE_T 8 +#if defined(SIZEOF_SIZE_T) +# define HAVE_SIZE_T 1 +#endif + +/* #undef SIZEOF_UCHAR */ +#if defined(SIZEOF_UCHAR) +# define HAVE_UCHAR 1 +#endif + +#define SIZEOF_UINT 4 +#if defined(SIZEOF_UINT) +# define HAVE_UINT 1 +#endif + +#define SIZEOF_ULONG 8 +#if defined(SIZEOF_ULONG) +# define HAVE_ULONG 1 +#endif + +/* #undef SIZEOF_INT8 */ +#if defined(SIZEOF_INT8) +# define HAVE_INT8 1 +#endif +/* #undef SIZEOF_UINT8 */ +#if defined(SIZEOF_UINT8) +# define HAVE_UINT8 1 +#endif + +/* #undef SIZEOF_INT16 */ +#if defined(SIZEOF_INT16) +# define HAVE_INT16 1 +#endif +/* #undef SIZEOF_UINT16 */ +#if defined(SIZEOF_UINT16) +# define HAVE_UINT16 1 +#endif + +/* #undef SIZEOF_INT32 */ +#if defined(SIZEOF_INT32) +# define HAVE_INT32 1 +#endif +/* #undef SIZEOF_UINT32 */ +#if defined(SIZEOF_UINT32) +# define HAVE_UINT32 1 +#endif +/* #undef SIZEOF_U_INT32_T */ +#if defined(SIZEOF_U_INT32_T) +# define HAVE_U_INT32_T 1 +#endif + +/* #undef SIZEOF_INT64 */ +#if defined(SIZEOF_INT64) +# define HAVE_INT64 1 +#endif +/* #undef SIZEOF_UINT64 */ +#if defined(SIZEOF_UINT64) +# define HAVE_UINT64 1 +#endif + +/* #undef SIZEOF_SOCKLEN_T */ +#if defined(SIZEOF_SOCKLEN_T) +# define HAVE_SOCKLEN_T 1 +#endif + +#define SOCKET_SIZE_TYPE socklen_t + +#define MARIADB_DEFAULT_CHARSET "latin1" + diff --git a/contrib/mariadb-connector-c-cmake/linux_x86_64/include/ma_config.h b/contrib/mariadb-connector-c-cmake/linux_x86_64/include/ma_config.h new file mode 100644 index 00000000000..90c42c97df6 --- /dev/null +++ b/contrib/mariadb-connector-c-cmake/linux_x86_64/include/ma_config.h @@ -0,0 +1,269 @@ + +/* + * Include file constants (processed in LibmysqlIncludeFiles.txt 1 + */ +#define HAVE_ALLOCA_H 1 +/* #undef HAVE_BIGENDIAN */ +#define HAVE_SETLOCALE 1 +#define HAVE_NL_LANGINFO 1 +#define HAVE_ARPA_INET_H 1 +#define HAVE_CRYPT_H 1 +#define HAVE_DIRENT_H 1 +#define HAVE_DLFCN_H 1 +#define HAVE_EXECINFO_H 1 +#define HAVE_FCNTL_H 1 +#define HAVE_FENV_H 1 +#define HAVE_FLOAT_H 1 +/* #undef HAVE_FPU_CONTROL_H */ +#define HAVE_GRP_H 1 +/* #undef HAVE_IEEEFP_H */ +#define HAVE_LIMITS_H 1 +#define HAVE_MALLOC_H 1 +#define HAVE_MEMORY_H 1 +#define HAVE_NETINET_IN_H 1 +#define HAVE_PATHS_H 1 +#define HAVE_PWD_H 1 +#define HAVE_SCHED_H 1 +/* #undef HAVE_SELECT_H */ +#define HAVE_STDDEF_H 1 +#define HAVE_STDINT_H 1 +#define HAVE_STDLIB_H 1 +#define HAVE_STRING_H 1 +#define HAVE_STRINGS_H 1 +/* #undef HAVE_SYNCH_H */ +/* #undef HAVE_SYS_FPU_H */ +#define HAVE_SYS_IOCTL_H 1 +#define HAVE_SYS_IPC_H 1 +#define HAVE_SYS_MMAN_H 1 +#define HAVE_SYS_PRCTL_H 1 +#define HAVE_SYS_SELECT_H 1 +#define HAVE_SYS_SHM_H 1 +#define HAVE_SYS_SOCKET_H 1 +#define HAVE_SYS_STAT_H 1 +/* #undef HAVE_SYS_STREAM_H */ +#define HAVE_SYS_TIMEB_H 1 +#define HAVE_SYS_TYPES_H 1 +#define HAVE_SYS_UN_H 1 +/* #undef HAVE_SYSENT_H */ +#define HAVE_TERMIO_H 1 +#define HAVE_TERMIOS_H 1 +#define HAVE_UNISTD_H 1 +#define HAVE_UTIME_H 1 +#define HAVE_UCONTEXT_H 1 + +/* + * function definitions - processed in LibmysqlFunctions.txt + */ +#define HAVE_ACCESS 1 +/* #undef HAVE_AIOWAIT */ +#define HAVE_ALARM 1 +/* #undef HAVE_ALLOCA */ +#define HAVE_BCMP 1 +/* #undef HAVE_BFILL */ +/* #undef HAVE_BMOVE */ +#define HAVE_BZERO 1 +#define HAVE_CLOCK_GETTIME 1 +/* #undef HAVE_COMPRESS */ +/* #undef HAVE_CRYPT */ +#define HAVE_DLERROR 1 +#define HAVE_DLOPEN 1 +#define HAVE_FCHMOD 1 +#define HAVE_FCNTL 1 +/* #undef HAVE_FCONVERT */ +#define HAVE_FDATASYNC 1 +#define HAVE_FESETROUND 1 +#define HAVE_FINITE 1 +#define HAVE_FSEEKO 1 +#define HAVE_FSYNC 1 +#define HAVE_GETADDRINFO 1 +#define HAVE_GETCWD 1 +#define HAVE_GETHOSTBYADDR_R 1 +#define HAVE_GETHOSTBYNAME_R 1 +/* #undef HAVE_GETHRTIME */ +#define HAVE_GETNAMEINFO 1 +#define HAVE_GETPAGESIZE 1 +#define HAVE_GETPASS 1 +/* #undef HAVE_GETPASSPHRASE */ +#define HAVE_GETPWNAM 1 +#define HAVE_GETPWUID 1 +#define HAVE_GETRLIMIT 1 +#define HAVE_GETRUSAGE 1 +#define HAVE_GETWD 1 +#define HAVE_GMTIME_R 1 +#define HAVE_INITGROUPS 1 +#define HAVE_LDIV 1 +#define HAVE_LOCALTIME_R 1 +#define HAVE_LOG2 1 +#define HAVE_LONGJMP 1 +#define HAVE_LSTAT 1 +#define HAVE_MADVISE 1 +#define HAVE_MALLINFO 1 +#define HAVE_MEMALIGN 1 +#define HAVE_MEMCPY 1 +#define HAVE_MEMMOVE 1 +#define HAVE_MKSTEMP 1 +#define HAVE_MLOCK 1 +#define HAVE_MLOCKALL 1 +#define HAVE_MMAP 1 +#define HAVE_MMAP64 1 +#define HAVE_PERROR 1 +#define HAVE_POLL 1 +#define HAVE_PREAD 1 +/* #undef HAVE_PTHREAD_ATTR_CREATE */ +#define HAVE_PTHREAD_ATTR_GETSTACKSIZE 1 +/* #undef HAVE_PTHREAD_ATTR_SETPRIO */ +#define HAVE_PTHREAD_ATTR_SETSCHEDPARAM 1 +#define HAVE_PTHREAD_ATTR_SETSCOPE 1 +#define HAVE_PTHREAD_ATTR_SETSTACKSIZE 1 +/* #undef HAVE_PTHREAD_CONDATTR_CREATE */ +/* #undef HAVE_PTHREAD_INIT */ +#define HAVE_PTHREAD_KEY_DELETE 1 +#define HAVE_PTHREAD_KILL 1 +#define HAVE_PTHREAD_RWLOCK_RDLOCK 1 +/* #undef HAVE_PTHREAD_SETPRIO_NP */ +#define HAVE_PTHREAD_SETSCHEDPARAM 1 +#define HAVE_PTHREAD_SIGMASK 1 +/* #undef HAVE_PTHREAD_THREADMASK */ +/* #undef HAVE_PTHREAD_YIELD_NP */ +#define HAVE_READDIR_R 1 +#define HAVE_READLINK 1 +#define HAVE_REALPATH 1 +#define HAVE_RENAME 1 +#define HAVE_SCHED_YIELD 1 +#define HAVE_SELECT 1 +/* #undef HAVE_SETFD */ +/* #undef HAVE_SETFILEPOINTER */ +#define HAVE_SIGNAL 1 +#define HAVE_SIGACTION 1 +/* #undef HAVE_SIGTHREADMASK */ +#define HAVE_SIGWAIT 1 +#define HAVE_SLEEP 1 +#define HAVE_SNPRINTF 1 +/* #undef HAVE_SQLITE */ +#define HAVE_STPCPY 1 +#define HAVE_STRERROR 1 +/* #undef HAVE_STRLCPY */ +#define HAVE_STRNLEN 1 +#define HAVE_STRPBRK 1 +#define HAVE_STRSEP 1 +#define HAVE_STRSTR 1 +#define HAVE_STRTOK_R 1 +#define HAVE_STRTOL 1 +#define HAVE_STRTOLL 1 +#define HAVE_STRTOUL 1 +#define HAVE_STRTOULL 1 +/* #undef HAVE_TELL */ +/* #undef HAVE_THR_SETCONCURRENCY */ +/* #undef HAVE_THR_YIELD */ +#define HAVE_VASPRINTF 1 +#define HAVE_VSNPRINTF 1 + +/* + * types and sizes + */ +/* Types we may use */ +#define SIZEOF_CHAR 1 +#if defined(SIZEOF_CHAR) +# define HAVE_CHAR 1 +#endif + +#define SIZEOF_CHARP 8 +#if defined(SIZEOF_CHARP) +# define HAVE_CHARP 1 +#endif + +#define SIZEOF_SHORT 2 +#if defined(SIZEOF_SHORT) +# define HAVE_SHORT 1 +#endif + +#define SIZEOF_INT 4 +#if defined(SIZEOF_INT) +# define HAVE_INT 1 +#endif + +#define SIZEOF_LONG 8 +#if defined(SIZEOF_LONG) +# define HAVE_LONG 1 +#endif + +#define SIZEOF_LONG_LONG 8 +#if defined(SIZEOF_LONG_LONG) +# define HAVE_LONG_LONG 1 +#endif + + +#define SIZEOF_SIGSET_T 128 +#if defined(SIZEOF_SIGSET_T) +# define HAVE_SIGSET_T 1 +#endif + +#define SIZEOF_SIZE_T 8 +#if defined(SIZEOF_SIZE_T) +# define HAVE_SIZE_T 1 +#endif + +/* #undef SIZEOF_UCHAR */ +#if defined(SIZEOF_UCHAR) +# define HAVE_UCHAR 1 +#endif + +#define SIZEOF_UINT 4 +#if defined(SIZEOF_UINT) +# define HAVE_UINT 1 +#endif + +#define SIZEOF_ULONG 8 +#if defined(SIZEOF_ULONG) +# define HAVE_ULONG 1 +#endif + +/* #undef SIZEOF_INT8 */ +#if defined(SIZEOF_INT8) +# define HAVE_INT8 1 +#endif +/* #undef SIZEOF_UINT8 */ +#if defined(SIZEOF_UINT8) +# define HAVE_UINT8 1 +#endif + +/* #undef SIZEOF_INT16 */ +#if defined(SIZEOF_INT16) +# define HAVE_INT16 1 +#endif +/* #undef SIZEOF_UINT16 */ +#if defined(SIZEOF_UINT16) +# define HAVE_UINT16 1 +#endif + +/* #undef SIZEOF_INT32 */ +#if defined(SIZEOF_INT32) +# define HAVE_INT32 1 +#endif +/* #undef SIZEOF_UINT32 */ +#if defined(SIZEOF_UINT32) +# define HAVE_UINT32 1 +#endif +/* #undef SIZEOF_U_INT32_T */ +#if defined(SIZEOF_U_INT32_T) +# define HAVE_U_INT32_T 1 +#endif + +/* #undef SIZEOF_INT64 */ +#if defined(SIZEOF_INT64) +# define HAVE_INT64 1 +#endif +/* #undef SIZEOF_UINT64 */ +#if defined(SIZEOF_UINT64) +# define HAVE_UINT64 1 +#endif + +/* #undef SIZEOF_SOCKLEN_T */ +#if defined(SIZEOF_SOCKLEN_T) +# define HAVE_SOCKLEN_T 1 +#endif + +#define SOCKET_SIZE_TYPE socklen_t + +#define MARIADB_DEFAULT_CHARSET "latin1" + diff --git a/contrib/mariadb-connector-c-cmake/linux_x86_64/include/mariadb_version.h b/contrib/mariadb-connector-c-cmake/linux_x86_64/include/mariadb_version.h new file mode 100644 index 00000000000..821a7f8add2 --- /dev/null +++ b/contrib/mariadb-connector-c-cmake/linux_x86_64/include/mariadb_version.h @@ -0,0 +1,36 @@ +/* Copyright Abandoned 1996, 1999, 2001 MySQL AB + This file is public domain and comes with NO WARRANTY of any kind */ + +/* Version numbers for protocol & mysqld */ + +#ifndef _mariadb_version_h_ +#define _mariadb_version_h_ + +#ifdef _CUSTOMCONFIG_ +#include +#else +#define PROTOCOL_VERSION 10 +#define MARIADB_CLIENT_VERSION_STR "10.3.6" +#define MARIADB_BASE_VERSION "mariadb-10.3" +#define MARIADB_VERSION_ID 100306 +#define MYSQL_VERSION_ID 100306 +#define MARIADB_PORT 3306 +#define MARIADB_UNIX_ADDR "/var/run/mysqld/mysqld.sock" +#define MYSQL_CONFIG_NAME "my" + +#define MARIADB_PACKAGE_VERSION "3.0.6" +#define MARIADB_PACKAGE_VERSION_ID 30006 +#define MARIADB_SYSTEM_TYPE "Linux" +#define MARIADB_MACHINE_TYPE "x86_64" +#define MARIADB_PLUGINDIR "lib/mariadb/plugin" + +/* mysqld compile time options */ +#ifndef MYSQL_CHARSET +#define MYSQL_CHARSET "" +#endif +#endif + +/* Source information */ +#define CC_SOURCE_REVISION "a0fd36cc5a5313414a5a2ebe9322577a29b4782a" + +#endif /* _mariadb_version_h_ */ diff --git a/contrib/mariadb-connector-c-cmake/linux_x86_64/libmariadb/ma_client_plugin.c b/contrib/mariadb-connector-c-cmake/linux_x86_64/libmariadb/ma_client_plugin.c new file mode 100644 index 00000000000..b7fdcdbcb85 --- /dev/null +++ b/contrib/mariadb-connector-c-cmake/linux_x86_64/libmariadb/ma_client_plugin.c @@ -0,0 +1,499 @@ +/* Copyright (C) 2010 - 2012 Sergei Golubchik and Monty Program Ab + 2015-2016 MariaDB Corporation AB + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public + License as published by the Free Software Foundation; either + version 2 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Library General Public License for more details. + + You should have received a copy of the GNU Library General Public + License along with this library; if not see + or write to the Free Software Foundation, Inc., + 51 Franklin St., Fifth Floor, Boston, MA 02110, USA */ + +/** + @file + + Support code for the client side (libmariadb) plugins + + Client plugins are somewhat different from server plugins, they are simpler. + + They do not need to be installed or in any way explicitly loaded on the + client, they are loaded automatically on demand. + One client plugin per shared object, soname *must* match the plugin name. + + There is no reference counting and no unloading either. +*/ + +#if _MSC_VER +/* Silence warnings about variable 'unused' being used. */ +#define FORCE_INIT_OF_VARS 1 +#endif + +#include +#include +#include +#include +#include + +#include "errmsg.h" +#include + +struct st_client_plugin_int { + struct st_client_plugin_int *next; + void *dlhandle; + struct st_mysql_client_plugin *plugin; +}; + +static my_bool initialized= 0; +static MA_MEM_ROOT mem_root; + +static uint valid_plugins[][2]= { + {MYSQL_CLIENT_AUTHENTICATION_PLUGIN, MYSQL_CLIENT_AUTHENTICATION_PLUGIN_INTERFACE_VERSION}, + {MARIADB_CLIENT_PVIO_PLUGIN, MARIADB_CLIENT_PVIO_PLUGIN_INTERFACE_VERSION}, + {MARIADB_CLIENT_TRACE_PLUGIN, MARIADB_CLIENT_TRACE_PLUGIN_INTERFACE_VERSION}, + {MARIADB_CLIENT_CONNECTION_PLUGIN, MARIADB_CLIENT_CONNECTION_PLUGIN_INTERFACE_VERSION}, + {0, 0} +}; + +/* + Loaded plugins are stored in a linked list. + The list is append-only, the elements are added to the head (like in a stack). + The elements are added under a mutex, but the list can be read and traversed + without any mutex because once an element is added to the list, it stays + there. The main purpose of a mutex is to prevent two threads from + loading the same plugin twice in parallel. +*/ + + +struct st_client_plugin_int *plugin_list[MYSQL_CLIENT_MAX_PLUGINS + MARIADB_CLIENT_MAX_PLUGINS]; +#ifdef THREAD +static pthread_mutex_t LOCK_load_client_plugin; +#endif + + extern struct st_mysql_client_plugin mysql_native_password_client_plugin; + extern struct st_mysql_client_plugin mysql_old_password_client_plugin; + extern struct st_mysql_client_plugin pvio_socket_client_plugin; + + +struct st_mysql_client_plugin *mysql_client_builtins[]= +{ + (struct st_mysql_client_plugin *)&mysql_native_password_client_plugin, + (struct st_mysql_client_plugin *)&mysql_old_password_client_plugin, + (struct st_mysql_client_plugin *)&pvio_socket_client_plugin, + + 0 +}; + + +static int is_not_initialized(MYSQL *mysql, const char *name) +{ + if (initialized) + return 0; + + my_set_error(mysql, CR_AUTH_PLUGIN_CANNOT_LOAD, + SQLSTATE_UNKNOWN, ER(CR_AUTH_PLUGIN_CANNOT_LOAD), + name, "not initialized"); + return 1; +} + +static int get_plugin_nr(uint type) +{ + uint i= 0; + for(; valid_plugins[i][1]; i++) + if (valid_plugins[i][0] == type) + return i; + return -1; +} + +static const char *check_plugin_version(struct st_mysql_client_plugin *plugin, unsigned int version) +{ + if (plugin->interface_version < version || + (plugin->interface_version >> 8) > (version >> 8)) + return "Incompatible client plugin interface"; + return 0; +} + +/** + finds a plugin in the list + + @param name plugin name to search for + @param type plugin type + + @note this does NOT necessarily need a mutex, take care! + + @retval a pointer to a found plugin or 0 +*/ +static struct st_mysql_client_plugin *find_plugin(const char *name, int type) +{ + struct st_client_plugin_int *p; + int plugin_nr= get_plugin_nr(type); + + DBUG_ASSERT(initialized); + if (plugin_nr == -1) + return 0; + + if (!name) + return plugin_list[plugin_nr]->plugin; + + for (p= plugin_list[plugin_nr]; p; p= p->next) + { + if (strcmp(p->plugin->name, name) == 0) + return p->plugin; + } + return NULL; +} + + +/** + verifies the plugin and adds it to the list + + @param mysql MYSQL structure (for error reporting) + @param plugin plugin to install + @param dlhandle a handle to the shared object (returned by dlopen) + or 0 if the plugin was not dynamically loaded + @param argc number of arguments in the 'va_list args' + @param args arguments passed to the plugin initialization function + + @retval a pointer to an installed plugin or 0 +*/ + +static struct st_mysql_client_plugin * +add_plugin(MYSQL *mysql, struct st_mysql_client_plugin *plugin, void *dlhandle, + int argc, va_list args) +{ + const char *errmsg; + struct st_client_plugin_int plugin_int, *p; + char errbuf[1024]; + int plugin_nr; + + DBUG_ASSERT(initialized); + + plugin_int.plugin= plugin; + plugin_int.dlhandle= dlhandle; + + if ((plugin_nr= get_plugin_nr(plugin->type)) == -1) + { + errmsg= "Unknown client plugin type"; + goto err1; + } + if ((errmsg= check_plugin_version(plugin, valid_plugins[plugin_nr][1]))) + goto err1; + + /* Call the plugin initialization function, if any */ + if (plugin->init && plugin->init(errbuf, sizeof(errbuf), argc, args)) + { + errmsg= errbuf; + goto err1; + } + + p= (struct st_client_plugin_int *) + ma_memdup_root(&mem_root, (char *)&plugin_int, sizeof(plugin_int)); + + if (!p) + { + errmsg= "Out of memory"; + goto err2; + } + +#ifdef THREAD + safe_mutex_assert_owner(&LOCK_load_client_plugin); +#endif + + p->next= plugin_list[plugin_nr]; + plugin_list[plugin_nr]= p; + + return plugin; + +err2: + if (plugin->deinit) + plugin->deinit(); +err1: + my_set_error(mysql, CR_AUTH_PLUGIN_CANNOT_LOAD, SQLSTATE_UNKNOWN, + ER(CR_AUTH_PLUGIN_CANNOT_LOAD), plugin->name, errmsg); + if (dlhandle) + (void)dlclose(dlhandle); + return NULL; +} + + +/** + Loads plugins which are specified in the environment variable + LIBMYSQL_PLUGINS. + + Multiple plugins must be separated by semicolon. This function doesn't + return or log an error. + + The function is be called by mysql_client_plugin_init + + @todo + Support extended syntax, passing parameters to plugins, for example + LIBMYSQL_PLUGINS="plugin1(param1,param2);plugin2;..." + or + LIBMYSQL_PLUGINS="plugin1=int:param1,str:param2;plugin2;..." +*/ + +static void load_env_plugins(MYSQL *mysql) +{ + char *plugs, *free_env, *s= getenv("LIBMYSQL_PLUGINS"); + + if (ma_check_env_str(s)) + return; + + free_env= strdup(s); + plugs= s= free_env; + + do { + if ((s= strchr(plugs, ';'))) + *s= '\0'; + mysql_load_plugin(mysql, plugs, -1, 0); + plugs= s + 1; + } while (s); + + free(free_env); +} + +/********** extern functions to be used by libmariadb *********************/ + +/** + Initializes the client plugin layer. + + This function must be called before any other client plugin function. + + @retval 0 successful + @retval != 0 error occurred +*/ + +int mysql_client_plugin_init() +{ + MYSQL mysql; + struct st_mysql_client_plugin **builtin; + va_list unused; + LINT_INIT_STRUCT(unused); + + if (initialized) + return 0; + + memset(&mysql, 0, sizeof(mysql)); /* dummy mysql for set_mysql_extended_error */ + + pthread_mutex_init(&LOCK_load_client_plugin, MY_MUTEX_INIT_SLOW); + ma_init_alloc_root(&mem_root, 128, 128); + + memset(&plugin_list, 0, sizeof(plugin_list)); + + initialized= 1; + + pthread_mutex_lock(&LOCK_load_client_plugin); + for (builtin= mysql_client_builtins; *builtin; builtin++) + add_plugin(&mysql, *builtin, 0, 0, unused); + + pthread_mutex_unlock(&LOCK_load_client_plugin); + + load_env_plugins(&mysql); + + return 0; +} + + +/** + Deinitializes the client plugin layer. + + Unloades all client plugins and frees any associated resources. +*/ + +void mysql_client_plugin_deinit() +{ + int i; + struct st_client_plugin_int *p; + + if (!initialized) + return; + + for (i=0; i < MYSQL_CLIENT_MAX_PLUGINS; i++) + for (p= plugin_list[i]; p; p= p->next) + { + if (p->plugin->deinit) + p->plugin->deinit(); + if (p->dlhandle) + (void)dlclose(p->dlhandle); + } + + memset(&plugin_list, 0, sizeof(plugin_list)); + initialized= 0; + ma_free_root(&mem_root, MYF(0)); + pthread_mutex_destroy(&LOCK_load_client_plugin); +} + +/************* public facing functions, for client consumption *********/ + +/* see for a full description */ +struct st_mysql_client_plugin * STDCALL +mysql_client_register_plugin(MYSQL *mysql, + struct st_mysql_client_plugin *plugin) +{ + va_list unused; + LINT_INIT_STRUCT(unused); + + if (is_not_initialized(mysql, plugin->name)) + return NULL; + + pthread_mutex_lock(&LOCK_load_client_plugin); + + /* make sure the plugin wasn't loaded meanwhile */ + if (find_plugin(plugin->name, plugin->type)) + { + my_set_error(mysql, CR_AUTH_PLUGIN_CANNOT_LOAD, + SQLSTATE_UNKNOWN, ER(CR_AUTH_PLUGIN_CANNOT_LOAD), + plugin->name, "it is already loaded"); + plugin= NULL; + } + else + plugin= add_plugin(mysql, plugin, 0, 0, unused); + + pthread_mutex_unlock(&LOCK_load_client_plugin); + return plugin; +} + + +/* see for a full description */ +struct st_mysql_client_plugin * STDCALL +mysql_load_plugin_v(MYSQL *mysql, const char *name, int type, + int argc, va_list args) +{ + const char *errmsg; +#ifdef _WIN32 + char errbuf[1024]; +#endif + char dlpath[FN_REFLEN+1]; + void *sym, *dlhandle = NULL; + struct st_mysql_client_plugin *plugin; + char *env_plugin_dir= getenv("MARIADB_PLUGIN_DIR"); + + CLEAR_CLIENT_ERROR(mysql); + if (is_not_initialized(mysql, name)) + return NULL; + + pthread_mutex_lock(&LOCK_load_client_plugin); + + /* make sure the plugin wasn't loaded meanwhile */ + if (type >= 0 && find_plugin(name, type)) + { + errmsg= "it is already loaded"; + goto err; + } + + /* Compile dll path */ + snprintf(dlpath, sizeof(dlpath) - 1, "%s/%s%s", + mysql->options.extension && mysql->options.extension->plugin_dir ? + mysql->options.extension->plugin_dir : (env_plugin_dir) ? env_plugin_dir : + MARIADB_PLUGINDIR, name, SO_EXT); + + /* Open new dll handle */ + if (!(dlhandle= dlopen((const char *)dlpath, RTLD_NOW))) + { +#ifdef _WIN32 + char winmsg[255]; + size_t len; + winmsg[0] = 0; + FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, + NULL, + GetLastError(), + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + winmsg, 255, NULL); + len= strlen(winmsg); + while (len > 0 && (winmsg[len - 1] == '\n' || winmsg[len - 1] == '\r')) + len--; + if (len) + winmsg[len] = 0; + snprintf(errbuf, sizeof(errbuf), "%s Library path is '%s'", winmsg, dlpath); + errmsg= errbuf; +#else + errmsg= dlerror(); +#endif + goto err; + } + + + if (!(sym= dlsym(dlhandle, plugin_declarations_sym))) + { + errmsg= "not a plugin"; + (void)dlclose(dlhandle); + goto err; + } + + plugin= (struct st_mysql_client_plugin*)sym; + + if (type >=0 && type != plugin->type) + { + errmsg= "type mismatch"; + goto err; + } + + if (strcmp(name, plugin->name)) + { + errmsg= "name mismatch"; + goto err; + } + + if (type < 0 && find_plugin(name, plugin->type)) + { + errmsg= "it is already loaded"; + goto err; + } + + plugin= add_plugin(mysql, plugin, dlhandle, argc, args); + + pthread_mutex_unlock(&LOCK_load_client_plugin); + + return plugin; + +err: + if (dlhandle) + dlclose(dlhandle); + pthread_mutex_unlock(&LOCK_load_client_plugin); + my_set_error(mysql, CR_AUTH_PLUGIN_CANNOT_LOAD, SQLSTATE_UNKNOWN, + ER(CR_AUTH_PLUGIN_CANNOT_LOAD), name, errmsg); + return NULL; +} + + +/* see for a full description */ +struct st_mysql_client_plugin * STDCALL +mysql_load_plugin(MYSQL *mysql, const char *name, int type, int argc, ...) +{ + struct st_mysql_client_plugin *p; + va_list args; + va_start(args, argc); + p= mysql_load_plugin_v(mysql, name, type, argc, args); + va_end(args); + return p; +} + +/* see for a full description */ +struct st_mysql_client_plugin * STDCALL +mysql_client_find_plugin(MYSQL *mysql, const char *name, int type) +{ + struct st_mysql_client_plugin *p; + int plugin_nr= get_plugin_nr(type); + + if (is_not_initialized(mysql, name)) + return NULL; + + if (plugin_nr == -1) + { + my_set_error(mysql, CR_AUTH_PLUGIN_CANNOT_LOAD, SQLSTATE_UNKNOWN, + ER(CR_AUTH_PLUGIN_CANNOT_LOAD), name, "invalid type"); + } + + if ((p= find_plugin(name, type))) + return p; + + /* not found, load it */ + return mysql_load_plugin(mysql, name, type, 0); +} + diff --git a/contrib/murmurhash/CMakeLists.txt b/contrib/murmurhash/CMakeLists.txt new file mode 100644 index 00000000000..c5e467a2d6d --- /dev/null +++ b/contrib/murmurhash/CMakeLists.txt @@ -0,0 +1,7 @@ +add_library(murmurhash + src/murmurhash2.cpp + src/murmurhash3.cpp + include/murmurhash2.h + include/murmurhash3.h) + +target_include_directories (murmurhash PUBLIC include) diff --git a/contrib/murmurhash/LICENSE b/contrib/murmurhash/LICENSE new file mode 100644 index 00000000000..f6cdede60b8 --- /dev/null +++ b/contrib/murmurhash/LICENSE @@ -0,0 +1 @@ +MurmurHash was written by Austin Appleby, and is placed in the publicdomain. The author hereby disclaims copyright to this source code. diff --git a/contrib/murmurhash/README b/contrib/murmurhash/README new file mode 100644 index 00000000000..5428d30b26d --- /dev/null +++ b/contrib/murmurhash/README @@ -0,0 +1,6 @@ +Original URL: https://github.com/aappleby/smhasher + +version: +commit 61a0530f28277f2e850bfc39600ce61d02b518de +author aappleby@gmail.com +date 2016-01-09T06:07:17Z diff --git a/contrib/murmurhash/include/murmurhash2.h b/contrib/murmurhash/include/murmurhash2.h new file mode 100644 index 00000000000..e95cf2a4d85 --- /dev/null +++ b/contrib/murmurhash/include/murmurhash2.h @@ -0,0 +1,35 @@ +//----------------------------------------------------------------------------- +// MurmurHash2 was written by Austin Appleby, and is placed in the public +// domain. The author hereby disclaims copyright to this source code. + +#ifndef _MURMURHASH2_H_ +#define _MURMURHASH2_H_ + +//----------------------------------------------------------------------------- +// Platform-specific functions and macros + +// Microsoft Visual Studio + +#if defined(_MSC_VER) && (_MSC_VER < 1600) + +typedef unsigned char uint8_t; +typedef unsigned int uint32_t; +typedef unsigned __int64 uint64_t; + +// Other compilers + +#else // defined(_MSC_VER) + +#include + +#endif // !defined(_MSC_VER) + +uint32_t MurmurHash2 (const void * key, int len, uint32_t seed); +uint64_t MurmurHash64A (const void * key, int len, uint64_t seed); +uint64_t MurmurHash64B (const void * key, int len, uint64_t seed); +uint32_t MurmurHash2A (const void * key, int len, uint32_t seed); +uint32_t MurmurHashNeutral2 (const void * key, int len, uint32_t seed); +uint32_t MurmurHashAligned2 (const void * key, int len, uint32_t seed); + +#endif // _MURMURHASH2_H_ + diff --git a/contrib/murmurhash/include/murmurhash3.h b/contrib/murmurhash/include/murmurhash3.h new file mode 100644 index 00000000000..e1c6d34976c --- /dev/null +++ b/contrib/murmurhash/include/murmurhash3.h @@ -0,0 +1,37 @@ +//----------------------------------------------------------------------------- +// MurmurHash3 was written by Austin Appleby, and is placed in the public +// domain. The author hereby disclaims copyright to this source code. + +#ifndef _MURMURHASH3_H_ +#define _MURMURHASH3_H_ + +//----------------------------------------------------------------------------- +// Platform-specific functions and macros + +// Microsoft Visual Studio + +#if defined(_MSC_VER) && (_MSC_VER < 1600) + +typedef unsigned char uint8_t; +typedef unsigned int uint32_t; +typedef unsigned __int64 uint64_t; + +// Other compilers + +#else // defined(_MSC_VER) + +#include + +#endif // !defined(_MSC_VER) + +//----------------------------------------------------------------------------- + +void MurmurHash3_x86_32 ( const void * key, int len, uint32_t seed, void * out ); + +void MurmurHash3_x86_128 ( const void * key, int len, uint32_t seed, void * out ); + +void MurmurHash3_x64_128 ( const void * key, int len, uint32_t seed, void * out ); + +//----------------------------------------------------------------------------- + +#endif // _MURMURHASH3_H_ diff --git a/contrib/murmurhash/src/murmurhash2.cpp b/contrib/murmurhash/src/murmurhash2.cpp new file mode 100644 index 00000000000..8a41ba02d98 --- /dev/null +++ b/contrib/murmurhash/src/murmurhash2.cpp @@ -0,0 +1,421 @@ +// MurmurHash2 was written by Austin Appleby, and is placed in the public +// domain. The author hereby disclaims copyright to this source code. + +// Note - This code makes a few assumptions about how your machine behaves - + +// 1. We can read a 4-byte value from any address without crashing +// 2. sizeof(int) == 4 + +// And it has a few limitations - + +// 1. It will not work incrementally. +// 2. It will not produce the same results on little-endian and big-endian +// machines. + +#include "murmurhash2.h" + +// Platform-specific functions and macros +// Microsoft Visual Studio + +#if defined(_MSC_VER) + +#define BIG_CONSTANT(x) (x) + +// Other compilers + +#else // defined(_MSC_VER) + +#define BIG_CONSTANT(x) (x##LLU) + +#endif // !defined(_MSC_VER) + + +uint32_t MurmurHash2(const void * key, int len, uint32_t seed) +{ + // 'm' and 'r' are mixing constants generated offline. + // They're not really 'magic', they just happen to work well. + + const uint32_t m = 0x5bd1e995; + const int r = 24; + + // Initialize the hash to a 'random' value + + uint32_t h = seed ^ len; + + // Mix 4 bytes at a time into the hash + + const unsigned char * data = reinterpret_cast(key); + + while (len >= 4) + { + uint32_t k = *reinterpret_cast(data); + k *= m; + k ^= k >> r; + k *= m; + + h *= m; + h ^= k; + + data += 4; + len -= 4; + } + + // Handle the last few bytes of the input array + + switch (len) + { + case 3: h ^= data[2] << 16; + case 2: h ^= data[1] << 8; + case 1: h ^= data[0]; + h *= m; + }; + + // Do a few final mixes of the hash to ensure the last few + // bytes are well-incorporated. + + h ^= h >> 13; + h *= m; + h ^= h >> 15; + + return h; +} + +// MurmurHash2, 64-bit versions, by Austin Appleby + +// The same caveats as 32-bit MurmurHash2 apply here - beware of alignment +// and endian-ness issues if used across multiple platforms. + +// 64-bit hash for 64-bit platforms + +uint64_t MurmurHash64A(const void * key, int len, uint64_t seed) +{ + const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995); + const int r = 47; + + uint64_t h = seed ^ (len * m); + + const uint64_t * data = reinterpret_cast(key); + const uint64_t * end = data + (len/8); + + while (data != end) + { + uint64_t k = *data++; + + k *= m; + k ^= k >> r; + k *= m; + + h ^= k; + h *= m; + } + + const unsigned char * data2 = reinterpret_cast(data); + + switch (len & 7) + { + case 7: h ^= static_cast(data2[6]) << 48; + case 6: h ^= static_cast(data2[5]) << 40; + case 5: h ^= static_cast(data2[4]) << 32; + case 4: h ^= static_cast(data2[3]) << 24; + case 3: h ^= static_cast(data2[2]) << 16; + case 2: h ^= static_cast(data2[1]) << 8; + case 1: h ^= static_cast(data2[0]); + h *= m; + }; + + h ^= h >> r; + h *= m; + h ^= h >> r; + + return h; +} + + +// 64-bit hash for 32-bit platforms + +uint64_t MurmurHash64B(const void * key, int len, uint64_t seed) +{ + const uint32_t m = 0x5bd1e995; + const int r = 24; + + uint32_t h1 = static_cast(seed) ^ len; + uint32_t h2 = static_cast(seed >> 32); + + const uint32_t * data = reinterpret_cast(key); + + while (len >= 8) + { + uint32_t k1 = *data++; + k1 *= m; k1 ^= k1 >> r; k1 *= m; + h1 *= m; h1 ^= k1; + len -= 4; + + uint32_t k2 = *data++; + k2 *= m; k2 ^= k2 >> r; k2 *= m; + h2 *= m; h2 ^= k2; + len -= 4; + } + + if (len >= 4) + { + uint32_t k1 = *data++; + k1 *= m; k1 ^= k1 >> r; k1 *= m; + h1 *= m; h1 ^= k1; + len -= 4; + } + + switch (len) + { + case 3: h2 ^= reinterpret_cast(data)[2] << 16; + case 2: h2 ^= reinterpret_cast(data)[1] << 8; + case 1: h2 ^= reinterpret_cast(data)[0]; + h2 *= m; + }; + + h1 ^= h2 >> 18; h1 *= m; + h2 ^= h1 >> 22; h2 *= m; + h1 ^= h2 >> 17; h1 *= m; + h2 ^= h1 >> 19; h2 *= m; + + uint64_t h = h1; + + h = (h << 32) | h2; + + return h; +} + +// MurmurHash2A, by Austin Appleby + +// This is a variant of MurmurHash2 modified to use the Merkle-Damgard +// construction. Bulk speed should be identical to Murmur2, small-key speed +// will be 10%-20% slower due to the added overhead at the end of the hash. + +// This variant fixes a minor issue where null keys were more likely to +// collide with each other than expected, and also makes the function +// more amenable to incremental implementations. + +#define mmix(h,k) { k *= m; k ^= k >> r; k *= m; h *= m; h ^= k; } + +uint32_t MurmurHash2A(const void * key, int len, uint32_t seed) +{ + const uint32_t m = 0x5bd1e995; + const int r = 24; + uint32_t l = len; + + const unsigned char * data = reinterpret_cast(key); + + uint32_t h = seed; + + while (len >= 4) + { + uint32_t k = *reinterpret_cast(data); + mmix(h,k); + data += 4; + len -= 4; + } + + uint32_t t = 0; + + switch (len) + { + case 3: t ^= data[2] << 16; + case 2: t ^= data[1] << 8; + case 1: t ^= data[0]; + }; + + mmix(h,t); + mmix(h,l); + + h ^= h >> 13; + h *= m; + h ^= h >> 15; + + return h; +} + +// MurmurHashNeutral2, by Austin Appleby + +// Same as MurmurHash2, but endian- and alignment-neutral. +// Half the speed though, alas. + +uint32_t MurmurHashNeutral2(const void * key, int len, uint32_t seed) +{ + const uint32_t m = 0x5bd1e995; + const int r = 24; + + uint32_t h = seed ^ len; + + const unsigned char * data = reinterpret_cast(key); + + while (len >= 4) + { + uint32_t k; + + k = data[0]; + k |= data[1] << 8; + k |= data[2] << 16; + k |= data[3] << 24; + + k *= m; + k ^= k >> r; + k *= m; + + h *= m; + h ^= k; + + data += 4; + len -= 4; + } + + switch (len) + { + case 3: h ^= data[2] << 16; + case 2: h ^= data[1] << 8; + case 1: h ^= data[0]; + h *= m; + }; + + h ^= h >> 13; + h *= m; + h ^= h >> 15; + + return h; +} + +//----------------------------------------------------------------------------- +// MurmurHashAligned2, by Austin Appleby + +// Same algorithm as MurmurHash2, but only does aligned reads - should be safer +// on certain platforms. + +// Performance will be lower than MurmurHash2 + +#define MIX(h,k,m) { k *= m; k ^= k >> r; k *= m; h *= m; h ^= k; } + + +uint32_t MurmurHashAligned2(const void * key, int len, uint32_t seed) +{ + const uint32_t m = 0x5bd1e995; + const int r = 24; + + const unsigned char * data = reinterpret_cast(key); + + uint32_t h = seed ^ len; + + int align = reinterpret_cast(data) & 3; + + if (align && (len >= 4)) + { + // Pre-load the temp registers + + uint32_t t = 0, d = 0; + + switch (align) + { + case 1: t |= data[2] << 16; + case 2: t |= data[1] << 8; + case 3: t |= data[0]; + } + + t <<= (8 * align); + + data += 4-align; + len -= 4-align; + + int sl = 8 * (4-align); + int sr = 8 * align; + + // Mix + + while (len >= 4) + { + d = *(reinterpret_cast(data)); + t = (t >> sr) | (d << sl); + + uint32_t k = t; + + MIX(h,k,m); + + t = d; + + data += 4; + len -= 4; + } + + // Handle leftover data in temp registers + + d = 0; + + if (len >= align) + { + switch (align) + { + case 3: d |= data[2] << 16; + case 2: d |= data[1] << 8; + case 1: d |= data[0]; + } + + uint32_t k = (t >> sr) | (d << sl); + MIX(h,k,m); + + data += align; + len -= align; + + //---------- + // Handle tail bytes + + switch (len) + { + case 3: h ^= data[2] << 16; + case 2: h ^= data[1] << 8; + case 1: h ^= data[0]; + h *= m; + }; + } + else + { + switch (len) + { + case 3: d |= data[2] << 16; + case 2: d |= data[1] << 8; + case 1: d |= data[0]; + case 0: h ^= (t >> sr) | (d << sl); + h *= m; + } + } + + h ^= h >> 13; + h *= m; + h ^= h >> 15; + + return h; + } + else + { + while (len >= 4) + { + uint32_t k = *reinterpret_cast(data); + + MIX(h,k,m); + + data += 4; + len -= 4; + } + + // Handle tail bytes + + switch (len) + { + case 3: h ^= data[2] << 16; + case 2: h ^= data[1] << 8; + case 1: h ^= data[0]; + h *= m; + }; + + h ^= h >> 13; + h *= m; + h ^= h >> 15; + + return h; + } +} \ No newline at end of file diff --git a/contrib/murmurhash/src/murmurhash3.cpp b/contrib/murmurhash/src/murmurhash3.cpp new file mode 100644 index 00000000000..2831bf5c73b --- /dev/null +++ b/contrib/murmurhash/src/murmurhash3.cpp @@ -0,0 +1,331 @@ +// MurmurHash3 was written by Austin Appleby, and is placed in the public +// domain. The author hereby disclaims copyright to this source code. + +// Note - The x86 and x64 versions do _not_ produce the same results, as the +// algorithms are optimized for their respective platforms. You can still +// compile and run any of them on any platform, but your performance with the +// non-native version will be less than optimal. + +#include "murmurhash3.h" + +//----------------------------------------------------------------------------- +// Platform-specific functions and macros + +// Microsoft Visual Studio + +#if defined(_MSC_VER) + +#define FORCE_INLINE __forceinline + +#include + +#define ROTL32(x,y) _rotl(x,y) +#define ROTL64(x,y) _rotl64(x,y) + +#define BIG_CONSTANT(x) (x) + +// Other compilers + +#else // defined(_MSC_VER) + +#define FORCE_INLINE inline __attribute__((always_inline)) + +inline uint32_t rotl32 ( uint32_t x, int8_t r ) +{ + return (x << r) | (x >> (32 - r)); +} + +inline uint64_t rotl64 ( uint64_t x, int8_t r ) +{ + return (x << r) | (x >> (64 - r)); +} + +#define ROTL32(x,y) rotl32(x,y) +#define ROTL64(x,y) rotl64(x,y) + +#define BIG_CONSTANT(x) (x##LLU) + +#endif // !defined(_MSC_VER) + +//----------------------------------------------------------------------------- +// Block read - if your platform needs to do endian-swapping or can only +// handle aligned reads, do the conversion here + +FORCE_INLINE uint32_t getblock32 ( const uint32_t * p, int i ) +{ + return p[i]; +} + +FORCE_INLINE uint64_t getblock64 ( const uint64_t * p, int i ) +{ + return p[i]; +} + +//----------------------------------------------------------------------------- +// Finalization mix - force all bits of a hash block to avalanche + +FORCE_INLINE uint32_t fmix32 ( uint32_t h ) +{ + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + + return h; +} + +//---------- + +FORCE_INLINE uint64_t fmix64 ( uint64_t k ) +{ + k ^= k >> 33; + k *= BIG_CONSTANT(0xff51afd7ed558ccd); + k ^= k >> 33; + k *= BIG_CONSTANT(0xc4ceb9fe1a85ec53); + k ^= k >> 33; + + return k; +} + +//----------------------------------------------------------------------------- + +void MurmurHash3_x86_32 ( const void * key, int len, + uint32_t seed, void * out ) +{ + const uint8_t * data = (const uint8_t*)key; + const int nblocks = len / 4; + + uint32_t h1 = seed; + + const uint32_t c1 = 0xcc9e2d51; + const uint32_t c2 = 0x1b873593; + + //---------- + // body + + const uint32_t * blocks = (const uint32_t *)(data + nblocks*4); + + for(int i = -nblocks; i; i++) + { + uint32_t k1 = getblock32(blocks,i); + + k1 *= c1; + k1 = ROTL32(k1,15); + k1 *= c2; + + h1 ^= k1; + h1 = ROTL32(h1,13); + h1 = h1*5+0xe6546b64; + } + + //---------- + // tail + + const uint8_t * tail = (const uint8_t*)(data + nblocks*4); + + uint32_t k1 = 0; + + switch(len & 3) + { + case 3: k1 ^= tail[2] << 16; + case 2: k1 ^= tail[1] << 8; + case 1: k1 ^= tail[0]; + k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; + }; + + //---------- + // finalization + + h1 ^= len; + + h1 = fmix32(h1); + + *(uint32_t*)out = h1; +} + +//----------------------------------------------------------------------------- + +void MurmurHash3_x86_128 ( const void * key, const int len, + uint32_t seed, void * out ) +{ + const uint8_t * data = (const uint8_t*)key; + const int nblocks = len / 16; + + uint32_t h1 = seed; + uint32_t h2 = seed; + uint32_t h3 = seed; + uint32_t h4 = seed; + + const uint32_t c1 = 0x239b961b; + const uint32_t c2 = 0xab0e9789; + const uint32_t c3 = 0x38b34ae5; + const uint32_t c4 = 0xa1e38b93; + + //---------- + // body + + const uint32_t * blocks = (const uint32_t *)(data + nblocks*16); + + for(int i = -nblocks; i; i++) + { + uint32_t k1 = getblock32(blocks,i*4+0); + uint32_t k2 = getblock32(blocks,i*4+1); + uint32_t k3 = getblock32(blocks,i*4+2); + uint32_t k4 = getblock32(blocks,i*4+3); + + k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; + + h1 = ROTL32(h1,19); h1 += h2; h1 = h1*5+0x561ccd1b; + + k2 *= c2; k2 = ROTL32(k2,16); k2 *= c3; h2 ^= k2; + + h2 = ROTL32(h2,17); h2 += h3; h2 = h2*5+0x0bcaa747; + + k3 *= c3; k3 = ROTL32(k3,17); k3 *= c4; h3 ^= k3; + + h3 = ROTL32(h3,15); h3 += h4; h3 = h3*5+0x96cd1c35; + + k4 *= c4; k4 = ROTL32(k4,18); k4 *= c1; h4 ^= k4; + + h4 = ROTL32(h4,13); h4 += h1; h4 = h4*5+0x32ac3b17; + } + + //---------- + // tail + + const uint8_t * tail = (const uint8_t*)(data + nblocks*16); + + uint32_t k1 = 0; + uint32_t k2 = 0; + uint32_t k3 = 0; + uint32_t k4 = 0; + + switch(len & 15) + { + case 15: k4 ^= tail[14] << 16; + case 14: k4 ^= tail[13] << 8; + case 13: k4 ^= tail[12] << 0; + k4 *= c4; k4 = ROTL32(k4,18); k4 *= c1; h4 ^= k4; + + case 12: k3 ^= tail[11] << 24; + case 11: k3 ^= tail[10] << 16; + case 10: k3 ^= tail[ 9] << 8; + case 9: k3 ^= tail[ 8] << 0; + k3 *= c3; k3 = ROTL32(k3,17); k3 *= c4; h3 ^= k3; + + case 8: k2 ^= tail[ 7] << 24; + case 7: k2 ^= tail[ 6] << 16; + case 6: k2 ^= tail[ 5] << 8; + case 5: k2 ^= tail[ 4] << 0; + k2 *= c2; k2 = ROTL32(k2,16); k2 *= c3; h2 ^= k2; + + case 4: k1 ^= tail[ 3] << 24; + case 3: k1 ^= tail[ 2] << 16; + case 2: k1 ^= tail[ 1] << 8; + case 1: k1 ^= tail[ 0] << 0; + k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; + }; + + //---------- + // finalization + + h1 ^= len; h2 ^= len; h3 ^= len; h4 ^= len; + + h1 += h2; h1 += h3; h1 += h4; + h2 += h1; h3 += h1; h4 += h1; + + h1 = fmix32(h1); + h2 = fmix32(h2); + h3 = fmix32(h3); + h4 = fmix32(h4); + + h1 += h2; h1 += h3; h1 += h4; + h2 += h1; h3 += h1; h4 += h1; + + ((uint32_t*)out)[0] = h1; + ((uint32_t*)out)[1] = h2; + ((uint32_t*)out)[2] = h3; + ((uint32_t*)out)[3] = h4; +} + +//----------------------------------------------------------------------------- + +void MurmurHash3_x64_128 ( const void * key, const int len, + const uint32_t seed, void * out ) +{ + const uint8_t * data = (const uint8_t*)key; + const int nblocks = len / 16; + + uint64_t h1 = seed; + uint64_t h2 = seed; + + const uint64_t c1 = BIG_CONSTANT(0x87c37b91114253d5); + const uint64_t c2 = BIG_CONSTANT(0x4cf5ad432745937f); + + //---------- + // body + + const uint64_t * blocks = (const uint64_t *)(data); + + for(int i = 0; i < nblocks; i++) + { + uint64_t k1 = getblock64(blocks,i*2+0); + uint64_t k2 = getblock64(blocks,i*2+1); + + k1 *= c1; k1 = ROTL64(k1,31); k1 *= c2; h1 ^= k1; + + h1 = ROTL64(h1,27); h1 += h2; h1 = h1*5+0x52dce729; + + k2 *= c2; k2 = ROTL64(k2,33); k2 *= c1; h2 ^= k2; + + h2 = ROTL64(h2,31); h2 += h1; h2 = h2*5+0x38495ab5; + } + + //---------- + // tail + + const uint8_t * tail = (const uint8_t*)(data + nblocks*16); + + uint64_t k1 = 0; + uint64_t k2 = 0; + + switch(len & 15) + { + case 15: k2 ^= ((uint64_t)tail[14]) << 48; + case 14: k2 ^= ((uint64_t)tail[13]) << 40; + case 13: k2 ^= ((uint64_t)tail[12]) << 32; + case 12: k2 ^= ((uint64_t)tail[11]) << 24; + case 11: k2 ^= ((uint64_t)tail[10]) << 16; + case 10: k2 ^= ((uint64_t)tail[ 9]) << 8; + case 9: k2 ^= ((uint64_t)tail[ 8]) << 0; + k2 *= c2; k2 = ROTL64(k2,33); k2 *= c1; h2 ^= k2; + + case 8: k1 ^= ((uint64_t)tail[ 7]) << 56; + case 7: k1 ^= ((uint64_t)tail[ 6]) << 48; + case 6: k1 ^= ((uint64_t)tail[ 5]) << 40; + case 5: k1 ^= ((uint64_t)tail[ 4]) << 32; + case 4: k1 ^= ((uint64_t)tail[ 3]) << 24; + case 3: k1 ^= ((uint64_t)tail[ 2]) << 16; + case 2: k1 ^= ((uint64_t)tail[ 1]) << 8; + case 1: k1 ^= ((uint64_t)tail[ 0]) << 0; + k1 *= c1; k1 = ROTL64(k1,31); k1 *= c2; h1 ^= k1; + }; + + //---------- + // finalization + + h1 ^= len; h2 ^= len; + + h1 += h2; + h2 += h1; + + h1 = fmix64(h1); + h2 = fmix64(h2); + + h1 += h2; + h2 += h1; + + ((uint64_t*)out)[0] = h1; + ((uint64_t*)out)[1] = h2; +} diff --git a/contrib/poco b/contrib/poco index 8238852d7ab..20c1d877773 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit 8238852d7ab2a4abdf87adff233b3b83686f4fe4 +Subproject commit 20c1d877773b6a672f1bbfe3290dfea42a117ed5 diff --git a/contrib/re2_st/CMakeLists.txt b/contrib/re2_st/CMakeLists.txt index cd0f97e08f3..6bc7fd8f343 100644 --- a/contrib/re2_st/CMakeLists.txt +++ b/contrib/re2_st/CMakeLists.txt @@ -12,26 +12,27 @@ endforeach () add_library (re2_st ${RE2_ST_SOURCES}) target_compile_definitions (re2_st PRIVATE NDEBUG NO_THREADS re2=re2_st) -target_include_directories (re2_st PRIVATE . PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${RE2_SOURCE_DIR}) +target_include_directories (re2_st PRIVATE .) +target_include_directories (re2_st SYSTEM PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${RE2_SOURCE_DIR}) file (MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/re2_st) foreach (FILENAME filtered_re2.h re2.h set.h stringpiece.h) - add_custom_command (OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/re2_st/${FILENAME}" - COMMAND ${CMAKE_COMMAND} -DSOURCE_FILENAME="${RE2_SOURCE_DIR}/re2/${FILENAME}" - -DTARGET_FILENAME="${CMAKE_CURRENT_BINARY_DIR}/re2_st/${FILENAME}" - -P "${CMAKE_CURRENT_SOURCE_DIR}/re2_transform.cmake" - COMMENT "Creating ${FILENAME} for re2_st library.") - add_custom_target (transform_${FILENAME} DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/re2_st/${FILENAME}") - add_dependencies (re2_st transform_${FILENAME}) + add_custom_command (OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/re2_st/${FILENAME}" + COMMAND ${CMAKE_COMMAND} -DSOURCE_FILENAME="${RE2_SOURCE_DIR}/re2/${FILENAME}" + -DTARGET_FILENAME="${CMAKE_CURRENT_BINARY_DIR}/re2_st/${FILENAME}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/re2_transform.cmake" + COMMENT "Creating ${FILENAME} for re2_st library.") + add_custom_target (transform_${FILENAME} DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/re2_st/${FILENAME}") + add_dependencies (re2_st transform_${FILENAME}) endforeach () file (MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/util) foreach (FILENAME mutex.h) - add_custom_command (OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/util/${FILENAME}" - COMMAND ${CMAKE_COMMAND} -DSOURCE_FILENAME="${RE2_SOURCE_DIR}/util/${FILENAME}" - -DTARGET_FILENAME="${CMAKE_CURRENT_BINARY_DIR}/util/${FILENAME}" - -P "${CMAKE_CURRENT_SOURCE_DIR}/re2_transform.cmake" - COMMENT "Creating ${FILENAME} for re2_st library.") - add_custom_target (transform_${FILENAME} DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/util/${FILENAME}") - add_dependencies (re2_st transform_${FILENAME}) + add_custom_command (OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/util/${FILENAME}" + COMMAND ${CMAKE_COMMAND} -DSOURCE_FILENAME="${RE2_SOURCE_DIR}/util/${FILENAME}" + -DTARGET_FILENAME="${CMAKE_CURRENT_BINARY_DIR}/util/${FILENAME}" + -P "${CMAKE_CURRENT_SOURCE_DIR}/re2_transform.cmake" + COMMENT "Creating ${FILENAME} for re2_st library.") + add_custom_target (transform_${FILENAME} DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/util/${FILENAME}") + add_dependencies (re2_st transform_${FILENAME}) endforeach () diff --git a/contrib/ssl b/contrib/ssl index 6fbe1c6f404..dbbbcdbbd17 160000 --- a/contrib/ssl +++ b/contrib/ssl @@ -1 +1 @@ -Subproject commit 6fbe1c6f404193989c5f6a63115d80fbe34ce2a3 +Subproject commit dbbbcdbbd17785566f8f9c107b714f9e213d7293 diff --git a/contrib/unixodbc b/contrib/unixodbc new file mode 160000 index 00000000000..b0ad30f7f62 --- /dev/null +++ b/contrib/unixodbc @@ -0,0 +1 @@ +Subproject commit b0ad30f7f6289c12b76f04bfb9d466374bb32168 diff --git a/contrib/unixodbc-cmake/CMakeLists.txt b/contrib/unixodbc-cmake/CMakeLists.txt new file mode 100644 index 00000000000..33c6e115065 --- /dev/null +++ b/contrib/unixodbc-cmake/CMakeLists.txt @@ -0,0 +1,288 @@ +set(ODBC_SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/unixodbc) +set(ODBC_BINARY_DIR ${CMAKE_BINARY_DIR}/contrib/unixodbc) + + +set(SRCS +${ODBC_SOURCE_DIR}/libltdl/lt__alloc.c +${ODBC_SOURCE_DIR}/libltdl/lt__strl.c +${ODBC_SOURCE_DIR}/libltdl/ltdl.c +${ODBC_SOURCE_DIR}/libltdl/lt_dlloader.c +${ODBC_SOURCE_DIR}/libltdl/slist.c +${ODBC_SOURCE_DIR}/libltdl/lt_error.c +${ODBC_SOURCE_DIR}/libltdl/loaders/dlopen.c +${ODBC_SOURCE_DIR}/libltdl/loaders/preopen.c +#${ODBC_SOURCE_DIR}/libltdl/lt__dirent.c +#${ODBC_SOURCE_DIR}/libltdl/lt__argz.c +#${ODBC_SOURCE_DIR}/libltdl/loaders/dld_link.c +#${ODBC_SOURCE_DIR}/libltdl/loaders/load_add_on.c +#${ODBC_SOURCE_DIR}/libltdl/loaders/shl_load.c +#${ODBC_SOURCE_DIR}/libltdl/loaders/loadlibrary.c +#${ODBC_SOURCE_DIR}/libltdl/loaders/dyld.c + +# This file is generated by 'libtool' inside libltdl directory and then removed. +${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/libltdl/libltdlcS.c +) + +add_library(ltdl ${LINK_MODE} ${SRCS}) + +target_include_directories(ltdl PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/libltdl) +target_include_directories(ltdl PUBLIC ${ODBC_SOURCE_DIR}/libltdl) +target_include_directories(ltdl PUBLIC ${ODBC_SOURCE_DIR}/libltdl/libltdl) + +target_compile_definitions(ltdl PRIVATE -DHAVE_CONFIG_H -DLTDL -DLTDLOPEN=libltdlc) + +target_compile_options(ltdl PRIVATE -Wno-constant-logical-operand -Wno-unknown-warning-option -O2) + + +set(SRCS +${ODBC_SOURCE_DIR}/DriverManager/__attribute.c +${ODBC_SOURCE_DIR}/DriverManager/__connection.c +${ODBC_SOURCE_DIR}/DriverManager/__handles.c +${ODBC_SOURCE_DIR}/DriverManager/__info.c +${ODBC_SOURCE_DIR}/DriverManager/__stats.c +${ODBC_SOURCE_DIR}/DriverManager/SQLAllocConnect.c +${ODBC_SOURCE_DIR}/DriverManager/SQLAllocEnv.c +${ODBC_SOURCE_DIR}/DriverManager/SQLAllocHandle.c +${ODBC_SOURCE_DIR}/DriverManager/SQLAllocHandleStd.c +${ODBC_SOURCE_DIR}/DriverManager/SQLAllocStmt.c +${ODBC_SOURCE_DIR}/DriverManager/SQLBindCol.c +${ODBC_SOURCE_DIR}/DriverManager/SQLBindParam.c +${ODBC_SOURCE_DIR}/DriverManager/SQLBindParameter.c +${ODBC_SOURCE_DIR}/DriverManager/SQLBrowseConnect.c +${ODBC_SOURCE_DIR}/DriverManager/SQLBrowseConnectW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLBulkOperations.c +${ODBC_SOURCE_DIR}/DriverManager/SQLCancel.c +${ODBC_SOURCE_DIR}/DriverManager/SQLCancelHandle.c +${ODBC_SOURCE_DIR}/DriverManager/SQLCloseCursor.c +${ODBC_SOURCE_DIR}/DriverManager/SQLColAttribute.c +${ODBC_SOURCE_DIR}/DriverManager/SQLColAttributes.c +${ODBC_SOURCE_DIR}/DriverManager/SQLColAttributesW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLColAttributeW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLColumnPrivileges.c +${ODBC_SOURCE_DIR}/DriverManager/SQLColumnPrivilegesW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLColumns.c +${ODBC_SOURCE_DIR}/DriverManager/SQLColumnsW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLConnect.c +${ODBC_SOURCE_DIR}/DriverManager/SQLConnectW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLCopyDesc.c +${ODBC_SOURCE_DIR}/DriverManager/SQLDataSources.c +${ODBC_SOURCE_DIR}/DriverManager/SQLDataSourcesW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLDescribeCol.c +${ODBC_SOURCE_DIR}/DriverManager/SQLDescribeColW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLDescribeParam.c +${ODBC_SOURCE_DIR}/DriverManager/SQLDisconnect.c +${ODBC_SOURCE_DIR}/DriverManager/SQLDriverConnect.c +${ODBC_SOURCE_DIR}/DriverManager/SQLDriverConnectW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLDrivers.c +${ODBC_SOURCE_DIR}/DriverManager/SQLDriversW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLEndTran.c +${ODBC_SOURCE_DIR}/DriverManager/SQLError.c +${ODBC_SOURCE_DIR}/DriverManager/SQLErrorW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLExecDirect.c +${ODBC_SOURCE_DIR}/DriverManager/SQLExecDirectW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLExecute.c +${ODBC_SOURCE_DIR}/DriverManager/SQLExtendedFetch.c +${ODBC_SOURCE_DIR}/DriverManager/SQLFetch.c +${ODBC_SOURCE_DIR}/DriverManager/SQLFetchScroll.c +${ODBC_SOURCE_DIR}/DriverManager/SQLForeignKeys.c +${ODBC_SOURCE_DIR}/DriverManager/SQLForeignKeysW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLFreeConnect.c +${ODBC_SOURCE_DIR}/DriverManager/SQLFreeEnv.c +${ODBC_SOURCE_DIR}/DriverManager/SQLFreeHandle.c +${ODBC_SOURCE_DIR}/DriverManager/SQLFreeStmt.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetConnectAttr.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetConnectAttrW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetConnectOption.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetConnectOptionW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetCursorName.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetCursorNameW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetData.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetDescField.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetDescFieldW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetDescRec.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetDescRecW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetDiagField.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetDiagFieldW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetDiagRec.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetDiagRecW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetEnvAttr.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetFunctions.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetInfo.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetInfoW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetStmtAttr.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetStmtAttrW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetStmtOption.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetTypeInfo.c +${ODBC_SOURCE_DIR}/DriverManager/SQLGetTypeInfoW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLMoreResults.c +${ODBC_SOURCE_DIR}/DriverManager/SQLNativeSql.c +${ODBC_SOURCE_DIR}/DriverManager/SQLNativeSqlW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLNumParams.c +${ODBC_SOURCE_DIR}/DriverManager/SQLNumResultCols.c +${ODBC_SOURCE_DIR}/DriverManager/SQLParamData.c +${ODBC_SOURCE_DIR}/DriverManager/SQLParamOptions.c +${ODBC_SOURCE_DIR}/DriverManager/SQLPrepare.c +${ODBC_SOURCE_DIR}/DriverManager/SQLPrepareW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLPrimaryKeys.c +${ODBC_SOURCE_DIR}/DriverManager/SQLPrimaryKeysW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLProcedureColumns.c +${ODBC_SOURCE_DIR}/DriverManager/SQLProcedureColumnsW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLProcedures.c +${ODBC_SOURCE_DIR}/DriverManager/SQLProceduresW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLPutData.c +${ODBC_SOURCE_DIR}/DriverManager/SQLRowCount.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetConnectAttr.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetConnectAttrW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetConnectOption.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetConnectOptionW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetCursorName.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetCursorNameW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetDescField.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetDescFieldW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetDescRec.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetEnvAttr.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetParam.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetPos.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetScrollOptions.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetStmtAttr.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetStmtAttrW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetStmtOption.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSetStmtOptionW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSpecialColumns.c +${ODBC_SOURCE_DIR}/DriverManager/SQLSpecialColumnsW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLStatistics.c +${ODBC_SOURCE_DIR}/DriverManager/SQLStatisticsW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLTablePrivileges.c +${ODBC_SOURCE_DIR}/DriverManager/SQLTablePrivilegesW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLTables.c +${ODBC_SOURCE_DIR}/DriverManager/SQLTablesW.c +${ODBC_SOURCE_DIR}/DriverManager/SQLTransact.c + +${ODBC_SOURCE_DIR}/odbcinst/_logging.c +${ODBC_SOURCE_DIR}/odbcinst/_odbcinst_ConfigModeINI.c +${ODBC_SOURCE_DIR}/odbcinst/ODBCINSTConstructProperties.c +${ODBC_SOURCE_DIR}/odbcinst/ODBCINSTDestructProperties.c +${ODBC_SOURCE_DIR}/odbcinst/_odbcinst_GetEntries.c +${ODBC_SOURCE_DIR}/odbcinst/_odbcinst_GetSections.c +${ODBC_SOURCE_DIR}/odbcinst/ODBCINSTSetProperty.c +${ODBC_SOURCE_DIR}/odbcinst/_odbcinst_SystemINI.c +${ODBC_SOURCE_DIR}/odbcinst/_odbcinst_UserINI.c +${ODBC_SOURCE_DIR}/odbcinst/ODBCINSTValidateProperties.c +${ODBC_SOURCE_DIR}/odbcinst/ODBCINSTValidateProperty.c +${ODBC_SOURCE_DIR}/odbcinst/SQLConfigDataSource.c +${ODBC_SOURCE_DIR}/odbcinst/SQLConfigDriver.c +${ODBC_SOURCE_DIR}/odbcinst/SQLCreateDataSource.c +${ODBC_SOURCE_DIR}/odbcinst/_SQLDriverConnectPrompt.c +${ODBC_SOURCE_DIR}/odbcinst/SQLGetAvailableDrivers.c +${ODBC_SOURCE_DIR}/odbcinst/SQLGetConfigMode.c +${ODBC_SOURCE_DIR}/odbcinst/_SQLGetInstalledDrivers.c +${ODBC_SOURCE_DIR}/odbcinst/SQLGetInstalledDrivers.c +${ODBC_SOURCE_DIR}/odbcinst/SQLGetPrivateProfileString.c +${ODBC_SOURCE_DIR}/odbcinst/SQLGetTranslator.c +${ODBC_SOURCE_DIR}/odbcinst/SQLInstallDriverEx.c +${ODBC_SOURCE_DIR}/odbcinst/SQLInstallDriverManager.c +${ODBC_SOURCE_DIR}/odbcinst/SQLInstallerError.c +${ODBC_SOURCE_DIR}/odbcinst/SQLInstallODBC.c +${ODBC_SOURCE_DIR}/odbcinst/SQLInstallTranslatorEx.c +${ODBC_SOURCE_DIR}/odbcinst/SQLManageDataSources.c +${ODBC_SOURCE_DIR}/odbcinst/SQLPostInstallerError.c +${ODBC_SOURCE_DIR}/odbcinst/SQLReadFileDSN.c +${ODBC_SOURCE_DIR}/odbcinst/SQLRemoveDriver.c +${ODBC_SOURCE_DIR}/odbcinst/SQLRemoveDriverManager.c +${ODBC_SOURCE_DIR}/odbcinst/SQLRemoveDSNFromIni.c +${ODBC_SOURCE_DIR}/odbcinst/SQLRemoveTranslator.c +${ODBC_SOURCE_DIR}/odbcinst/SQLSetConfigMode.c +${ODBC_SOURCE_DIR}/odbcinst/SQLValidDSN.c +${ODBC_SOURCE_DIR}/odbcinst/SQLWriteDSNToIni.c +${ODBC_SOURCE_DIR}/odbcinst/SQLWriteFileDSN.c +${ODBC_SOURCE_DIR}/odbcinst/_SQLWriteInstalledDrivers.c +${ODBC_SOURCE_DIR}/odbcinst/SQLWritePrivateProfileString.c + +${ODBC_SOURCE_DIR}/ini/iniAllTrim.c +${ODBC_SOURCE_DIR}/ini/iniAppend.c +${ODBC_SOURCE_DIR}/ini/iniClose.c +${ODBC_SOURCE_DIR}/ini/iniCommit.c +${ODBC_SOURCE_DIR}/ini/iniCursor.c +${ODBC_SOURCE_DIR}/ini/iniDelete.c +${ODBC_SOURCE_DIR}/ini/_iniDump.c +${ODBC_SOURCE_DIR}/ini/iniElement.c +${ODBC_SOURCE_DIR}/ini/iniElementCount.c +${ODBC_SOURCE_DIR}/ini/iniGetBookmark.c +${ODBC_SOURCE_DIR}/ini/iniGotoBookmark.c +${ODBC_SOURCE_DIR}/ini/iniObject.c +${ODBC_SOURCE_DIR}/ini/iniObjectDelete.c +${ODBC_SOURCE_DIR}/ini/iniObjectEOL.c +${ODBC_SOURCE_DIR}/ini/iniObjectFirst.c +${ODBC_SOURCE_DIR}/ini/iniObjectInsert.c +${ODBC_SOURCE_DIR}/ini/iniObjectLast.c +${ODBC_SOURCE_DIR}/ini/iniObjectNext.c +${ODBC_SOURCE_DIR}/ini/_iniObjectRead.c +${ODBC_SOURCE_DIR}/ini/iniObjectSeek.c +${ODBC_SOURCE_DIR}/ini/iniObjectSeekSure.c +${ODBC_SOURCE_DIR}/ini/iniObjectUpdate.c +${ODBC_SOURCE_DIR}/ini/iniOpen.c +${ODBC_SOURCE_DIR}/ini/iniProperty.c +${ODBC_SOURCE_DIR}/ini/iniPropertyDelete.c +${ODBC_SOURCE_DIR}/ini/iniPropertyEOL.c +${ODBC_SOURCE_DIR}/ini/iniPropertyFirst.c +${ODBC_SOURCE_DIR}/ini/iniPropertyInsert.c +${ODBC_SOURCE_DIR}/ini/iniPropertyLast.c +${ODBC_SOURCE_DIR}/ini/iniPropertyNext.c +${ODBC_SOURCE_DIR}/ini/_iniPropertyRead.c +${ODBC_SOURCE_DIR}/ini/iniPropertySeek.c +${ODBC_SOURCE_DIR}/ini/iniPropertySeekSure.c +${ODBC_SOURCE_DIR}/ini/iniPropertyUpdate.c +${ODBC_SOURCE_DIR}/ini/iniPropertyValue.c +${ODBC_SOURCE_DIR}/ini/_iniScanUntilObject.c +${ODBC_SOURCE_DIR}/ini/iniToUpper.c +${ODBC_SOURCE_DIR}/ini/iniValue.c + +${ODBC_SOURCE_DIR}/log/logClear.c +${ODBC_SOURCE_DIR}/log/logClose.c +${ODBC_SOURCE_DIR}/log/_logFreeMsg.c +${ODBC_SOURCE_DIR}/log/logOn.c +${ODBC_SOURCE_DIR}/log/logOpen.c +${ODBC_SOURCE_DIR}/log/logPeekMsg.c +${ODBC_SOURCE_DIR}/log/logPopMsg.c +${ODBC_SOURCE_DIR}/log/logPushMsg.c + +${ODBC_SOURCE_DIR}/lst/_lstAdjustCurrent.c +${ODBC_SOURCE_DIR}/lst/lstAppend.c +${ODBC_SOURCE_DIR}/lst/lstClose.c +${ODBC_SOURCE_DIR}/lst/lstDelete.c +${ODBC_SOURCE_DIR}/lst/_lstDump.c +${ODBC_SOURCE_DIR}/lst/lstEOL.c +${ODBC_SOURCE_DIR}/lst/lstFirst.c +${ODBC_SOURCE_DIR}/lst/_lstFreeItem.c +${ODBC_SOURCE_DIR}/lst/lstGetBookMark.c +${ODBC_SOURCE_DIR}/lst/lstGet.c +${ODBC_SOURCE_DIR}/lst/lstGotoBookMark.c +${ODBC_SOURCE_DIR}/lst/lstGoto.c +${ODBC_SOURCE_DIR}/lst/lstInsert.c +${ODBC_SOURCE_DIR}/lst/lstLast.c +${ODBC_SOURCE_DIR}/lst/lstNext.c +${ODBC_SOURCE_DIR}/lst/_lstNextValidItem.c +${ODBC_SOURCE_DIR}/lst/lstOpen.c +${ODBC_SOURCE_DIR}/lst/lstOpenCursor.c +${ODBC_SOURCE_DIR}/lst/lstPrev.c +${ODBC_SOURCE_DIR}/lst/_lstPrevValidItem.c +${ODBC_SOURCE_DIR}/lst/lstSeek.c +${ODBC_SOURCE_DIR}/lst/lstSeekItem.c +${ODBC_SOURCE_DIR}/lst/lstSet.c +${ODBC_SOURCE_DIR}/lst/lstSetFreeFunc.c +${ODBC_SOURCE_DIR}/lst/_lstVisible.c +) + +add_library(unixodbc ${LINK_MODE} ${SRCS}) + +target_link_libraries(unixodbc ltdl) + +# SYSTEM_FILE_PATH was changed to /etc + +target_include_directories(unixodbc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/private) +target_include_directories(unixodbc PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64) +target_include_directories(unixodbc PUBLIC ${ODBC_SOURCE_DIR}/include) + +target_compile_definitions(unixodbc PRIVATE -DHAVE_CONFIG_H) + +target_compile_options(unixodbc PRIVATE -Wno-dangling-else -Wno-parentheses -Wno-misleading-indentation -Wno-unknown-warning-option -O2) diff --git a/contrib/unixodbc-cmake/linux_x86_64/libltdl/config.h b/contrib/unixodbc-cmake/linux_x86_64/libltdl/config.h new file mode 100644 index 00000000000..194779b2b98 --- /dev/null +++ b/contrib/unixodbc-cmake/linux_x86_64/libltdl/config.h @@ -0,0 +1,181 @@ +/* config.h. Generated from config-h.in by configure. */ +/* config-h.in. Generated from configure.ac by autoheader. */ + +/* Define to 1 if you have the `argz_add' function. */ +#define HAVE_ARGZ_ADD 1 + +/* Define to 1 if you have the `argz_append' function. */ +#define HAVE_ARGZ_APPEND 1 + +/* Define to 1 if you have the `argz_count' function. */ +#define HAVE_ARGZ_COUNT 1 + +/* Define to 1 if you have the `argz_create_sep' function. */ +#define HAVE_ARGZ_CREATE_SEP 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARGZ_H 1 + +/* Define to 1 if you have the `argz_insert' function. */ +#define HAVE_ARGZ_INSERT 1 + +/* Define to 1 if you have the `argz_next' function. */ +#define HAVE_ARGZ_NEXT 1 + +/* Define to 1 if you have the `argz_stringify' function. */ +#define HAVE_ARGZ_STRINGIFY 1 + +/* Define to 1 if you have the `closedir' function. */ +#define HAVE_CLOSEDIR 1 + +/* Define to 1 if you have the declaration of `cygwin_conv_path', and to 0 if + you don't. */ +/* #undef HAVE_DECL_CYGWIN_CONV_PATH */ + +/* Define to 1 if you have the header file. */ +#define HAVE_DIRENT_H 1 + +/* Define if you have the GNU dld library. */ +/* #undef HAVE_DLD */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_DLD_H */ + +/* Define to 1 if you have the `dlerror' function. */ +#define HAVE_DLERROR 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_DL_H */ + +/* Define if you have the _dyld_func_lookup function. */ +/* #undef HAVE_DYLD */ + +/* Define to 1 if the system has the type `error_t'. */ +#define HAVE_ERROR_T 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define if you have the libdl library or equivalent. */ +#define HAVE_LIBDL 1 + +/* Define if libdlloader will be built on this platform */ +#define HAVE_LIBDLLOADER 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_MACH_O_DYLD_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the `opendir' function. */ +#define HAVE_OPENDIR 1 + +/* Define if libtool can extract symbol lists from object files. */ +#define HAVE_PRELOADED_SYMBOLS 1 + +/* Define to 1 if you have the `readdir' function. */ +#define HAVE_READDIR 1 + +/* Define if you have the shl_load function. */ +/* #undef HAVE_SHL_LOAD */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the `strlcat' function. */ +/* #undef HAVE_STRLCAT */ + +/* Define to 1 if you have the `strlcpy' function. */ +/* #undef HAVE_STRLCPY */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_DL_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* This value is set to 1 to indicate that the system argz facility works */ +#define HAVE_WORKING_ARGZ 1 + +/* Define if the OS needs help to load dependent libraries for dlopen(). */ +/* #undef LTDL_DLOPEN_DEPLIBS */ + +/* Define to the system default library search path. */ +#define LT_DLSEARCH_PATH "/lib:/usr/lib:/usr/lib/x86_64-linux-gnu/libfakeroot:/usr/local/lib:/usr/local/lib/x86_64-linux-gnu:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/lib32:/usr/lib32" + +/* The archive extension */ +#define LT_LIBEXT "a" + +/* The archive prefix */ +#define LT_LIBPREFIX "lib" + +/* Define to the extension used for runtime loadable modules, say, ".so". */ +#define LT_MODULE_EXT ".so" + +/* Define to the name of the environment variable that determines the run-time + module search path. */ +#define LT_MODULE_PATH_VAR "LD_LIBRARY_PATH" + +/* Define to the sub-directory where libtool stores uninstalled libraries. */ +#define LT_OBJDIR ".libs/" + +/* Define to the shared library suffix, say, ".dylib". */ +/* #undef LT_SHARED_EXT */ + +/* Define to the shared archive member specification, say "(shr.o)". */ +/* #undef LT_SHARED_LIB_MEMBER */ + +/* Define if dlsym() requires a leading underscore in symbol names. */ +/* #undef NEED_USCORE */ + +/* Name of package */ +#define PACKAGE "libltdl" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "bug-libtool@gnu.org" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "libltdl" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "libltdl 2.4.3a" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "libltdl" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "2.4.3a" + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Version number of package */ +#define VERSION "2.4.3a" + +/* Define so that glibc/gnulib argp.h does not typedef error_t. */ +/* #undef __error_t_defined */ + +/* Define to a type to use for 'error_t' if it is not otherwise available. */ +/* #undef error_t */ diff --git a/contrib/unixodbc-cmake/linux_x86_64/libltdl/libltdlcS.c b/contrib/unixodbc-cmake/linux_x86_64/libltdl/libltdlcS.c new file mode 100644 index 00000000000..ca866eb5986 --- /dev/null +++ b/contrib/unixodbc-cmake/linux_x86_64/libltdl/libltdlcS.c @@ -0,0 +1,53 @@ +/* libltdlcS.c - symbol resolution table for 'libltdlc' dlsym emulation. */ +/* Generated by libtool (GNU libtool) 2.4.6 */ + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined __GNUC__ && (((__GNUC__ == 4) && (__GNUC_MINOR__ >= 4)) || (__GNUC__ > 4)) +#pragma GCC diagnostic ignored "-Wstrict-prototypes" +#endif + +/* Keep this code in sync between libtool.m4, ltmain, lt_system.h, and tests. */ +#if defined _WIN32 || defined __CYGWIN__ || defined _WIN32_WCE +/* DATA imports from DLLs on WIN32 can't be const, because runtime + relocations are performed -- see ld's documentation on pseudo-relocs. */ +# define LT_DLSYM_CONST +#elif defined __osf__ +/* This system does not cope well with relocations in const data. */ +# define LT_DLSYM_CONST +#else +# define LT_DLSYM_CONST const +#endif + +#define STREQ(s1, s2) (strcmp ((s1), (s2)) == 0) + +/* External symbol declarations for the compiler. */ +extern int dlopen_LTX_get_vtable(); + +/* The mapping between symbol names and symbols. */ +typedef struct { + const char *name; + void *address; +} lt_dlsymlist; +extern LT_DLSYM_CONST lt_dlsymlist +lt_libltdlc_LTX_preloaded_symbols[]; +LT_DLSYM_CONST lt_dlsymlist +lt_libltdlc_LTX_preloaded_symbols[] = +{ {"libltdlc", (void *) 0}, + {"dlopen.a", (void *) 0}, + {"dlopen_LTX_get_vtable", (void *) &dlopen_LTX_get_vtable}, + {0, (void *) 0} +}; + +/* This works around a problem in FreeBSD linker */ +#ifdef FREEBSD_WORKAROUND +static const void *lt_preloaded_setup() { + return lt_libltdlc_LTX_preloaded_symbols; +} +#endif + +#ifdef __cplusplus +} +#endif diff --git a/contrib/unixodbc-cmake/linux_x86_64/private/config.h b/contrib/unixodbc-cmake/linux_x86_64/private/config.h new file mode 100644 index 00000000000..d80a4da4665 --- /dev/null +++ b/contrib/unixodbc-cmake/linux_x86_64/private/config.h @@ -0,0 +1,496 @@ +/* config.h. Generated from config.h.in by configure. */ +/* config.h.in. Generated from configure.ac by autoheader. */ + +/* Encoding to use for CHAR */ +#define ASCII_ENCODING "auto-search" + +/* Install bindir */ +#define BIN_PREFIX "/usr/local/bin" + +/* Use a semaphore to allow ODBCConfig to display running counts */ +/* #undef COLLECT_STATS */ + +/* Define to one of `_getb67', `GETB67', `getb67' for Cray-2 and Cray-YMP + systems. This function is required for `alloca.c' support on those systems. + */ +/* #undef CRAY_STACKSEG_END */ + +/* Define to 1 if using `alloca.c'. */ +/* #undef C_ALLOCA */ + +/* Lib directory */ +#define DEFLIB_PATH "/usr/local/lib" + +/* Using ini cacheing */ +#define ENABLE_INI_CACHING /**/ + +/* Install exec_prefix */ +#define EXEC_PREFIX "/usr/local" + +/* Disable the precise but slow checking of the validity of handles */ +/* #undef FAST_HANDLE_VALIDATE */ + +/* Define to 1 if you have `alloca', as a function or macro. */ +#define HAVE_ALLOCA 1 + +/* Define to 1 if you have and it should be used (not on Ultrix). + */ +#define HAVE_ALLOCA_H 1 + +/* Define to 1 if you have the `argz_add' function. */ +#define HAVE_ARGZ_ADD 1 + +/* Define to 1 if you have the `argz_append' function. */ +#define HAVE_ARGZ_APPEND 1 + +/* Define to 1 if you have the `argz_count' function. */ +#define HAVE_ARGZ_COUNT 1 + +/* Define to 1 if you have the `argz_create_sep' function. */ +#define HAVE_ARGZ_CREATE_SEP 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARGZ_H 1 + +/* Define to 1 if you have the `argz_insert' function. */ +#define HAVE_ARGZ_INSERT 1 + +/* Define to 1 if you have the `argz_next' function. */ +#define HAVE_ARGZ_NEXT 1 + +/* Define to 1 if you have the `argz_stringify' function. */ +#define HAVE_ARGZ_STRINGIFY 1 + +/* Define to 1 if you have the `atoll' function. */ +#define HAVE_ATOLL 1 + +/* Define to 1 if you have the `closedir' function. */ +#define HAVE_CLOSEDIR 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_CRYPT_H 1 + +/* Define to 1 if you have the declaration of `cygwin_conv_path', and to 0 if + you don't. */ +/* #undef HAVE_DECL_CYGWIN_CONV_PATH */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +#define HAVE_DIRENT_H 1 + +/* Define if you have the GNU dld library. */ +/* #undef HAVE_DLD */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_DLD_H */ + +/* Define to 1 if you have the `dlerror' function. */ +#define HAVE_DLERROR 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_DL_H */ + +/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */ +/* #undef HAVE_DOPRNT */ + +/* Define if you have the _dyld_func_lookup function. */ +/* #undef HAVE_DYLD */ + +/* Define to 1 if you have the `endpwent' function. */ +#define HAVE_ENDPWENT 1 + +/* Define to 1 if the system has the type `error_t'. */ +#define HAVE_ERROR_T 1 + +/* Define to 1 if you have the `ftime' function. */ +#define HAVE_FTIME 1 + +/* Define to 1 if you have the `ftok' function. */ +/* #undef HAVE_FTOK */ + +/* Define to 1 if you have the `getpwuid' function. */ +#define HAVE_GETPWUID 1 + +/* Define to 1 if you have the `gettimeofday' function. */ +#define HAVE_GETTIMEOFDAY 1 + +/* Define to 1 if you have the `getuid' function. */ +#define HAVE_GETUID 1 + +/* Define if you have the iconv() function. */ +#define HAVE_ICONV 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define if you have and nl_langinfo(CODESET). */ +#define HAVE_LANGINFO_CODESET 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_LANGINFO_H 1 + +/* Add -lcrypt to lib list */ +#define HAVE_LIBCRYPT /**/ + +/* Define if you have the libdl library or equivalent. */ +#define HAVE_LIBDL 1 + +/* Define if libdlloader will be built on this platform */ +#define HAVE_LIBDLLOADER 1 + +/* Use the -lpth thread library */ +/* #undef HAVE_LIBPTH */ + +/* Use -lpthread threading lib */ +#define HAVE_LIBPTHREAD 1 + +/* Use the -lthread threading lib */ +/* #undef HAVE_LIBTHREAD */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_LOCALE_H 1 + +/* Use rentrant version of localtime */ +#define HAVE_LOCALTIME_R 1 + +/* Define if you have long long */ +#define HAVE_LONG_LONG 1 + +/* Define this if a modern libltdl is already installed */ +#define HAVE_LTDL 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_MACH_O_DYLD_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_MALLOC_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_MSQL_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. */ +/* #undef HAVE_NDIR_H */ + +/* Define to 1 if you have the `nl_langinfo' function. */ +#define HAVE_NL_LANGINFO 1 + +/* Define to 1 if you have the `opendir' function. */ +#define HAVE_OPENDIR 1 + +/* Define if libtool can extract symbol lists from object files. */ +#define HAVE_PRELOADED_SYMBOLS 1 + +/* Define to 1 if the system has the type `ptrdiff_t'. */ +#define HAVE_PTRDIFF_T 1 + +/* Define to 1 if you have the `putenv' function. */ +#define HAVE_PUTENV 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_PWD_H 1 + +/* Define to 1 if you have the `readdir' function. */ +#define HAVE_READDIR 1 + +/* Add readline support */ +#define HAVE_READLINE 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_READLINE_HISTORY_H 1 + +/* Use the scandir lib */ +/* #undef HAVE_SCANDIR */ + +/* Define to 1 if you have the `semget' function. */ +/* #undef HAVE_SEMGET */ + +/* Define to 1 if you have the `semop' function. */ +/* #undef HAVE_SEMOP */ + +/* Define to 1 if you have the `setenv' function. */ +#define HAVE_SETENV 1 + +/* Define to 1 if you have the `setlocale' function. */ +#define HAVE_SETLOCALE 1 + +/* Define if you have the shl_load function. */ +/* #undef HAVE_SHL_LOAD */ + +/* Define to 1 if you have the `shmget' function. */ +/* #undef HAVE_SHMGET */ + +/* Define to 1 if you have the `snprintf' function. */ +/* #undef HAVE_SNPRINTF */ + +/* Define to 1 if you have the `socket' function. */ +#define HAVE_SOCKET 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDARG_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDDEF_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the `strcasecmp' function. */ +#define HAVE_STRCASECMP 1 + +/* Define to 1 if you have the `strchr' function. */ +#define HAVE_STRCHR 1 + +/* Define to 1 if you have the `strdup' function. */ +#define HAVE_STRDUP 1 + +/* Define to 1 if you have the `stricmp' function. */ +/* #undef HAVE_STRICMP */ + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the `strlcat' function. */ +/* #undef HAVE_STRLCAT */ + +/* Define to 1 if you have the `strlcpy' function. */ +/* #undef HAVE_STRLCPY */ + +/* Define to 1 if you have the `strncasecmp' function. */ +#define HAVE_STRNCASECMP 1 + +/* Define to 1 if you have the `strnicmp' function. */ +/* #undef HAVE_STRNICMP */ + +/* Define to 1 if you have the `strstr' function. */ +#define HAVE_STRSTR 1 + +/* Define to 1 if you have the `strtol' function. */ +#define HAVE_STRTOL 1 + +/* Define to 1 if you have the `strtoll' function. */ +#define HAVE_STRTOLL 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYNCH_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_DIR_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_DL_H */ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_SYS_MALLOC_H */ + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_NDIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SEM_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIMEB_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the `time' function. */ +#define HAVE_TIME 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_VARARGS_H */ + +/* Define to 1 if you have the `vprintf' function. */ +#define HAVE_VPRINTF 1 + +/* Define to 1 if you have the `vsnprintf' function. */ +#define HAVE_VSNPRINTF 1 + +/* This value is set to 1 to indicate that the system argz facility works */ +#define HAVE_WORKING_ARGZ 1 + +/* Define as const if the declaration of iconv() needs const. */ +#define ICONV_CONST + +/* Install includedir */ +#define INCLUDE_PREFIX "/usr/local/include" + +/* Lib directory */ +#define LIB_PREFIX "/usr/local/lib" + +/* Define if the OS needs help to load dependent libraries for dlopen(). */ +/* #undef LTDL_DLOPEN_DEPLIBS */ + +/* Define to the system default library search path. */ +#define LT_DLSEARCH_PATH "/lib:/usr/lib:/usr/lib/x86_64-linux-gnu/libfakeroot:/usr/local/lib:/usr/local/lib/x86_64-linux-gnu:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/lib32:/usr/lib32" + +/* The archive extension */ +#define LT_LIBEXT "a" + +/* The archive prefix */ +#define LT_LIBPREFIX "lib" + +/* Define to the extension used for runtime loadable modules, say, ".so". */ +#define LT_MODULE_EXT ".so" + +/* Define to the name of the environment variable that determines the run-time + module search path. */ +#define LT_MODULE_PATH_VAR "LD_LIBRARY_PATH" + +/* Define to the sub-directory where libtool stores uninstalled libraries. */ +#define LT_OBJDIR ".libs/" + +/* Define to the shared library suffix, say, ".dylib". */ +/* #undef LT_SHARED_EXT */ + +/* Define to the shared archive member specification, say "(shr.o)". */ +/* #undef LT_SHARED_LIB_MEMBER */ + +/* Define if you need semundo union */ +/* #undef NEED_SEMUNDO_UNION */ + +/* Define if dlsym() requires a leading underscore in symbol names. */ +/* #undef NEED_USCORE */ + +/* Using OSX */ +/* #undef OSXHEADER */ + +/* Name of package */ +#define PACKAGE "unixODBC" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "nick@unixodbc.org" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "unixODBC" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "unixODBC 2.3.6" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "unixODBC" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "2.3.6" + +/* Platform is 64 bit */ +#define PLATFORM64 /**/ + +/* Install prefix */ +#define PREFIX "/usr/local" + +/* Using QNX */ +/* #undef QNX_LIBLTDL */ + +/* Shared lib extension */ +#define SHLIBEXT ".so" + +/* The size of `long', as computed by sizeof. */ +#define SIZEOF_LONG 8 + +/* The size of `long int', as computed by sizeof. */ +#define SIZEOF_LONG_INT 8 + +/* If using the C implementation of alloca, define if you know the + direction of stack growth for your system; otherwise it will be + automatically deduced at runtime. + STACK_DIRECTION > 0 => grows toward higher addresses + STACK_DIRECTION < 0 => grows toward lower addresses + STACK_DIRECTION = 0 => direction of growth unknown */ +/* #undef STACK_DIRECTION */ + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* don't include unixODBC prefix in driver error messages */ +#define STRICT_ODBC_ERROR /**/ + +/* System file path */ +#define SYSTEM_FILE_PATH "/etc" + +/* Lib path */ +#define SYSTEM_LIB_PATH "/usr/local/lib" + +/* Define to 1 if you can safely include both and . */ +#define TIME_WITH_SYS_TIME 1 + +/* Define to 1 if your declares `struct tm'. */ +/* #undef TM_IN_SYS_TIME */ + +/* Encoding to use for UNICODE */ +#define UNICODE_ENCODING "auto-search" + +/* Flag that we are not using another DM */ +#define UNIXODBC /**/ + +/* We are building inside the unixODBC source tree */ +#define UNIXODBC_SOURCE /**/ + +/* Version number of package */ +#define VERSION "2.3.6" + +/* Work with IBM drivers that use 32 bit handles on 64 bit platforms */ +/* #undef WITH_HANDLE_REDIRECT */ + +/* Define to 1 if `lex' declares `yytext' as a `char *' by default, not a + `char[]'. */ +/* #undef YYTEXT_POINTER */ + +/* Build flag for AIX */ +/* #undef _ALL_SOURCE */ + +/* Build flag for AIX */ +/* #undef _LONG_LONG */ + +/* Build flag for AIX */ +/* #undef _THREAD_SAFE */ + +/* Define so that glibc/gnulib argp.h does not typedef error_t. */ +/* #undef __error_t_defined */ + +/* Define to empty if `const' does not conform to ANSI C. */ +/* #undef const */ + +/* Define to a type to use for 'error_t' if it is not otherwise available. */ +/* #undef error_t */ + +/* Define to `int' if doesn't define. */ +/* #undef gid_t */ + +/* Define to `unsigned int' if does not define. */ +/* #undef size_t */ + +/* Define to `int' if doesn't define. */ +/* #undef uid_t */ diff --git a/contrib/unixodbc-cmake/linux_x86_64/unixodbc_conf.h b/contrib/unixodbc-cmake/linux_x86_64/unixodbc_conf.h new file mode 100644 index 00000000000..6597c85cea6 --- /dev/null +++ b/contrib/unixodbc-cmake/linux_x86_64/unixodbc_conf.h @@ -0,0 +1,60 @@ +/* unixodbc_conf.h. Generated from unixodbc_conf.h.in by configure. */ +#ifndef HAVE_UNISTD_H +#define HAVE_UNISTD_H 1 +#endif + +#ifndef HAVE_PWD_H +#define HAVE_PWD_H 1 +#endif + +#ifndef HAVE_SYS_TIME_H +#define HAVE_SYS_TIME_H 1 +#endif + +#ifndef ODBC_STD +/* #undef ODBC_STD */ +#endif + +#ifndef UNICODE +/* #undef UNICODE */ +#endif + +#ifndef GUID_DEFINED +/* #undef GUID_DEFINED */ +#endif + +#ifndef SQL_WCHART_CONVERT +/* #undef SQL_WCHART_CONVERT */ +#endif + +#ifndef HAVE_LONG_LONG +#define HAVE_LONG_LONG 1 +#endif + +#ifndef ODBCINT64_TYPEA +/* #undef ODBCINT64_TYPEA */ +#endif + +#ifndef UODBCINT64_TYPE +/* #undef UODBCINT64_TYPE */ +#endif + +#ifndef DISABLE_INI_CACHING +/* #undef DISABLE_INI_CACHING */ +#endif + +#ifndef SIZEOF_LONG_INT +#define SIZEOF_LONG_INT 8 +#endif + +#ifndef ALLREADY_HAVE_WINDOWS_TYPE +/* #undef ALLREADY_HAVE_WINDOWS_TYPE */ +#endif + +#ifndef DONT_TD_VOID +/* #undef DONT_TD_VOID */ +#endif + +#ifndef DO_YOU_KNOW_WHAT_YOUR_ARE_DOING +/* #undef DO_YOU_KNOW_WHAT_YOUR_ARE_DOING */ +#endif diff --git a/contrib/zlib-ng b/contrib/zlib-ng index e07a52dbaa3..9173b89d467 160000 --- a/contrib/zlib-ng +++ b/contrib/zlib-ng @@ -1 +1 @@ -Subproject commit e07a52dbaa35d003f5659b221b29d220c091667b +Subproject commit 9173b89d46799582d20a30578e0aa9788bc7d6e1 diff --git a/contrib/zookeeper b/contrib/zookeeper deleted file mode 160000 index 438afae5af3..00000000000 --- a/contrib/zookeeper +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 438afae5af36c5be9c82d074f43a9bb19e0797c0 diff --git a/contrib/zstd b/contrib/zstd index f4340f46b23..255597502c3 160000 --- a/contrib/zstd +++ b/contrib/zstd @@ -1 +1 @@ -Subproject commit f4340f46b2387bc8de7d5320c0b83bb1499933ad +Subproject commit 255597502c3a4ef150abc964e376d4202a8c2929 diff --git a/contrib/zstd-cmake/CMakeLists.txt b/contrib/zstd-cmake/CMakeLists.txt index 6d8ecce2a44..3686e9ae1c0 100644 --- a/contrib/zstd-cmake/CMakeLists.txt +++ b/contrib/zstd-cmake/CMakeLists.txt @@ -125,6 +125,6 @@ IF (ZSTD_LEGACY_SUPPORT) ${LIBRARY_LEGACY_DIR}/zstd_v07.h) ENDIF (ZSTD_LEGACY_SUPPORT) -ADD_LIBRARY(zstd ${Sources} ${Headers}) +ADD_LIBRARY(zstd ${LINK_MODE} ${Sources} ${Headers}) target_include_directories (zstd PUBLIC ${LIBRARY_DIR}) diff --git a/copy_headers.sh b/copy_headers.sh index 7c106f2c3be..a32a809774e 100755 --- a/copy_headers.sh +++ b/copy_headers.sh @@ -1,4 +1,8 @@ -#!/bin/bash -e +#!/usr/bin/env bash + +set -e +#set -x +#echo "Args: $*"; env | sort # Этот скрипт собирает все заголовочные файлы, нужные для компиляции некоторого translation unit-а # и копирует их с сохранением путей в директорию DST. @@ -8,7 +12,7 @@ # Требуется clang, желательно наиболее свежий (trunk). # # Используется при сборке пакетов. -# Заголовочные файлы записываются в пакет clickhouse-server-base, в директорию /usr/share/clickhouse/headers. +# Заголовочные файлы записываются в пакет clickhouse-common, в директорию /usr/share/clickhouse/headers. # # Если вы хотите установить их самостоятельно, без сборки пакета, # чтобы clickhouse-server видел их там, где ожидается, выполните: @@ -17,7 +21,7 @@ SOURCE_PATH=${1:-.} DST=${2:-$SOURCE_PATH/../headers} -BUILD_PATH=${3:-$SOURCE_PATH/build} +BUILD_PATH=${BUILD_PATH=${3:-$SOURCE_PATH/build}} PATH="/usr/local/bin:/usr/local/sbin:/usr/bin:$PATH" @@ -30,23 +34,30 @@ START_HEADERS=$(echo \ $SOURCE_PATH/dbms/src/Interpreters/SpecializedAggregator.h \ $SOURCE_PATH/dbms/src/AggregateFunctions/AggregateFunction*.h) +for header in $START_HEADERS; do + START_HEADERS_INCLUDE+="-include $header " +done + # Опция -mcx16 для того, чтобы выбиралось больше заголовочных файлов (с запасом). # The latter options are the same that are added while building packages. +# TODO: Does not work on macos: GCC_ROOT=`$CLANG -v 2>&1 | grep "Selected GCC installation"| sed -n -e 's/^.*: //p'` for src_file in $(echo | $CLANG -M -xc++ -std=c++1z -Wall -Werror -msse4 -mcx16 -mpopcnt -O3 -g -fPIC -fstack-protector -D_FORTIFY_SOURCE=2 \ -I $GCC_ROOT/include \ -I $GCC_ROOT/include-fixed \ $(cat "$BUILD_PATH/include_directories.txt") \ - $(echo $START_HEADERS | sed -r -e 's/[^ ]+/-include \0/g') \ + $START_HEADERS_INCLUDE \ - | tr -d '\\' | - sed -r -e 's/^-\.o://'); + sed --posix -E -e 's/^-\.o://'); do dst_file=$src_file; - dst_file=$(echo $dst_file | sed -r -e 's/build\///') # for simplicity reasons, will put generated headers near the rest. - mkdir -p "$DST/$(echo $dst_file | sed -r -e 's/\/[^/]*$/\//')"; + [ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed --posix -E -e "s!^$BUILD_PATH!!") + [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed --posix -E -e "s!^$DESTDIR!!") + dst_file=$(echo $dst_file | sed --posix -E -e 's/build\///') # for simplicity reasons, will put generated headers near the rest. + mkdir -p "$DST/$(echo $dst_file | sed --posix -E -e 's/\/[^/]*$/\//')"; cp "$src_file" "$DST/$dst_file"; done @@ -56,19 +67,32 @@ done for src_file in $(ls -1 $($CLANG -v -xc++ - <<<'' 2>&1 | grep '^ /' | grep 'include' | grep -E '/lib/clang/|/include/clang/')/*.h | grep -vE 'arm|altivec|Intrin'); do - mkdir -p "$DST/$(echo $src_file | sed -r -e 's/\/[^/]*$/\//')"; - cp "$src_file" "$DST/$src_file"; + dst_file=$src_file; + [ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed --posix -E -e "s!^$BUILD_PATH!!") + [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed --posix -E -e "s!^$DESTDIR!!") + mkdir -p "$DST/$(echo $dst_file | sed --posix -E -e 's/\/[^/]*$/\//')"; + cp "$src_file" "$DST/$dst_file"; done -# Even more platform-specific headers -for src_file in $(ls -1 $SOURCE_PATH/contrib/boost/libs/smart_ptr/include/boost/smart_ptr/detail/*); -do - mkdir -p "$DST/$(echo $src_file | sed -r -e 's/\/[^/]*$/\//')"; - cp "$src_file" "$DST/$src_file"; -done +if [ -d "$SOURCE_PATH/contrib/boost/libs/smart_ptr/include/boost/smart_ptr/detail" ]; then + # Even more platform-specific headers + for src_file in $(ls -1 $SOURCE_PATH/contrib/boost/libs/smart_ptr/include/boost/smart_ptr/detail/*); + do + dst_file=$src_file; + [ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed --posix -E -e "s!^$BUILD_PATH!!") + [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed --posix -E -e "s!^$DESTDIR!!") + mkdir -p "$DST/$(echo $dst_file | sed --posix -E -e 's/\/[^/]*$/\//')"; + cp "$src_file" "$DST/$dst_file"; + done +fi -for src_file in $(ls -1 $SOURCE_PATH/contrib/boost/boost/smart_ptr/detail/*); -do - mkdir -p "$DST/$(echo $src_file | sed -r -e 's/\/[^/]*$/\//')"; - cp "$src_file" "$DST/$src_file"; -done +if [ -d "$SOURCE_PATH/contrib/boost/boost/smart_ptr/detail" ]; then + for src_file in $(ls -1 $SOURCE_PATH/contrib/boost/boost/smart_ptr/detail/*); + do + dst_file=$src_file; + [ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed --posix -E -e "s!^$BUILD_PATH!!") + [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed --posix -E -e "s!^$DESTDIR!!") + mkdir -p "$DST/$(echo $dst_file | sed --posix -E -e 's/\/[^/]*$/\//')"; + cp "$src_file" "$DST/$dst_file"; + done +fi diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index d740ce5a45c..cce97e4a57e 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -6,18 +6,12 @@ include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/find_vectorclass.cmake) set (CONFIG_VERSION ${CMAKE_CURRENT_BINARY_DIR}/src/Common/config_version.h) set (CONFIG_COMMON ${CMAKE_CURRENT_BINARY_DIR}/src/Common/config.h) -set (CONFIG_BUILD ${CMAKE_CURRENT_BINARY_DIR}/src/Common/config_build.cpp) include (cmake/version.cmake) message (STATUS "Will build ${VERSION_FULL}") configure_file (${CMAKE_CURRENT_SOURCE_DIR}/src/Common/config.h.in ${CONFIG_COMMON}) configure_file (${CMAKE_CURRENT_SOURCE_DIR}/src/Common/config_version.h.in ${CONFIG_VERSION}) -get_property (BUILD_COMPILE_DEFINITIONS DIRECTORY ${ClickHouse_SOURCE_DIR} PROPERTY COMPILE_DEFINITIONS) -get_property (BUILD_INCLUDE_DIRECTORIES DIRECTORY ${ClickHouse_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES) -string (TIMESTAMP BUILD_DATE "%Y-%m-%d" UTC) -configure_file (${CMAKE_CURRENT_SOURCE_DIR}/src/Common/config_build.cpp.in ${CONFIG_BUILD}) - if (NOT MSVC) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra") endif () @@ -27,7 +21,32 @@ if (NOT NO_WERROR) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror") endif () -find_package (Threads) +# Add some warnings that are not available even with -Wall -Wextra. + +if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra-semi -Wcomma -Winconsistent-missing-destructor-override -Wunused-exception-parameter -Wshadow-uncaptured-local") + + if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wredundant-parens -Wzero-as-null-pointer-constant") + endif () +endif () + +if (USE_DEBUG_HELPERS) + set (INCLUDE_DEBUG_HELPERS "-include ${ClickHouse_SOURCE_DIR}/libs/libcommon/include/common/iostream_debug_helpers.h") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${INCLUDE_DEBUG_HELPERS}") +endif () + +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # If we leave this optimization enabled, gcc-7 replaces a pair of SSE intrinsics (16 byte load, store) with a call to memcpy. + # It leads to slow code. This is compiler bug. It looks like this: + # + # (gdb) bt + #0 memcpy (destination=0x7faa6e9f1638, source=0x7faa81d9e9a8, size=16) at ../libs/libmemcpy/memcpy.h:11 + #1 0x0000000005341c5f in _mm_storeu_si128 (__B=..., __P=) at /usr/lib/gcc/x86_64-linux-gnu/7/include/emmintrin.h:720 + #2 memcpySmallAllowReadWriteOverflow15Impl (n=, src=, dst=) at ../dbms/src/Common/memcpySmall.h:37 + + add_definitions ("-fno-tree-loop-distribute-patterns") +endif () add_subdirectory (src) @@ -40,22 +59,19 @@ add_headers_and_sources(clickhouse_common_io src/Common) add_headers_and_sources(clickhouse_common_io src/Common/HashTable) add_headers_and_sources(clickhouse_common_io src/IO) -add_headers_and_sources(dbms src/Analyzers) add_headers_and_sources(dbms src/Core) add_headers_and_sources(dbms src/DataStreams) add_headers_and_sources(dbms src/DataTypes) add_headers_and_sources(dbms src/Databases) -add_headers_and_sources(dbms src/Dictionaries) -add_headers_and_sources(dbms src/Dictionaries/Embedded) -add_headers_and_sources(dbms src/Dictionaries/Embedded/GeodataProviders) add_headers_and_sources(dbms src/Interpreters) add_headers_and_sources(dbms src/Interpreters/ClusterProxy) add_headers_and_sources(dbms src/Columns) add_headers_and_sources(dbms src/Storages) add_headers_and_sources(dbms src/Storages/Distributed) +add_headers_and_sources(dbms src/Storages/Kafka) add_headers_and_sources(dbms src/Storages/MergeTree) add_headers_and_sources(dbms src/Client) -add_headers_only(dbms src/Server) +add_headers_and_sources(dbms src/Formats) list (APPEND clickhouse_common_io_sources ${CONFIG_BUILD}) list (APPEND clickhouse_common_io_headers ${CONFIG_VERSION} ${CONFIG_COMMON}) @@ -67,7 +83,6 @@ list (APPEND dbms_sources src/AggregateFunctions/AggregateFunctionFactory.cpp src/AggregateFunctions/AggregateFunctionCombinatorFactory.cpp src/AggregateFunctions/AggregateFunctionState.cpp - src/AggregateFunctions/FactoryHelpers.cpp src/AggregateFunctions/parseAggregateFunctionParameters.cpp) list (APPEND dbms_headers @@ -82,9 +97,9 @@ list (APPEND dbms_headers list (APPEND dbms_sources src/TableFunctions/ITableFunction.cpp src/TableFunctions/TableFunctionFactory.cpp) list (APPEND dbms_headers src/TableFunctions/ITableFunction.h src/TableFunctions/TableFunctionFactory.h) -add_library(clickhouse_common_io ${SPLIT_SHARED} ${clickhouse_common_io_headers} ${clickhouse_common_io_sources}) +add_library(clickhouse_common_io ${LINK_MODE} ${clickhouse_common_io_headers} ${clickhouse_common_io_sources}) -if (ARCH_FREEBSD) +if (OS_FREEBSD) target_compile_definitions (clickhouse_common_io PUBLIC CLOCK_MONOTONIC_COARSE=CLOCK_MONOTONIC_FAST) endif () @@ -95,10 +110,16 @@ if (MAKE_STATIC_LIBRARIES) add_library(dbms ${dbms_headers} ${dbms_sources}) else () add_library(dbms SHARED ${dbms_headers} ${dbms_sources}) - set_target_properties (dbms PROPERTIES SOVERSION ${VERSION_MAJOR} VERSION ${VERSION_SO} OUTPUT_NAME clickhouse) + set_target_properties (dbms PROPERTIES SOVERSION ${VERSION_MAJOR}.${VERSION_MINOR} VERSION ${VERSION_SO} OUTPUT_NAME clickhouse) install (TARGETS dbms LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT clickhouse) endif () +if (USE_EMBEDDED_COMPILER) + llvm_libs_all(REQUIRED_LLVM_LIBRARIES) + target_link_libraries (dbms PRIVATE ${REQUIRED_LLVM_LIBRARIES}) + target_include_directories (dbms SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS}) +endif () + if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL") # Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size. @@ -121,116 +142,152 @@ if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELW PROPERTIES COMPILE_FLAGS -g0) endif () -if (NOT ARCH_ARM) - set (LINK_LIBRARIES_ONLY_ON_X86_64 cpuid) +if (NOT ARCH_ARM AND CPUID_LIBRARY) + set (LINK_LIBRARIES_ONLY_ON_X86_64 ${CPUID_LIBRARY}) endif() target_link_libraries (clickhouse_common_io + PUBLIC common + PRIVATE string_utils + widechar_width ${LINK_LIBRARIES_ONLY_ON_X86_64} ${LZ4_LIBRARY} ${ZSTD_LIBRARY} ${DOUBLE_CONVERSION_LIBRARIES} + pocoext + PUBLIC ${Poco_Net_LIBRARY} - ${Poco_Data_LIBRARY} + ${Poco_Util_LIBRARY} + ${Poco_Foundation_LIBRARY} + ${RE2_LIBRARY} + ${RE2_ST_LIBRARY} + ${CITYHASH_LIBRARIES} + PRIVATE ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARY} + ${ELF_LIBRARY} + PUBLIC ${Boost_SYSTEM_LIBRARY} + PRIVATE + apple_rt ${CMAKE_DL_LIBS} ) target_link_libraries (dbms + PRIVATE clickhouse_parsers clickhouse_common_config + PUBLIC clickhouse_common_io + PRIVATE + clickhouse_dictionaries + clickhouse_dictionaries_embedded + PUBLIC + pocoext ${MYSQLXX_LIBRARY} - ${FARMHASH_LIBRARIES} - ${METROHASH_LIBRARIES} - ${RE2_LIBRARY} - ${RE2_ST_LIBRARY} - ${OPENSSL_CRYPTO_LIBRARY} + PRIVATE ${BTRIE_LIBRARIES} + ${Boost_PROGRAM_OPTIONS_LIBRARY} + PUBLIC + ${Boost_SYSTEM_LIBRARY} ) if (NOT USE_INTERNAL_RE2_LIBRARY) - target_include_directories (dbms BEFORE PRIVATE ${RE2_INCLUDE_DIR}) + target_include_directories (dbms SYSTEM BEFORE PRIVATE ${RE2_INCLUDE_DIR}) endif () if (NOT USE_INTERNAL_BOOST_LIBRARY) - target_include_directories (clickhouse_common_io BEFORE PUBLIC ${Boost_INCLUDE_DIRS}) + target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS}) endif () -if (Poco_DataODBC_FOUND) - target_link_libraries (dbms ${Poco_DataODBC_LIBRARY}) - target_include_directories (dbms PRIVATE ${ODBC_INCLUDE_DIRECTORIES}) +if (Poco_SQL_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY) + target_include_directories (clickhouse_common_io SYSTEM PRIVATE ${Poco_SQL_INCLUDE_DIR}) + target_include_directories (dbms SYSTEM PRIVATE ${Poco_SQL_INCLUDE_DIR}) endif() -if (Poco_MongoDB_FOUND) - target_link_libraries (dbms ${Poco_MongoDB_LIBRARY}) +if (USE_POCO_SQLODBC) + target_link_libraries (clickhouse_common_io PRIVATE ${Poco_SQL_LIBRARY}) + target_link_libraries (dbms PRIVATE ${Poco_SQLODBC_LIBRARY} ${Poco_SQL_LIBRARY}) + if (NOT USE_INTERNAL_POCO_LIBRARY) + target_include_directories (clickhouse_common_io SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQL_INCLUDE_DIR}) + target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR} PUBLIC ${Poco_SQL_INCLUDE_DIR}) + endif() endif() -if (Poco_NetSSL_FOUND) - target_link_libraries (clickhouse_common_io ${Poco_NetSSL_LIBRARY}) +#if (Poco_Data_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY) +if (Poco_Data_FOUND) + target_include_directories (clickhouse_common_io SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIR}) + target_include_directories (dbms SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIR}) endif() -target_link_libraries (dbms ${Poco_Foundation_LIBRARY}) +if (USE_POCO_DATAODBC) + target_link_libraries (clickhouse_common_io PRIVATE ${Poco_Data_LIBRARY}) + target_link_libraries (dbms PRIVATE ${Poco_DataODBC_LIBRARY}) + if (NOT USE_INTERNAL_POCO_LIBRARY) + target_include_directories (dbms SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR}) + endif() +endif() + +if (USE_POCO_MONGODB) + target_link_libraries (dbms PRIVATE ${Poco_MongoDB_LIBRARY}) +endif() + +if (USE_POCO_NETSSL) + target_link_libraries (clickhouse_common_io PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY}) + target_link_libraries (dbms PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY}) +endif() + +target_link_libraries (dbms PRIVATE ${Poco_Foundation_LIBRARY}) if (USE_ICU) - target_link_libraries (dbms ${ICU_LIBS}) - target_include_directories (dbms PRIVATE ${ICU_INCLUDE_DIR}) + target_link_libraries (dbms PRIVATE ${ICU_LIBRARIES}) endif () if (USE_CAPNP) - target_link_libraries (dbms ${CAPNP_LIBRARY}) + target_link_libraries (dbms PRIVATE ${CAPNP_LIBRARY}) if (NOT USE_INTERNAL_CAPNP_LIBRARY) - target_include_directories (dbms BEFORE PRIVATE ${CAPNP_INCLUDE_DIR}) + target_include_directories (dbms SYSTEM BEFORE PRIVATE ${CAPNP_INCLUDE_DIR}) endif () endif () if (USE_RDKAFKA) - target_link_libraries (dbms ${RDKAFKA_LIBRARY}) + target_link_libraries (dbms PRIVATE ${RDKAFKA_LIBRARY}) if (NOT USE_INTERNAL_RDKAFKA_LIBRARY) - target_include_directories (dbms BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR}) + target_include_directories (dbms SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR}) endif () endif () -target_link_libraries (dbms - Threads::Threads -) +target_link_libraries(dbms PRIVATE ${OPENSSL_CRYPTO_LIBRARY} Threads::Threads) -target_include_directories (dbms BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR}) -target_include_directories (dbms BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) +target_include_directories (dbms SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR}) +target_include_directories (dbms SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) if (NOT USE_INTERNAL_LZ4_LIBRARY) - target_include_directories (dbms BEFORE PRIVATE ${LZ4_INCLUDE_DIR}) + target_include_directories (dbms SYSTEM BEFORE PRIVATE ${LZ4_INCLUDE_DIR}) endif () if (NOT USE_INTERNAL_ZSTD_LIBRARY) - target_include_directories (dbms BEFORE PRIVATE ${ZSTD_INCLUDE_DIR}) + target_include_directories (dbms SYSTEM BEFORE PRIVATE ${ZSTD_INCLUDE_DIR}) +endif () + +if (USE_JEMALLOC) + target_include_directories (dbms SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR}) # used in Interpreters/AsynchronousMetrics.cpp endif () target_include_directories (dbms PUBLIC ${DBMS_INCLUDE_DIR}) target_include_directories (clickhouse_common_io PUBLIC ${DBMS_INCLUDE_DIR}) -target_include_directories (clickhouse_common_io PUBLIC ${PCG_RANDOM_INCLUDE_DIR}) -target_include_directories (clickhouse_common_io BEFORE PUBLIC ${DOUBLE_CONVERSION_INCLUDE_DIR}) +target_include_directories (clickhouse_common_io SYSTEM PUBLIC ${PCG_RANDOM_INCLUDE_DIR}) +target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${DOUBLE_CONVERSION_INCLUDE_DIR}) # also for copy_headers.sh: target_include_directories (clickhouse_common_io BEFORE PRIVATE ${COMMON_INCLUDE_DIR}) +add_subdirectory (programs) add_subdirectory (tests) if (ENABLE_TESTS) - include (${ClickHouse_SOURCE_DIR}/cmake/find_gtest.cmake) - - if (USE_INTERNAL_GTEST_LIBRARY) - # Google Test from sources - add_subdirectory(${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest ${CMAKE_CURRENT_BINARY_DIR}/googletest) - # avoid problems with - target_compile_definitions (gtest INTERFACE GTEST_HAS_POSIX_RE=0) - target_include_directories (gtest INTERFACE ${ClickHouse_SOURCE_DIR}/contrib/googletest/include) - endif () - - macro(grep_gtest_sources BASE_DIR DST_VAR) + macro (grep_gtest_sources BASE_DIR DST_VAR) # Cold match files that are not in tests/ directories file(GLOB_RECURSE "${DST_VAR}" RELATIVE "${BASE_DIR}" "gtest*.cpp") endmacro() @@ -238,6 +295,6 @@ if (ENABLE_TESTS) # attach all dbms gtest sources grep_gtest_sources(${ClickHouse_SOURCE_DIR}/dbms dbms_gtest_sources) add_executable(unit_tests_dbms ${dbms_gtest_sources}) - target_link_libraries(unit_tests_dbms gtest_main dbms) + target_link_libraries(unit_tests_dbms PRIVATE gtest_main dbms clickhouse_common_zookeeper) add_check(unit_tests_dbms) endif () diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index 990c638bf46..9f623279b99 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -1,29 +1,26 @@ # This strings autochanged from release_lib.sh: -set(VERSION_DESCRIBE v1.1.54371-testing) -set(VERSION_REVISION 54371) +set(VERSION_REVISION 54409 CACHE STRING "") +set(VERSION_MAJOR 18 CACHE STRING "") +set(VERSION_MINOR 14 CACHE STRING "") +set(VERSION_PATCH 9 CACHE STRING "") +set(VERSION_GITHASH 457f8fd495b2812940e69c15ab5b499cd863aae4 CACHE STRING "") +set(VERSION_DESCRIBE v18.14.9-testing CACHE STRING "") +set(VERSION_STRING 18.14.9 CACHE STRING "") # end of autochange -set (VERSION_MAJOR 1) -set (VERSION_MINOR 1) -set (VERSION_PATCH ${VERSION_REVISION}) -set (VERSION_EXTRA "") -set (VERSION_TWEAK "") +set(VERSION_EXTRA "" CACHE STRING "") +set(VERSION_TWEAK "" CACHE STRING "") -set (VERSION_STRING "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}") if (VERSION_TWEAK) - set(VERSION_STRING "${VERSION_STRING}.${VERSION_TWEAK}") + string(CONCAT VERSION_STRING ${VERSION_STRING} "." ${VERSION_TWEAK}) endif () + if (VERSION_EXTRA) - set(VERSION_STRING "${VERSION_STRING}${VERSION_EXTRA}") + string(CONCAT VERSION_STRING ${VERSION_STRING} "." ${VERSION_EXTRA}) endif () -set (VERSION_FULL "${PROJECT_NAME} ${VERSION_STRING}") +set (VERSION_NAME "${PROJECT_NAME}" CACHE STRING "") +set (VERSION_FULL "${VERSION_NAME} ${VERSION_STRING}" CACHE STRING "") +set (VERSION_SO "${VERSION_STRING}" CACHE STRING "") -if (APPLE) - # dirty hack: ld: malformed 64-bit a.b.c.d.e version number: 1.1.54160 - math (EXPR VERSION_SO1 "${VERSION_REVISION}/255") - math (EXPR VERSION_SO2 "${VERSION_REVISION}%255") - set (VERSION_SO "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_SO1}.${VERSION_SO2}") -else () - set (VERSION_SO "${VERSION_STRING}") -endif () +math (EXPR VERSION_INTEGER "${VERSION_PATCH} + ${VERSION_MINOR}*1000 + ${VERSION_MAJOR}*1000000") diff --git a/dbms/programs/CMakeLists.txt b/dbms/programs/CMakeLists.txt new file mode 100644 index 00000000000..9d7c6f2cda1 --- /dev/null +++ b/dbms/programs/CMakeLists.txt @@ -0,0 +1,168 @@ +# 'clickhouse' binary is a multi purpose tool, +# that contain multiple execution modes (client, server, etc.) +# each of them is built and linked as a separate library, defined below. + +option (ENABLE_CLICKHOUSE_ALL "Enable all tools" ON) +option (ENABLE_CLICKHOUSE_SERVER "Enable clickhouse-server" ${ENABLE_CLICKHOUSE_ALL}) +option (ENABLE_CLICKHOUSE_CLIENT "Enable clickhouse-client" ${ENABLE_CLICKHOUSE_ALL}) +option (ENABLE_CLICKHOUSE_LOCAL "Enable clickhouse-local" ${ENABLE_CLICKHOUSE_ALL}) +option (ENABLE_CLICKHOUSE_BENCHMARK "Enable clickhouse-benchmark" ${ENABLE_CLICKHOUSE_ALL}) +option (ENABLE_CLICKHOUSE_PERFORMANCE "Enable clickhouse-performance-test" ${ENABLE_CLICKHOUSE_ALL}) +option (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG "Enable clickhouse-extract-from-config" ${ENABLE_CLICKHOUSE_ALL}) +option (ENABLE_CLICKHOUSE_COMPRESSOR "Enable clickhouse-compressor" ${ENABLE_CLICKHOUSE_ALL}) +option (ENABLE_CLICKHOUSE_COPIER "Enable clickhouse-copier" ${ENABLE_CLICKHOUSE_ALL}) +option (ENABLE_CLICKHOUSE_FORMAT "Enable clickhouse-format" ${ENABLE_CLICKHOUSE_ALL}) +option (ENABLE_CLICKHOUSE_OBFUSCATOR "Enable clickhouse-obfuscator" ${ENABLE_CLICKHOUSE_ALL}) +option (ENABLE_CLICKHOUSE_ODBC_BRIDGE "Enable clickhouse-odbc-bridge" ${ENABLE_CLICKHOUSE_ALL}) + +configure_file (config_tools.h.in ${CMAKE_CURRENT_BINARY_DIR}/config_tools.h) + +add_subdirectory (server) +add_subdirectory (client) +add_subdirectory (local) +add_subdirectory (benchmark) +add_subdirectory (performance-test) +add_subdirectory (extract-from-config) +add_subdirectory (compressor) +add_subdirectory (copier) +add_subdirectory (format) +add_subdirectory (clang) +add_subdirectory (obfuscator) +add_subdirectory (odbc-bridge) + +if (CLICKHOUSE_SPLIT_BINARY) + set (CLICKHOUSE_ALL_TARGETS clickhouse-server clickhouse-client clickhouse-local clickhouse-benchmark clickhouse-performance-test + clickhouse-extract-from-config clickhouse-compressor clickhouse-format clickhouse-copier clickhouse-odbc-bridge) + + if (USE_EMBEDDED_COMPILER) + list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-clang clickhouse-lld) + endif () + + set_target_properties(${CLICKHOUSE_ALL_TARGETS} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) + + add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_ALL_TARGETS}) + add_custom_target (clickhouse ALL DEPENDS clickhouse-bundle) +else () + if (USE_EMBEDDED_COMPILER) + # before add_executable ! + link_directories (${LLVM_LIBRARY_DIRS}) + endif () + add_executable (clickhouse main.cpp) + target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils) + target_include_directories (clickhouse BEFORE PRIVATE ${COMMON_INCLUDE_DIR}) + target_include_directories (clickhouse PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) + + if (ENABLE_CLICKHOUSE_SERVER) + target_link_libraries (clickhouse PRIVATE clickhouse-server-lib) + endif () + if (ENABLE_CLICKHOUSE_CLIENT) + target_link_libraries (clickhouse PRIVATE clickhouse-client-lib) + endif () + if (ENABLE_CLICKHOUSE_LOCAL) + target_link_libraries (clickhouse PRIVATE clickhouse-local-lib) + endif () + if (ENABLE_CLICKHOUSE_BENCHMARK) + target_link_libraries (clickhouse PRIVATE clickhouse-benchmark-lib) + endif () + if (ENABLE_CLICKHOUSE_PERFORMANCE) + target_link_libraries (clickhouse PRIVATE clickhouse-performance-test-lib) + endif () + if (ENABLE_CLICKHOUSE_COPIER) + target_link_libraries (clickhouse PRIVATE clickhouse-copier-lib) + endif () + if (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG) + target_link_libraries (clickhouse PRIVATE clickhouse-extract-from-config-lib) + endif () + if (ENABLE_CLICKHOUSE_COMPRESSOR) + target_link_libraries (clickhouse PRIVATE clickhouse-compressor-lib) + endif () + if (ENABLE_CLICKHOUSE_FORMAT) + target_link_libraries (clickhouse PRIVATE clickhouse-format-lib) + endif () + if (ENABLE_CLICKHOUSE_OBFUSCATOR) + target_link_libraries (clickhouse PRIVATE clickhouse-obfuscator-lib) + endif () + if (USE_EMBEDDED_COMPILER) + target_link_libraries (clickhouse PRIVATE clickhouse-compiler-lib) + endif () + if (ENABLE_CLICKHOUSE_ODBC_BRIDGE) + target_link_libraries (clickhouse PRIVATE clickhouse-odbc-bridge-lib) + endif() + + set (CLICKHOUSE_BUNDLE) + if (ENABLE_CLICKHOUSE_SERVER) + add_custom_target (clickhouse-server ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-server DEPENDS clickhouse) + install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-server DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-server) + endif () + if (ENABLE_CLICKHOUSE_CLIENT) + add_custom_target (clickhouse-client ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-client DEPENDS clickhouse) + install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-client DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-client) + endif () + if (ENABLE_CLICKHOUSE_LOCAL) + add_custom_target (clickhouse-local ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-local DEPENDS clickhouse) + install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-local DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-local) + endif () + if (ENABLE_CLICKHOUSE_BENCHMARK) + add_custom_target (clickhouse-benchmark ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-benchmark DEPENDS clickhouse) + install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-benchmark DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-benchmark) + endif () + if (ENABLE_CLICKHOUSE_PERFORMANCE) + add_custom_target (clickhouse-performance-test ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-performance-test DEPENDS clickhouse) + install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-performance-test DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-performance-test) + endif () + if (ENABLE_CLICKHOUSE_COPIER) + add_custom_target (clickhouse-copier ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-copier DEPENDS clickhouse) + install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-copier DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-copier) + endif () + if (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG) + add_custom_target (clickhouse-extract-from-config ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-extract-from-config DEPENDS clickhouse) + install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-extract-from-config DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-extract-from-config) + endif () + if (ENABLE_CLICKHOUSE_COMPRESSOR) + add_custom_target (clickhouse-compressor ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-compressor DEPENDS clickhouse) + install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-compressor DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-compressor) + endif () + if (ENABLE_CLICKHOUSE_FORMAT) + add_custom_target (clickhouse-format ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-format DEPENDS clickhouse) + install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-format DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-format) + endif () + if (ENABLE_CLICKHOUSE_COPIER) + add_custom_target (clickhouse-obfuscator ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-obfuscator DEPENDS clickhouse) + install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-obfuscator DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-obfuscator) + endif () + if (ENABLE_CLICKHOUSE_ODBC_BRIDGE) + add_custom_target (clickhouse-odbc-bridge ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-odbc-bridge DEPENDS clickhouse) + install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-odbc-bridge DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-odbc-bridge) + endif () + + + # install always because depian package want this files: + add_custom_target (clickhouse-clang ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-clang DEPENDS clickhouse) + add_custom_target (clickhouse-lld ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-lld DEPENDS clickhouse) + list(APPEND CLICKHOUSE_BUNDLE clickhouse-clang clickhouse-lld) + + install (TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + + install (FILES + ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-clang + ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-lld + DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + + add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_BUNDLE}) + +endif () + +if (TARGET clickhouse-server AND TARGET copy-headers) + add_dependencies(clickhouse-server copy-headers) +endif () diff --git a/dbms/src/Server/Benchmark.cpp b/dbms/programs/benchmark/Benchmark.cpp similarity index 82% rename from dbms/src/Server/Benchmark.cpp rename to dbms/programs/benchmark/Benchmark.cpp index ee4d2dd8802..07c65e27f9a 100644 --- a/dbms/src/Server/Benchmark.cpp +++ b/dbms/programs/benchmark/Benchmark.cpp @@ -26,10 +26,12 @@ #include #include #include +#include #include #include #include -#include "InterruptListener.h" +#include +#include /** A tool for evaluating ClickHouse performance. @@ -41,23 +43,21 @@ namespace DB namespace ErrorCodes { - extern const int POCO_EXCEPTION; - extern const int STD_EXCEPTION; - extern const int UNKNOWN_EXCEPTION; extern const int BAD_ARGUMENTS; + extern const int EMPTY_DATA_PASSED; } -class Benchmark +class Benchmark : public Poco::Util::Application { public: Benchmark(unsigned concurrency_, double delay_, - const String & host_, UInt16 port_, const String & default_database_, + const String & host_, UInt16 port_, bool secure_, const String & default_database_, const String & user_, const String & password_, const String & stage, bool randomize_, size_t max_iterations_, double max_time_, const String & json_path_, const ConnectionTimeouts & timeouts, const Settings & settings_) : concurrency(concurrency_), delay(delay_), queue(concurrency), - connections(concurrency, host_, port_, default_database_, user_, password_, timeouts), + connections(concurrency, host_, port_, default_database_, user_, password_, timeouts, "benchmark", Protocol::Compression::Enable, secure_ ? Protocol::Secure::Enable : Protocol::Secure::Disable), randomize(randomize_), max_iterations(max_iterations_), max_time(max_time_), json_path(json_path_), settings(settings_), global_context(Context::createGlobal()), pool(concurrency) { @@ -76,13 +76,26 @@ public: else throw Exception("Unknown query processing stage: " + stage, ErrorCodes::BAD_ARGUMENTS); + } + + void initialize(Poco::Util::Application & self [[maybe_unused]]) + { + std::string home_path; + const char * home_path_cstr = getenv("HOME"); + if (home_path_cstr) + home_path = home_path_cstr; + + configReadClient(config(), home_path); + } + + int main(const std::vector &) + { if (!json_path.empty() && Poco::File(json_path).exists()) /// Clear file with previous results - { Poco::File(json_path).remove(); - } readQueries(); - run(); + runBenchmark(); + return 0; } private: @@ -109,6 +122,8 @@ private: /// Don't execute new queries after timelimit or SIGINT or exception std::atomic shutdown{false}; + std::atomic queries_executed{0}; + struct Stats { Stopwatch watch; @@ -167,7 +182,7 @@ private: } if (queries.empty()) - throw Exception("Empty list of queries."); + throw Exception("Empty list of queries.", ErrorCodes::EMPTY_DATA_PASSED); std::cerr << "Loaded " << queries.size() << " queries.\n"; } @@ -219,7 +234,7 @@ private: return true; } - void run() + void runBenchmark() { pcg64 generator(randomSeed()); std::uniform_int_distribution distribution(0, queries.size() - 1); @@ -237,10 +252,12 @@ private: size_t query_index = randomize ? distribution(generator) : i % queries.size(); if (!tryPushQueryInteractively(queries[query_index], interrupt_listener)) + { + shutdown = true; break; + } } - shutdown = true; pool.wait(); info_total.watch.stop(); @@ -273,11 +290,12 @@ private: { extracted = queue.tryPop(query, 100); - if (shutdown) + if (shutdown || (max_iterations && queries_executed == max_iterations)) return; } execute(connection, query); + ++queries_executed; } } catch (...) @@ -403,6 +421,10 @@ public: } +#ifndef __clang__ +#pragma GCC optimize("-fno-var-tracking-assignments") +#endif + int mainEntryClickHouseBenchmark(int argc, char ** argv) { using namespace DB; @@ -414,20 +436,21 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) boost::program_options::options_description desc("Allowed options"); desc.add_options() - ("help", "produce help message") - ("concurrency,c", value()->default_value(1), "number of parallel queries") - ("delay,d", value()->default_value(1), "delay between intermediate reports in seconds (set 0 to disable reports)") - ("stage", value()->default_value("complete"), "request query processing up to specified stage") - ("iterations,i", value()->default_value(0), "amount of queries to be executed") - ("timelimit,t", value()->default_value(0.), "stop launch of queries after specified time limit") - ("randomize,r", value()->default_value(false), "randomize order of execution") - ("json", value()->default_value(""), "write final report to specified file in JSON format") - ("host,h", value()->default_value("localhost"), "") - ("port", value()->default_value(9000), "") - ("user", value()->default_value("default"), "") - ("password", value()->default_value(""), "") - ("database", value()->default_value("default"), "") - ("stacktrace", "print stack traces of exceptions") + ("help", "produce help message") + ("concurrency,c", value()->default_value(1), "number of parallel queries") + ("delay,d", value()->default_value(1), "delay between intermediate reports in seconds (set 0 to disable reports)") + ("stage", value()->default_value("complete"), "request query processing up to specified stage") + ("iterations,i", value()->default_value(0), "amount of queries to be executed") + ("timelimit,t", value()->default_value(0.), "stop launch of queries after specified time limit") + ("randomize,r", value()->default_value(false), "randomize order of execution") + ("json", value()->default_value(""), "write final report to specified file in JSON format") + ("host,h", value()->default_value("localhost"), "") + ("port", value()->default_value(9000), "") + ("secure,s", "Use TLS connection") + ("user", value()->default_value("default"), "") + ("password", value()->default_value(""), "") + ("database", value()->default_value("default"), "") + ("stacktrace", "print stack traces of exceptions") #define DECLARE_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) (#NAME, boost::program_options::value (), DESCRIPTION) APPLY_FOR_SETTINGS(DECLARE_SETTING) @@ -455,11 +478,14 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) APPLY_FOR_SETTINGS(EXTRACT_SETTING) #undef EXTRACT_SETTING + UseSSL use_ssl; + Benchmark benchmark( options["concurrency"].as(), options["delay"].as(), options["host"].as(), options["port"].as(), + options.count("secure"), options["database"].as(), options["user"].as(), options["password"].as(), @@ -470,6 +496,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) options["json"].as(), ConnectionTimeouts::getTCPTimeoutsWithoutFailover(settings), settings); + return benchmark.run(); } catch (...) { diff --git a/dbms/programs/benchmark/CMakeLists.txt b/dbms/programs/benchmark/CMakeLists.txt new file mode 100644 index 00000000000..af11c600b2d --- /dev/null +++ b/dbms/programs/benchmark/CMakeLists.txt @@ -0,0 +1,8 @@ +add_library (clickhouse-benchmark-lib ${LINK_MODE} Benchmark.cpp) +target_link_libraries (clickhouse-benchmark-lib PRIVATE clickhouse_aggregate_functions clickhouse-client-lib clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) +target_include_directories (clickhouse-benchmark-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR}) + +if (CLICKHOUSE_SPLIT_BINARY) + add_executable (clickhouse-benchmark clickhouse-benchmark.cpp) + target_link_libraries (clickhouse-benchmark PRIVATE clickhouse-benchmark-lib clickhouse_aggregate_functions) +endif () diff --git a/dbms/src/Server/clickhouse-benchmark.cpp b/dbms/programs/benchmark/clickhouse-benchmark.cpp similarity index 100% rename from dbms/src/Server/clickhouse-benchmark.cpp rename to dbms/programs/benchmark/clickhouse-benchmark.cpp diff --git a/dbms/programs/clang/CMakeLists.txt b/dbms/programs/clang/CMakeLists.txt new file mode 100644 index 00000000000..dec21ac611e --- /dev/null +++ b/dbms/programs/clang/CMakeLists.txt @@ -0,0 +1,38 @@ +if (USE_EMBEDDED_COMPILER) + add_subdirectory ("Compiler-${LLVM_VERSION}") +endif () + +if (CLICKHOUSE_SPLIT_BINARY) + if (USE_EMBEDDED_COMPILER) + link_directories (${LLVM_LIBRARY_DIRS}) + add_executable (clickhouse-clang clickhouse-clang.cpp) + target_link_libraries (clickhouse-clang PRIVATE clickhouse-compiler-lib) + add_executable (clickhouse-lld clickhouse-lld.cpp) + target_link_libraries (clickhouse-lld PRIVATE clickhouse-compiler-lib) + install (TARGETS clickhouse-clang clickhouse-lld RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) + endif () +endif () + +set (TMP_HEADERS_DIR "${CMAKE_CURRENT_BINARY_DIR}/${INTERNAL_COMPILER_HEADERS_RELATIVE}") +# Make and install empty dir for debian package if compiler disabled +add_custom_target (make-headers-directory ALL COMMAND ${CMAKE_COMMAND} -E make_directory ${TMP_HEADERS_DIR}) +install (DIRECTORY ${TMP_HEADERS_DIR} DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/clickhouse/${INTERNAL_COMPILER_HEADERS_DIR} COMPONENT clickhouse) +# TODO: fix on macos copy_headers.sh: sed --posix + +if (USE_EMBEDDED_COMPILER) + set (COPY_HEADERS_COMPILER "${CMAKE_CURRENT_BINARY_DIR}/../${INTERNAL_COMPILER_EXECUTABLE}") + set (COPY_HEADERS_DEPENDS clickhouse-clang) +elseif (EXISTS ${INTERNAL_COMPILER_BIN_ROOT}${INTERNAL_COMPILER_EXECUTABLE}) + set (COPY_HEADERS_COMPILER "${INTERNAL_COMPILER_BIN_ROOT}${INTERNAL_COMPILER_EXECUTABLE}") +endif () + +if (COPY_HEADERS_COMPILER AND OS_LINUX) + add_custom_target (copy-headers ALL env CLANG=${COPY_HEADERS_COMPILER} BUILD_PATH=${ClickHouse_BINARY_DIR} DESTDIR=${ClickHouse_SOURCE_DIR} ${ClickHouse_SOURCE_DIR}/copy_headers.sh ${ClickHouse_SOURCE_DIR} ${TMP_HEADERS_DIR} DEPENDS ${COPY_HEADERS_DEPENDS} WORKING_DIRECTORY ${ClickHouse_SOURCE_DIR} SOURCES ${ClickHouse_SOURCE_DIR}/copy_headers.sh) + + if (USE_INTERNAL_LLVM_LIBRARY) + set (CLANG_HEADERS_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm/clang/lib/Headers") + set (CLANG_HEADERS_DEST "${TMP_HEADERS_DIR}/usr/local/lib/clang/${LLVM_VERSION}/include") # original: ${LLVM_LIBRARY_OUTPUT_INTDIR}/clang/${CLANG_VERSION}/include + add_custom_target (copy-headers-clang ALL ${CMAKE_COMMAND} -E make_directory ${CLANG_HEADERS_DEST} && ${CMAKE_COMMAND} -E copy_if_different ${CLANG_HEADERS_DIR}/* ${CLANG_HEADERS_DEST} ) + add_dependencies (copy-headers copy-headers-clang) + endif () +endif () diff --git a/dbms/src/Server/Compiler-5.0.0/CMakeLists.txt b/dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt similarity index 65% rename from dbms/src/Server/Compiler-5.0.0/CMakeLists.txt rename to dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt index 739db1cf448..7fe0cd92ef7 100644 --- a/dbms/src/Server/Compiler-5.0.0/CMakeLists.txt +++ b/dbms/programs/clang/Compiler-5.0.0/CMakeLists.txt @@ -1,6 +1,8 @@ -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error") +add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG) -add_library(clickhouse-compiler-lib +link_directories(${LLVM_LIBRARY_DIRS}) + +add_library(clickhouse-compiler-lib ${LINK_MODE} driver.cpp cc1_main.cpp cc1as_main.cpp @@ -8,16 +10,13 @@ add_library(clickhouse-compiler-lib target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0) -llvm_map_components_to_libraries(REQUIRED_LLVM_LIBRARIES all) +string(REPLACE "${INCLUDE_DEBUG_HELPERS}" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) # cant compile with -fno-rtti -# We link statically with zlib, and LLVM (sometimes) tries to bring its own dependency. -list(REMOVE_ITEM REQUIRED_LLVM_LIBRARIES "-lz") -# Wrong library in freebsd: -list(REMOVE_ITEM REQUIRED_LLVM_LIBRARIES "-l/usr/lib/libexecinfo.so") +llvm_libs_all(REQUIRED_LLVM_LIBRARIES) message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}") -target_include_directories(clickhouse-compiler-lib PRIVATE ${LLVM_INCLUDE_DIRS}) +target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) # This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory. @@ -47,7 +46,8 @@ LLVMSupport #PollyISL #PollyPPCG -libtinfo.a - PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARY} Threads::Threads +${MALLOC_LIBRARIES} +${GLIBC_COMPATIBILITY_LIBRARIES} +${MEMCPY_LIBRARIES} ) diff --git a/dbms/src/Server/Compiler-5.0.0/LICENSE.TXT b/dbms/programs/clang/Compiler-5.0.0/LICENSE.TXT similarity index 100% rename from dbms/src/Server/Compiler-5.0.0/LICENSE.TXT rename to dbms/programs/clang/Compiler-5.0.0/LICENSE.TXT diff --git a/dbms/src/Server/Compiler-5.0.0/cc1_main.cpp b/dbms/programs/clang/Compiler-5.0.0/cc1_main.cpp similarity index 100% rename from dbms/src/Server/Compiler-5.0.0/cc1_main.cpp rename to dbms/programs/clang/Compiler-5.0.0/cc1_main.cpp diff --git a/dbms/src/Server/Compiler-5.0.0/cc1as_main.cpp b/dbms/programs/clang/Compiler-5.0.0/cc1as_main.cpp similarity index 100% rename from dbms/src/Server/Compiler-5.0.0/cc1as_main.cpp rename to dbms/programs/clang/Compiler-5.0.0/cc1as_main.cpp diff --git a/dbms/src/Server/Compiler-5.0.0/driver.cpp b/dbms/programs/clang/Compiler-5.0.0/driver.cpp similarity index 100% rename from dbms/src/Server/Compiler-5.0.0/driver.cpp rename to dbms/programs/clang/Compiler-5.0.0/driver.cpp diff --git a/dbms/programs/clang/Compiler-5.0.0/lld.cpp b/dbms/programs/clang/Compiler-5.0.0/lld.cpp new file mode 100644 index 00000000000..5af29868864 --- /dev/null +++ b/dbms/programs/clang/Compiler-5.0.0/lld.cpp @@ -0,0 +1,23 @@ +#include "lld/Driver/Driver.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/PrettyStackTrace.h" +#include "llvm/Support/Signals.h" + +using namespace lld; +using namespace llvm; +using namespace llvm::sys; + +int mainEntryClickHouseLLD(int Argc, char **Argv) +{ + // Standard set up, so program fails gracefully. + sys::PrintStackTraceOnErrorSignal(Argv[0]); + PrettyStackTraceProgram StackPrinter(Argc, Argv); + llvm_shutdown_obj Shutdown; + + std::vector Args(Argv, Argv + Argc); + return !elf::link(Args, true); +} diff --git a/dbms/src/Server/Compiler-5.0.1 b/dbms/programs/clang/Compiler-5.0.1 similarity index 100% rename from dbms/src/Server/Compiler-5.0.1 rename to dbms/programs/clang/Compiler-5.0.1 diff --git a/dbms/programs/clang/Compiler-5.0.2 b/dbms/programs/clang/Compiler-5.0.2 new file mode 120000 index 00000000000..7c8af57399f --- /dev/null +++ b/dbms/programs/clang/Compiler-5.0.2 @@ -0,0 +1 @@ +Compiler-5.0.0 \ No newline at end of file diff --git a/dbms/src/Server/Compiler-6.0.0/CMakeLists.txt b/dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt similarity index 64% rename from dbms/src/Server/Compiler-6.0.0/CMakeLists.txt rename to dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt index 95db6a7e1d1..b96bdb0647a 100644 --- a/dbms/src/Server/Compiler-6.0.0/CMakeLists.txt +++ b/dbms/programs/clang/Compiler-6.0.0/CMakeLists.txt @@ -1,6 +1,8 @@ -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error") +add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG) -add_library(clickhouse-compiler-lib +link_directories(${LLVM_LIBRARY_DIRS}) + +add_library(clickhouse-compiler-lib ${LINK_MODE} driver.cpp cc1_main.cpp cc1as_main.cpp @@ -8,23 +10,20 @@ add_library(clickhouse-compiler-lib target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0) -llvm_map_components_to_libraries(REQUIRED_LLVM_LIBRARIES all) +string(REPLACE "${INCLUDE_DEBUG_HELPERS}" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) # cant compile with -fno-rtti -# We link statically with zlib, and LLVM (sometimes) tries to bring its own dependency. -list(REMOVE_ITEM REQUIRED_LLVM_LIBRARIES "-lz") -# Wrong library in freebsd: -list(REMOVE_ITEM REQUIRED_LLVM_LIBRARIES "-l/usr/lib/libexecinfo.so") +llvm_libs_all(REQUIRED_LLVM_LIBRARIES) message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}") -target_include_directories(clickhouse-compiler-lib PRIVATE ${LLVM_INCLUDE_DIRS}) +target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) # This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory. target_link_libraries(clickhouse-compiler-lib PRIVATE clangBasic clangCodeGen clangDriver -clangFrontend +clangFrontend clangFrontendTool clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend clangParse clangSerialization clangSema clangEdit clangStaticAnalyzerCheckers @@ -47,7 +46,8 @@ ${REQUIRED_LLVM_LIBRARIES} #PollyISL #PollyPPCG -libtinfo.a - PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARY} Threads::Threads +${MALLOC_LIBRARIES} +${GLIBC_COMPATIBILITY_LIBRARIES} +${MEMCPY_LIBRARIES} ) diff --git a/dbms/src/Server/Compiler-6.0.0/LICENSE.TXT b/dbms/programs/clang/Compiler-6.0.0/LICENSE.TXT similarity index 100% rename from dbms/src/Server/Compiler-6.0.0/LICENSE.TXT rename to dbms/programs/clang/Compiler-6.0.0/LICENSE.TXT diff --git a/dbms/src/Server/Compiler-6.0.0/cc1_main.cpp b/dbms/programs/clang/Compiler-6.0.0/cc1_main.cpp similarity index 100% rename from dbms/src/Server/Compiler-6.0.0/cc1_main.cpp rename to dbms/programs/clang/Compiler-6.0.0/cc1_main.cpp diff --git a/dbms/src/Server/Compiler-6.0.0/cc1as_main.cpp b/dbms/programs/clang/Compiler-6.0.0/cc1as_main.cpp similarity index 99% rename from dbms/src/Server/Compiler-6.0.0/cc1as_main.cpp rename to dbms/programs/clang/Compiler-6.0.0/cc1as_main.cpp index 191c7df7ab1..caf8409054a 100644 --- a/dbms/src/Server/Compiler-6.0.0/cc1as_main.cpp +++ b/dbms/programs/clang/Compiler-6.0.0/cc1as_main.cpp @@ -441,7 +441,7 @@ static bool ExecuteAssembler(AssemblerInvocation &Opts, auto Pair = StringRef(S).split('='); auto Sym = Pair.first; auto Val = Pair.second; - int64_t Value; + int64_t Value = 0; // We have already error checked this in the driver. Val.getAsInteger(0, Value); Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value); diff --git a/dbms/src/Server/Compiler-6.0.0/driver.cpp b/dbms/programs/clang/Compiler-6.0.0/driver.cpp similarity index 100% rename from dbms/src/Server/Compiler-6.0.0/driver.cpp rename to dbms/programs/clang/Compiler-6.0.0/driver.cpp diff --git a/dbms/programs/clang/Compiler-6.0.0/lld.cpp b/dbms/programs/clang/Compiler-6.0.0/lld.cpp new file mode 100644 index 00000000000..696ff84dfe6 --- /dev/null +++ b/dbms/programs/clang/Compiler-6.0.0/lld.cpp @@ -0,0 +1,23 @@ +#include "lld/Common/Driver.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/PrettyStackTrace.h" +#include "llvm/Support/Signals.h" + +using namespace lld; +using namespace llvm; +using namespace llvm::sys; + +int mainEntryClickHouseLLD(int Argc, char **Argv) +{ + // Standard set up, so program fails gracefully. + sys::PrintStackTraceOnErrorSignal(Argv[0]); + PrettyStackTraceProgram StackPrinter(Argc, Argv); + llvm_shutdown_obj Shutdown; + + std::vector Args(Argv, Argv + Argc); + return !elf::link(Args, true); +} diff --git a/dbms/src/Server/Compiler-6.0.0svn b/dbms/programs/clang/Compiler-6.0.0svn similarity index 100% rename from dbms/src/Server/Compiler-6.0.0svn rename to dbms/programs/clang/Compiler-6.0.0svn diff --git a/dbms/src/Server/Compiler-7.0.0 b/dbms/programs/clang/Compiler-6.0.1 similarity index 100% rename from dbms/src/Server/Compiler-7.0.0 rename to dbms/programs/clang/Compiler-6.0.1 diff --git a/dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt b/dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt new file mode 100644 index 00000000000..8b6ba6be994 --- /dev/null +++ b/dbms/programs/clang/Compiler-7.0.0/CMakeLists.txt @@ -0,0 +1,49 @@ +add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG) + +link_directories(${LLVM_LIBRARY_DIRS}) + +add_library(clickhouse-compiler-lib ${LINK_MODE} + driver.cpp + cc1_main.cpp + cc1gen_reproducer_main.cpp + cc1as_main.cpp + lld.cpp) + +target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0) + +string(REPLACE "${INCLUDE_DEBUG_HELPERS}" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) # cant compile with -fno-rtti + +llvm_libs_all(REQUIRED_LLVM_LIBRARIES) + +message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}") + +target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) + +# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory. + +target_link_libraries(clickhouse-compiler-lib PRIVATE +clangBasic clangCodeGen clangDriver +clangFrontend +clangFrontendTool +clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend +clangParse clangSerialization clangSema clangEdit clangStaticAnalyzerCheckers +clangASTMatchers clangStaticAnalyzerCore clangAnalysis clangAST clangRewrite clangLex clangBasic +clangCrossTU clangIndex + +lldCOFF +lldDriver +lldELF +lldMinGW +lldMachO +lldReaderWriter +lldYAML +lldCommon +lldCore + +${REQUIRED_LLVM_LIBRARIES} + +PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARY} Threads::Threads +${MALLOC_LIBRARIES} +${GLIBC_COMPATIBILITY_LIBRARIES} +${MEMCPY_LIBRARIES} +) diff --git a/dbms/programs/clang/Compiler-7.0.0/cc1_main.cpp b/dbms/programs/clang/Compiler-7.0.0/cc1_main.cpp new file mode 100644 index 00000000000..214bfa72476 --- /dev/null +++ b/dbms/programs/clang/Compiler-7.0.0/cc1_main.cpp @@ -0,0 +1,239 @@ +//===-- cc1_main.cpp - Clang CC1 Compiler Frontend ------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This is the entry point to the clang -cc1 functionality, which implements the +// core compiler functionality along with a number of additional tools for +// demonstration and testing purposes. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Option/Arg.h" +#include "clang/CodeGen/ObjectFilePCHContainerOperations.h" +#include "clang/Config/config.h" +#include "clang/Basic/Stack.h" +#include "clang/Driver/DriverDiagnostic.h" +#include "clang/Driver/Options.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/CompilerInvocation.h" +#include "clang/Frontend/FrontendDiagnostic.h" +#include "clang/Frontend/TextDiagnosticBuffer.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" +#include "clang/Frontend/Utils.h" +#include "clang/FrontendTool/Utils.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/Config/llvm-config.h" +#include "llvm/LinkAllPasses.h" +#include "llvm/Option/ArgList.h" +#include "llvm/Option/OptTable.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/Signals.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/Timer.h" +#include "llvm/Support/raw_ostream.h" +#include + +#ifdef CLANG_HAVE_RLIMITS +#include +#endif + +// have no .a version in packages +#undef LINK_POLLY_INTO_TOOLS + +using namespace clang; +using namespace llvm::opt; + +//===----------------------------------------------------------------------===// +// Main driver +//===----------------------------------------------------------------------===// + +static void LLVMErrorHandler(void *UserData, const std::string &Message, + bool GenCrashDiag) { + DiagnosticsEngine &Diags = *static_cast(UserData); + + Diags.Report(diag::err_fe_error_backend) << Message; + + // Run the interrupt handlers to make sure any special cleanups get done, in + // particular that we remove files registered with RemoveFileOnSignal. + llvm::sys::RunInterruptHandlers(); + + // We cannot recover from llvm errors. When reporting a fatal error, exit + // with status 70 to generate crash diagnostics. For BSD systems this is + // defined as an internal software error. Otherwise, exit with status 1. + exit(GenCrashDiag ? 70 : 1); +} + +#ifdef LINK_POLLY_INTO_TOOLS +namespace polly { +void initializePollyPasses(llvm::PassRegistry &Registry); +} +#endif + +#ifdef CLANG_HAVE_RLIMITS +#if defined(__linux__) && defined(__PIE__) +static size_t getCurrentStackAllocation() { + // If we can't compute the current stack usage, allow for 512K of command + // line arguments and environment. + size_t Usage = 512 * 1024; + if (FILE *StatFile = fopen("/proc/self/stat", "r")) { + // We assume that the stack extends from its current address to the end of + // the environment space. In reality, there is another string literal (the + // program name) after the environment, but this is close enough (we only + // need to be within 100K or so). + unsigned long StackPtr, EnvEnd; + // Disable silly GCC -Wformat warning that complains about length + // modifiers on ignored format specifiers. We want to retain these + // for documentation purposes even though they have no effect. +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat" +#endif + if (fscanf(StatFile, + "%*d %*s %*c %*d %*d %*d %*d %*d %*u %*lu %*lu %*lu %*lu %*lu " + "%*lu %*ld %*ld %*ld %*ld %*ld %*ld %*llu %*lu %*ld %*lu %*lu " + "%*lu %*lu %lu %*lu %*lu %*lu %*lu %*lu %*llu %*lu %*lu %*d %*d " + "%*u %*u %*llu %*lu %*ld %*lu %*lu %*lu %*lu %*lu %*lu %lu %*d", + &StackPtr, &EnvEnd) == 2) { +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif + Usage = StackPtr < EnvEnd ? EnvEnd - StackPtr : StackPtr - EnvEnd; + } + fclose(StatFile); + } + return Usage; +} + +#include + +LLVM_ATTRIBUTE_NOINLINE +static void ensureStackAddressSpace() { + // Linux kernels prior to 4.1 will sometimes locate the heap of a PIE binary + // relatively close to the stack (they are only guaranteed to be 128MiB + // apart). This results in crashes if we happen to heap-allocate more than + // 128MiB before we reach our stack high-water mark. + // + // To avoid these crashes, ensure that we have sufficient virtual memory + // pages allocated before we start running. + size_t Curr = getCurrentStackAllocation(); + const int kTargetStack = DesiredStackSize - 256 * 1024; + if (Curr < kTargetStack) { + volatile char *volatile Alloc = + static_cast(alloca(kTargetStack - Curr)); + Alloc[0] = 0; + Alloc[kTargetStack - Curr - 1] = 0; + } +} +#else +static void ensureStackAddressSpace() {} +#endif + +/// Attempt to ensure that we have at least 8MiB of usable stack space. +static void ensureSufficientStack() { + struct rlimit rlim; + if (getrlimit(RLIMIT_STACK, &rlim) != 0) + return; + + // Increase the soft stack limit to our desired level, if necessary and + // possible. + if (rlim.rlim_cur != RLIM_INFINITY && + rlim.rlim_cur < rlim_t(DesiredStackSize)) { + // Try to allocate sufficient stack. + if (rlim.rlim_max == RLIM_INFINITY || + rlim.rlim_max >= rlim_t(DesiredStackSize)) + rlim.rlim_cur = DesiredStackSize; + else if (rlim.rlim_cur == rlim.rlim_max) + return; + else + rlim.rlim_cur = rlim.rlim_max; + + if (setrlimit(RLIMIT_STACK, &rlim) != 0 || + rlim.rlim_cur != DesiredStackSize) + return; + } + + // We should now have a stack of size at least DesiredStackSize. Ensure + // that we can actually use that much, if necessary. + ensureStackAddressSpace(); +} +#else +static void ensureSufficientStack() {} +#endif + +int cc1_main(ArrayRef Argv, const char *Argv0, void *MainAddr) { + ensureSufficientStack(); + + std::unique_ptr Clang(new CompilerInstance()); + IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); + + // Register the support for object-file-wrapped Clang modules. + auto PCHOps = Clang->getPCHContainerOperations(); + PCHOps->registerWriter(llvm::make_unique()); + PCHOps->registerReader(llvm::make_unique()); + + // Initialize targets first, so that --version shows registered targets. + llvm::InitializeAllTargets(); + llvm::InitializeAllTargetMCs(); + llvm::InitializeAllAsmPrinters(); + llvm::InitializeAllAsmParsers(); + +#ifdef LINK_POLLY_INTO_TOOLS + llvm::PassRegistry &Registry = *llvm::PassRegistry::getPassRegistry(); + polly::initializePollyPasses(Registry); +#endif + + // Buffer diagnostics from argument parsing so that we can output them using a + // well formed diagnostic object. + IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); + TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer; + DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer); + bool Success = CompilerInvocation::CreateFromArgs( + Clang->getInvocation(), Argv.begin(), Argv.end(), Diags); + + // Infer the builtin include path if unspecified. + if (Clang->getHeaderSearchOpts().UseBuiltinIncludes && + Clang->getHeaderSearchOpts().ResourceDir.empty()) + Clang->getHeaderSearchOpts().ResourceDir = + CompilerInvocation::GetResourcesPath(Argv0, MainAddr); + + // Create the actual diagnostics engine. + Clang->createDiagnostics(); + if (!Clang->hasDiagnostics()) + return 1; + + // Set an error handler, so that any LLVM backend diagnostics go through our + // error handler. + llvm::install_fatal_error_handler(LLVMErrorHandler, + static_cast(&Clang->getDiagnostics())); + + DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics()); + if (!Success) + return 1; + + // Execute the frontend actions. + Success = ExecuteCompilerInvocation(Clang.get()); + + // If any timers were active but haven't been destroyed yet, print their + // results now. This happens in -disable-free mode. + llvm::TimerGroup::printAll(llvm::errs()); + + // Our error handler depends on the Diagnostics object, which we're + // potentially about to delete. Uninstall the handler now so that any + // later errors use the default handling behavior instead. + llvm::remove_fatal_error_handler(); + + // When running with -disable-free, don't do any destruction or shutdown. + if (Clang->getFrontendOpts().DisableFree) { + BuryPointer(std::move(Clang)); + return !Success; + } + + return !Success; +} diff --git a/dbms/programs/clang/Compiler-7.0.0/cc1as_main.cpp b/dbms/programs/clang/Compiler-7.0.0/cc1as_main.cpp new file mode 100644 index 00000000000..d93b1f5cb1d --- /dev/null +++ b/dbms/programs/clang/Compiler-7.0.0/cc1as_main.cpp @@ -0,0 +1,572 @@ +//===-- cc1as_main.cpp - Clang Assembler ---------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This is the entry point to the clang -cc1as functionality, which implements +// the direct interface to the LLVM MC based assembler. +// +//===----------------------------------------------------------------------===// + +#include "clang/Basic/Diagnostic.h" +#include "clang/Basic/DiagnosticOptions.h" +#include "clang/Driver/DriverDiagnostic.h" +#include "clang/Driver/Options.h" +#include "clang/Frontend/FrontendDiagnostic.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" +#include "clang/Frontend/Utils.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/ADT/Triple.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/MC/MCAsmBackend.h" +#include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCCodeEmitter.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCObjectFileInfo.h" +#include "llvm/MC/MCObjectWriter.h" +#include "llvm/MC/MCParser/MCAsmParser.h" +#include "llvm/MC/MCParser/MCTargetAsmParser.h" +#include "llvm/MC/MCRegisterInfo.h" +#include "llvm/MC/MCStreamer.h" +#include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/MC/MCTargetOptions.h" +#include "llvm/Option/Arg.h" +#include "llvm/Option/ArgList.h" +#include "llvm/Option/OptTable.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/FormattedStream.h" +#include "llvm/Support/Host.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Signals.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/TargetRegistry.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/Timer.h" +#include "llvm/Support/raw_ostream.h" +#include +#include +using namespace clang; +using namespace clang::driver; +using namespace clang::driver::options; +using namespace llvm; +using namespace llvm::opt; + +namespace { + +/// Helper class for representing a single invocation of the assembler. +struct AssemblerInvocation { + /// @name Target Options + /// @{ + + /// The name of the target triple to assemble for. + std::string Triple; + + /// If given, the name of the target CPU to determine which instructions + /// are legal. + std::string CPU; + + /// The list of target specific features to enable or disable -- this should + /// be a list of strings starting with '+' or '-'. + std::vector Features; + + /// The list of symbol definitions. + std::vector SymbolDefs; + + /// @} + /// @name Language Options + /// @{ + + std::vector IncludePaths; + unsigned NoInitialTextSection : 1; + unsigned SaveTemporaryLabels : 1; + unsigned GenDwarfForAssembly : 1; + unsigned RelaxELFRelocations : 1; + unsigned DwarfVersion; + std::string DwarfDebugFlags; + std::string DwarfDebugProducer; + std::string DebugCompilationDir; + std::map DebugPrefixMap; + llvm::DebugCompressionType CompressDebugSections = + llvm::DebugCompressionType::None; + std::string MainFileName; + std::string SplitDwarfFile; + + /// @} + /// @name Frontend Options + /// @{ + + std::string InputFile; + std::vector LLVMArgs; + std::string OutputPath; + enum FileType { + FT_Asm, ///< Assembly (.s) output, transliterate mode. + FT_Null, ///< No output, for timing purposes. + FT_Obj ///< Object file output. + }; + FileType OutputType; + unsigned ShowHelp : 1; + unsigned ShowVersion : 1; + + /// @} + /// @name Transliterate Options + /// @{ + + unsigned OutputAsmVariant; + unsigned ShowEncoding : 1; + unsigned ShowInst : 1; + + /// @} + /// @name Assembler Options + /// @{ + + unsigned RelaxAll : 1; + unsigned NoExecStack : 1; + unsigned FatalWarnings : 1; + unsigned IncrementalLinkerCompatible : 1; + + /// The name of the relocation model to use. + std::string RelocationModel; + + /// @} + +public: + AssemblerInvocation() { + Triple = ""; + NoInitialTextSection = 0; + InputFile = "-"; + OutputPath = "-"; + OutputType = FT_Asm; + OutputAsmVariant = 0; + ShowInst = 0; + ShowEncoding = 0; + RelaxAll = 0; + NoExecStack = 0; + FatalWarnings = 0; + IncrementalLinkerCompatible = 0; + DwarfVersion = 0; + } + + static bool CreateFromArgs(AssemblerInvocation &Res, + ArrayRef Argv, + DiagnosticsEngine &Diags); +}; + +} + +bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts, + ArrayRef Argv, + DiagnosticsEngine &Diags) { + bool Success = true; + + // Parse the arguments. + std::unique_ptr OptTbl(createDriverOptTable()); + + const unsigned IncludedFlagsBitmask = options::CC1AsOption; + unsigned MissingArgIndex, MissingArgCount; + InputArgList Args = OptTbl->ParseArgs(Argv, MissingArgIndex, MissingArgCount, + IncludedFlagsBitmask); + + // Check for missing argument error. + if (MissingArgCount) { + Diags.Report(diag::err_drv_missing_argument) + << Args.getArgString(MissingArgIndex) << MissingArgCount; + Success = false; + } + + // Issue errors on unknown arguments. + for (const Arg *A : Args.filtered(OPT_UNKNOWN)) { + auto ArgString = A->getAsString(Args); + std::string Nearest; + if (OptTbl->findNearest(ArgString, Nearest, IncludedFlagsBitmask) > 1) + Diags.Report(diag::err_drv_unknown_argument) << ArgString; + else + Diags.Report(diag::err_drv_unknown_argument_with_suggestion) + << ArgString << Nearest; + Success = false; + } + + // Construct the invocation. + + // Target Options + Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple)); + Opts.CPU = Args.getLastArgValue(OPT_target_cpu); + Opts.Features = Args.getAllArgValues(OPT_target_feature); + + // Use the default target triple if unspecified. + if (Opts.Triple.empty()) + Opts.Triple = llvm::sys::getDefaultTargetTriple(); + + // Language Options + Opts.IncludePaths = Args.getAllArgValues(OPT_I); + Opts.NoInitialTextSection = Args.hasArg(OPT_n); + Opts.SaveTemporaryLabels = Args.hasArg(OPT_msave_temp_labels); + // Any DebugInfoKind implies GenDwarfForAssembly. + Opts.GenDwarfForAssembly = Args.hasArg(OPT_debug_info_kind_EQ); + + if (const Arg *A = Args.getLastArg(OPT_compress_debug_sections, + OPT_compress_debug_sections_EQ)) { + if (A->getOption().getID() == OPT_compress_debug_sections) { + // TODO: be more clever about the compression type auto-detection + Opts.CompressDebugSections = llvm::DebugCompressionType::GNU; + } else { + Opts.CompressDebugSections = + llvm::StringSwitch(A->getValue()) + .Case("none", llvm::DebugCompressionType::None) + .Case("zlib", llvm::DebugCompressionType::Z) + .Case("zlib-gnu", llvm::DebugCompressionType::GNU) + .Default(llvm::DebugCompressionType::None); + } + } + + Opts.RelaxELFRelocations = Args.hasArg(OPT_mrelax_relocations); + Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags); + Opts.DwarfDebugFlags = Args.getLastArgValue(OPT_dwarf_debug_flags); + Opts.DwarfDebugProducer = Args.getLastArgValue(OPT_dwarf_debug_producer); + Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir); + Opts.MainFileName = Args.getLastArgValue(OPT_main_file_name); + + for (const auto &Arg : Args.getAllArgValues(OPT_fdebug_prefix_map_EQ)) + Opts.DebugPrefixMap.insert(StringRef(Arg).split('=')); + + // Frontend Options + if (Args.hasArg(OPT_INPUT)) { + bool First = true; + for (const Arg *A : Args.filtered(OPT_INPUT)) { + if (First) { + Opts.InputFile = A->getValue(); + First = false; + } else { + Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args); + Success = false; + } + } + } + Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm); + Opts.OutputPath = Args.getLastArgValue(OPT_o); + Opts.SplitDwarfFile = Args.getLastArgValue(OPT_split_dwarf_file); + if (Arg *A = Args.getLastArg(OPT_filetype)) { + StringRef Name = A->getValue(); + unsigned OutputType = StringSwitch(Name) + .Case("asm", FT_Asm) + .Case("null", FT_Null) + .Case("obj", FT_Obj) + .Default(~0U); + if (OutputType == ~0U) { + Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name; + Success = false; + } else + Opts.OutputType = FileType(OutputType); + } + Opts.ShowHelp = Args.hasArg(OPT_help); + Opts.ShowVersion = Args.hasArg(OPT_version); + + // Transliterate Options + Opts.OutputAsmVariant = + getLastArgIntValue(Args, OPT_output_asm_variant, 0, Diags); + Opts.ShowEncoding = Args.hasArg(OPT_show_encoding); + Opts.ShowInst = Args.hasArg(OPT_show_inst); + + // Assemble Options + Opts.RelaxAll = Args.hasArg(OPT_mrelax_all); + Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack); + Opts.FatalWarnings = Args.hasArg(OPT_massembler_fatal_warnings); + Opts.RelocationModel = Args.getLastArgValue(OPT_mrelocation_model, "pic"); + Opts.IncrementalLinkerCompatible = + Args.hasArg(OPT_mincremental_linker_compatible); + Opts.SymbolDefs = Args.getAllArgValues(OPT_defsym); + + return Success; +} + +static std::unique_ptr +getOutputStream(StringRef Path, DiagnosticsEngine &Diags, bool Binary) { + // Make sure that the Out file gets unlinked from the disk if we get a + // SIGINT. + if (Path != "-") + sys::RemoveFileOnSignal(Path); + + std::error_code EC; + auto Out = llvm::make_unique( + Path, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text)); + if (EC) { + Diags.Report(diag::err_fe_unable_to_open_output) << Path << EC.message(); + return nullptr; + } + + return Out; +} + +static bool ExecuteAssembler(AssemblerInvocation &Opts, + DiagnosticsEngine &Diags) { + // Get the target specific parser. + std::string Error; + const Target *TheTarget = TargetRegistry::lookupTarget(Opts.Triple, Error); + if (!TheTarget) + return Diags.Report(diag::err_target_unknown_triple) << Opts.Triple; + + ErrorOr> Buffer = + MemoryBuffer::getFileOrSTDIN(Opts.InputFile); + + if (std::error_code EC = Buffer.getError()) { + Error = EC.message(); + return Diags.Report(diag::err_fe_error_reading) << Opts.InputFile; + } + + SourceMgr SrcMgr; + + // Tell SrcMgr about this buffer, which is what the parser will pick up. + SrcMgr.AddNewSourceBuffer(std::move(*Buffer), SMLoc()); + + // Record the location of the include directories so that the lexer can find + // it later. + SrcMgr.setIncludeDirs(Opts.IncludePaths); + + std::unique_ptr MRI(TheTarget->createMCRegInfo(Opts.Triple)); + assert(MRI && "Unable to create target register info!"); + + std::unique_ptr MAI(TheTarget->createMCAsmInfo(*MRI, Opts.Triple)); + assert(MAI && "Unable to create target asm info!"); + + // Ensure MCAsmInfo initialization occurs before any use, otherwise sections + // may be created with a combination of default and explicit settings. + MAI->setCompressDebugSections(Opts.CompressDebugSections); + + MAI->setRelaxELFRelocations(Opts.RelaxELFRelocations); + + bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj; + if (Opts.OutputPath.empty()) + Opts.OutputPath = "-"; + std::unique_ptr FDOS = + getOutputStream(Opts.OutputPath, Diags, IsBinary); + if (!FDOS) + return true; + std::unique_ptr DwoOS; + if (!Opts.SplitDwarfFile.empty()) + DwoOS = getOutputStream(Opts.SplitDwarfFile, Diags, IsBinary); + + // FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and + // MCObjectFileInfo needs a MCContext reference in order to initialize itself. + std::unique_ptr MOFI(new MCObjectFileInfo()); + + MCContext Ctx(MAI.get(), MRI.get(), MOFI.get(), &SrcMgr); + + bool PIC = false; + if (Opts.RelocationModel == "static") { + PIC = false; + } else if (Opts.RelocationModel == "pic") { + PIC = true; + } else { + assert(Opts.RelocationModel == "dynamic-no-pic" && + "Invalid PIC model!"); + PIC = false; + } + + MOFI->InitMCObjectFileInfo(Triple(Opts.Triple), PIC, Ctx); + if (Opts.SaveTemporaryLabels) + Ctx.setAllowTemporaryLabels(false); + if (Opts.GenDwarfForAssembly) + Ctx.setGenDwarfForAssembly(true); + if (!Opts.DwarfDebugFlags.empty()) + Ctx.setDwarfDebugFlags(StringRef(Opts.DwarfDebugFlags)); + if (!Opts.DwarfDebugProducer.empty()) + Ctx.setDwarfDebugProducer(StringRef(Opts.DwarfDebugProducer)); + if (!Opts.DebugCompilationDir.empty()) + Ctx.setCompilationDir(Opts.DebugCompilationDir); + if (!Opts.DebugPrefixMap.empty()) + for (const auto &KV : Opts.DebugPrefixMap) + Ctx.addDebugPrefixMapEntry(KV.first, KV.second); + if (!Opts.MainFileName.empty()) + Ctx.setMainFileName(StringRef(Opts.MainFileName)); + Ctx.setDwarfVersion(Opts.DwarfVersion); + + // Build up the feature string from the target feature list. + std::string FS; + if (!Opts.Features.empty()) { + FS = Opts.Features[0]; + for (unsigned i = 1, e = Opts.Features.size(); i != e; ++i) + FS += "," + Opts.Features[i]; + } + + std::unique_ptr Str; + + std::unique_ptr MCII(TheTarget->createMCInstrInfo()); + std::unique_ptr STI( + TheTarget->createMCSubtargetInfo(Opts.Triple, Opts.CPU, FS)); + + raw_pwrite_stream *Out = FDOS.get(); + std::unique_ptr BOS; + + // FIXME: There is a bit of code duplication with addPassesToEmitFile. + if (Opts.OutputType == AssemblerInvocation::FT_Asm) { + MCInstPrinter *IP = TheTarget->createMCInstPrinter( + llvm::Triple(Opts.Triple), Opts.OutputAsmVariant, *MAI, *MCII, *MRI); + + std::unique_ptr CE; + if (Opts.ShowEncoding) + CE.reset(TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx)); + MCTargetOptions MCOptions; + std::unique_ptr MAB( + TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions)); + + auto FOut = llvm::make_unique(*Out); + Str.reset(TheTarget->createAsmStreamer( + Ctx, std::move(FOut), /*asmverbose*/ true, + /*useDwarfDirectory*/ true, IP, std::move(CE), std::move(MAB), + Opts.ShowInst)); + } else if (Opts.OutputType == AssemblerInvocation::FT_Null) { + Str.reset(createNullStreamer(Ctx)); + } else { + assert(Opts.OutputType == AssemblerInvocation::FT_Obj && + "Invalid file type!"); + if (!FDOS->supportsSeeking()) { + BOS = make_unique(*FDOS); + Out = BOS.get(); + } + + std::unique_ptr CE( + TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx)); + MCTargetOptions MCOptions; + std::unique_ptr MAB( + TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions)); + std::unique_ptr OW = + DwoOS ? MAB->createDwoObjectWriter(*Out, *DwoOS) + : MAB->createObjectWriter(*Out); + + Triple T(Opts.Triple); + Str.reset(TheTarget->createMCObjectStreamer( + T, Ctx, std::move(MAB), std::move(OW), std::move(CE), *STI, + Opts.RelaxAll, Opts.IncrementalLinkerCompatible, + /*DWARFMustBeAtTheEnd*/ true)); + Str.get()->InitSections(Opts.NoExecStack); + } + + // Assembly to object compilation should leverage assembly info. + Str->setUseAssemblerInfoForParsing(true); + + bool Failed = false; + + std::unique_ptr Parser( + createMCAsmParser(SrcMgr, Ctx, *Str.get(), *MAI)); + + // FIXME: init MCTargetOptions from sanitizer flags here. + MCTargetOptions Options; + std::unique_ptr TAP( + TheTarget->createMCAsmParser(*STI, *Parser, *MCII, Options)); + if (!TAP) + Failed = Diags.Report(diag::err_target_unknown_triple) << Opts.Triple; + + // Set values for symbols, if any. + for (auto &S : Opts.SymbolDefs) { + auto Pair = StringRef(S).split('='); + auto Sym = Pair.first; + auto Val = Pair.second; + int64_t Value = 1; + // We have already error checked this in the driver. + Val.getAsInteger(0, Value); + Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value); + } + + if (!Failed) { + Parser->setTargetParser(*TAP.get()); + Failed = Parser->Run(Opts.NoInitialTextSection); + } + + // Close Streamer first. + // It might have a reference to the output stream. + Str.reset(); + // Close the output stream early. + BOS.reset(); + FDOS.reset(); + + // Delete output file if there were errors. + if (Failed) { + if (Opts.OutputPath != "-") + sys::fs::remove(Opts.OutputPath); + if (!Opts.SplitDwarfFile.empty() && Opts.SplitDwarfFile != "-") + sys::fs::remove(Opts.SplitDwarfFile); + } + + return Failed; +} + +static void LLVMErrorHandler(void *UserData, const std::string &Message, + bool GenCrashDiag) { + DiagnosticsEngine &Diags = *static_cast(UserData); + + Diags.Report(diag::err_fe_error_backend) << Message; + + // We cannot recover from llvm errors. + exit(1); +} + +int cc1as_main(ArrayRef Argv, const char *Argv0, void *MainAddr) { + // Initialize targets and assembly printers/parsers. + InitializeAllTargetInfos(); + InitializeAllTargetMCs(); + InitializeAllAsmParsers(); + + // Construct our diagnostic client. + IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); + TextDiagnosticPrinter *DiagClient + = new TextDiagnosticPrinter(errs(), &*DiagOpts); + DiagClient->setPrefix("clang -cc1as"); + IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); + DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient); + + // Set an error handler, so that any LLVM backend diagnostics go through our + // error handler. + ScopedFatalErrorHandler FatalErrorHandler + (LLVMErrorHandler, static_cast(&Diags)); + + // Parse the arguments. + AssemblerInvocation Asm; + if (!AssemblerInvocation::CreateFromArgs(Asm, Argv, Diags)) + return 1; + + if (Asm.ShowHelp) { + std::unique_ptr Opts(driver::createDriverOptTable()); + Opts->PrintHelp(llvm::outs(), "clang -cc1as", "Clang Integrated Assembler", + /*Include=*/driver::options::CC1AsOption, /*Exclude=*/0, + /*ShowAllAliases=*/false); + return 0; + } + + // Honor -version. + // + // FIXME: Use a better -version message? + if (Asm.ShowVersion) { + llvm::cl::PrintVersionMessage(); + return 0; + } + + // Honor -mllvm. + // + // FIXME: Remove this, one day. + if (!Asm.LLVMArgs.empty()) { + unsigned NumArgs = Asm.LLVMArgs.size(); + auto Args = llvm::make_unique(NumArgs + 2); + Args[0] = "clang (LLVM option parsing)"; + for (unsigned i = 0; i != NumArgs; ++i) + Args[i + 1] = Asm.LLVMArgs[i].c_str(); + Args[NumArgs + 1] = nullptr; + llvm::cl::ParseCommandLineOptions(NumArgs + 1, Args.get()); + } + + // Execute the invocation, unless there were parsing errors. + bool Failed = Diags.hasErrorOccurred() || ExecuteAssembler(Asm, Diags); + + // If any timers were active but haven't been destroyed yet, print their + // results now. + TimerGroup::printAll(errs()); + + return !!Failed; +} diff --git a/dbms/programs/clang/Compiler-7.0.0/cc1gen_reproducer_main.cpp b/dbms/programs/clang/Compiler-7.0.0/cc1gen_reproducer_main.cpp new file mode 100644 index 00000000000..a4c034d8d35 --- /dev/null +++ b/dbms/programs/clang/Compiler-7.0.0/cc1gen_reproducer_main.cpp @@ -0,0 +1,196 @@ +//===-- cc1gen_reproducer_main.cpp - Clang reproducer generator ----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This is the entry point to the clang -cc1gen-reproducer functionality, which +// generates reproducers for invocations for clang-based tools. +// +//===----------------------------------------------------------------------===// + +#include "clang/Basic/Diagnostic.h" +#include "clang/Basic/LLVM.h" +#include "clang/Basic/VirtualFileSystem.h" +#include "clang/Driver/Compilation.h" +#include "clang/Driver/Driver.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/YAMLTraits.h" +#include "llvm/Support/raw_ostream.h" + +using namespace clang; + +namespace { + +struct UnsavedFileHash { + std::string Name; + std::string MD5; +}; + +struct ClangInvocationInfo { + std::string Toolchain; + std::string LibclangOperation; + std::string LibclangOptions; + std::vector Arguments; + std::vector InvocationArguments; + std::vector UnsavedFileHashes; + bool Dump = false; +}; + +} // end anonymous namespace + +LLVM_YAML_IS_SEQUENCE_VECTOR(UnsavedFileHash) + +namespace llvm { +namespace yaml { + +template <> struct MappingTraits { + static void mapping(IO &IO, UnsavedFileHash &Info) { + IO.mapRequired("name", Info.Name); + IO.mapRequired("md5", Info.MD5); + } +}; + +template <> struct MappingTraits { + static void mapping(IO &IO, ClangInvocationInfo &Info) { + IO.mapRequired("toolchain", Info.Toolchain); + IO.mapOptional("libclang.operation", Info.LibclangOperation); + IO.mapOptional("libclang.opts", Info.LibclangOptions); + IO.mapRequired("args", Info.Arguments); + IO.mapOptional("invocation-args", Info.InvocationArguments); + IO.mapOptional("unsaved_file_hashes", Info.UnsavedFileHashes); + } +}; + +} // end namespace yaml +} // end namespace llvm + +static std::string generateReproducerMetaInfo(const ClangInvocationInfo &Info) { + std::string Result; + llvm::raw_string_ostream OS(Result); + OS << '{'; + bool NeedComma = false; + auto EmitKey = [&](StringRef Key) { + if (NeedComma) + OS << ", "; + NeedComma = true; + OS << '"' << Key << "\": "; + }; + auto EmitStringKey = [&](StringRef Key, StringRef Value) { + if (Value.empty()) + return; + EmitKey(Key); + OS << '"' << Value << '"'; + }; + EmitStringKey("libclang.operation", Info.LibclangOperation); + EmitStringKey("libclang.opts", Info.LibclangOptions); + if (!Info.InvocationArguments.empty()) { + EmitKey("invocation-args"); + OS << '['; + for (const auto &Arg : llvm::enumerate(Info.InvocationArguments)) { + if (Arg.index()) + OS << ','; + OS << '"' << Arg.value() << '"'; + } + OS << ']'; + } + OS << '}'; + // FIXME: Compare unsaved file hashes and report mismatch in the reproducer. + if (Info.Dump) + llvm::outs() << "REPRODUCER METAINFO: " << OS.str() << "\n"; + return std::move(OS.str()); +} + +/// Generates a reproducer for a set of arguments from a specific invocation. +static llvm::Optional +generateReproducerForInvocationArguments(ArrayRef Argv, + const ClangInvocationInfo &Info) { + using namespace driver; + auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(Argv[0]); + + IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions; + + IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); + DiagnosticsEngine Diags(DiagID, &*DiagOpts, new IgnoringDiagConsumer()); + ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false); + Driver TheDriver(Argv[0], llvm::sys::getDefaultTargetTriple(), Diags); + TheDriver.setTargetAndMode(TargetAndMode); + + std::unique_ptr C(TheDriver.BuildCompilation(Argv)); + if (C && !C->containsError()) { + for (const auto &J : C->getJobs()) { + if (const Command *Cmd = dyn_cast(&J)) { + Driver::CompilationDiagnosticReport Report; + TheDriver.generateCompilationDiagnostics( + *C, *Cmd, generateReproducerMetaInfo(Info), &Report); + return Report; + } + } + } + + return None; +} + +std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes); + +static void printReproducerInformation( + llvm::raw_ostream &OS, const ClangInvocationInfo &Info, + const driver::Driver::CompilationDiagnosticReport &Report) { + OS << "REPRODUCER:\n"; + OS << "{\n"; + OS << R"("files":[)"; + for (const auto &File : llvm::enumerate(Report.TemporaryFiles)) { + if (File.index()) + OS << ','; + OS << '"' << File.value() << '"'; + } + OS << "]\n}\n"; +} + +int cc1gen_reproducer_main(ArrayRef Argv, const char *Argv0, + void *MainAddr) { + if (Argv.size() < 1) { + llvm::errs() << "error: missing invocation file\n"; + return 1; + } + // Parse the invocation descriptor. + StringRef Input = Argv[0]; + llvm::ErrorOr> Buffer = + llvm::MemoryBuffer::getFile(Input); + if (!Buffer) { + llvm::errs() << "error: failed to read " << Input << ": " + << Buffer.getError().message() << "\n"; + return 1; + } + llvm::yaml::Input YAML(Buffer.get()->getBuffer()); + ClangInvocationInfo InvocationInfo; + YAML >> InvocationInfo; + if (Argv.size() > 1 && Argv[1] == StringRef("-v")) + InvocationInfo.Dump = true; + + // Create an invocation that will produce the reproducer. + std::vector DriverArgs; + for (const auto &Arg : InvocationInfo.Arguments) + DriverArgs.push_back(Arg.c_str()); + std::string Path = GetExecutablePath(Argv0, /*CanonicalPrefixes=*/true); + DriverArgs[0] = Path.c_str(); + llvm::Optional Report = + generateReproducerForInvocationArguments(DriverArgs, InvocationInfo); + + // Emit the information about the reproduce files to stdout. + int Result = 1; + if (Report) { + printReproducerInformation(llvm::outs(), InvocationInfo, *Report); + Result = 0; + } + + // Remove the input file. + llvm::sys::fs::remove(Input); + return Result; +} diff --git a/dbms/programs/clang/Compiler-7.0.0/driver.cpp b/dbms/programs/clang/Compiler-7.0.0/driver.cpp new file mode 100644 index 00000000000..79d71b08ba7 --- /dev/null +++ b/dbms/programs/clang/Compiler-7.0.0/driver.cpp @@ -0,0 +1,514 @@ +//===-- driver.cpp - Clang GCC-Compatible Driver --------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This is the entry point to the clang driver; it is a thin wrapper +// for functionality in the Driver clang library. +// +//===----------------------------------------------------------------------===// + +#include "clang/Driver/Driver.h" +#include "clang/Basic/DiagnosticOptions.h" +#include "clang/Driver/Compilation.h" +#include "clang/Driver/DriverDiagnostic.h" +#include "clang/Driver/Options.h" +#include "clang/Driver/ToolChain.h" +#include "clang/Frontend/ChainedDiagnosticConsumer.h" +#include "clang/Frontend/CompilerInvocation.h" +#include "clang/Frontend/SerializedDiagnosticPrinter.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" +#include "clang/Frontend/Utils.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Option/ArgList.h" +#include "llvm/Option/OptTable.h" +#include "llvm/Option/Option.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Host.h" +#include "llvm/Support/InitLLVM.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/Program.h" +#include "llvm/Support/Regex.h" +#include "llvm/Support/Signals.h" +#include "llvm/Support/StringSaver.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/Timer.h" +#include "llvm/Support/raw_ostream.h" +#include +#include +#include +using namespace clang; +using namespace clang::driver; +using namespace llvm::opt; + +std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes) { + if (!CanonicalPrefixes) { + SmallString<128> ExecutablePath(Argv0); + // Do a PATH lookup if Argv0 isn't a valid path. + if (!llvm::sys::fs::exists(ExecutablePath)) + if (llvm::ErrorOr P = + llvm::sys::findProgramByName(ExecutablePath)) + ExecutablePath = *P; + return ExecutablePath.str(); + } + + // This just needs to be some symbol in the binary; C++ doesn't + // allow taking the address of ::main however. + void *P = (void*) (intptr_t) GetExecutablePath; + return llvm::sys::fs::getMainExecutable(Argv0, P); +} + +static const char *GetStableCStr(std::set &SavedStrings, + StringRef S) { + return SavedStrings.insert(S).first->c_str(); +} + +/// ApplyQAOverride - Apply a list of edits to the input argument lists. +/// +/// The input string is a space separate list of edits to perform, +/// they are applied in order to the input argument lists. Edits +/// should be one of the following forms: +/// +/// '#': Silence information about the changes to the command line arguments. +/// +/// '^': Add FOO as a new argument at the beginning of the command line. +/// +/// '+': Add FOO as a new argument at the end of the command line. +/// +/// 's/XXX/YYY/': Substitute the regular expression XXX with YYY in the command +/// line. +/// +/// 'xOPTION': Removes all instances of the literal argument OPTION. +/// +/// 'XOPTION': Removes all instances of the literal argument OPTION, +/// and the following argument. +/// +/// 'Ox': Removes all flags matching 'O' or 'O[sz0-9]' and adds 'Ox' +/// at the end of the command line. +/// +/// \param OS - The stream to write edit information to. +/// \param Args - The vector of command line arguments. +/// \param Edit - The override command to perform. +/// \param SavedStrings - Set to use for storing string representations. +static void ApplyOneQAOverride(raw_ostream &OS, + SmallVectorImpl &Args, + StringRef Edit, + std::set &SavedStrings) { + // This does not need to be efficient. + + if (Edit[0] == '^') { + const char *Str = + GetStableCStr(SavedStrings, Edit.substr(1)); + OS << "### Adding argument " << Str << " at beginning\n"; + Args.insert(Args.begin() + 1, Str); + } else if (Edit[0] == '+') { + const char *Str = + GetStableCStr(SavedStrings, Edit.substr(1)); + OS << "### Adding argument " << Str << " at end\n"; + Args.push_back(Str); + } else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") && + Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) { + StringRef MatchPattern = Edit.substr(2).split('/').first; + StringRef ReplPattern = Edit.substr(2).split('/').second; + ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1); + + for (unsigned i = 1, e = Args.size(); i != e; ++i) { + // Ignore end-of-line response file markers + if (Args[i] == nullptr) + continue; + std::string Repl = llvm::Regex(MatchPattern).sub(ReplPattern, Args[i]); + + if (Repl != Args[i]) { + OS << "### Replacing '" << Args[i] << "' with '" << Repl << "'\n"; + Args[i] = GetStableCStr(SavedStrings, Repl); + } + } + } else if (Edit[0] == 'x' || Edit[0] == 'X') { + auto Option = Edit.substr(1); + for (unsigned i = 1; i < Args.size();) { + if (Option == Args[i]) { + OS << "### Deleting argument " << Args[i] << '\n'; + Args.erase(Args.begin() + i); + if (Edit[0] == 'X') { + if (i < Args.size()) { + OS << "### Deleting argument " << Args[i] << '\n'; + Args.erase(Args.begin() + i); + } else + OS << "### Invalid X edit, end of command line!\n"; + } + } else + ++i; + } + } else if (Edit[0] == 'O') { + for (unsigned i = 1; i < Args.size();) { + const char *A = Args[i]; + // Ignore end-of-line response file markers + if (A == nullptr) + continue; + if (A[0] == '-' && A[1] == 'O' && + (A[2] == '\0' || + (A[3] == '\0' && (A[2] == 's' || A[2] == 'z' || + ('0' <= A[2] && A[2] <= '9'))))) { + OS << "### Deleting argument " << Args[i] << '\n'; + Args.erase(Args.begin() + i); + } else + ++i; + } + OS << "### Adding argument " << Edit << " at end\n"; + Args.push_back(GetStableCStr(SavedStrings, '-' + Edit.str())); + } else { + OS << "### Unrecognized edit: " << Edit << "\n"; + } +} + +/// ApplyQAOverride - Apply a comma separate list of edits to the +/// input argument lists. See ApplyOneQAOverride. +static void ApplyQAOverride(SmallVectorImpl &Args, + const char *OverrideStr, + std::set &SavedStrings) { + raw_ostream *OS = &llvm::errs(); + + if (OverrideStr[0] == '#') { + ++OverrideStr; + OS = &llvm::nulls(); + } + + *OS << "### CCC_OVERRIDE_OPTIONS: " << OverrideStr << "\n"; + + // This does not need to be efficient. + + const char *S = OverrideStr; + while (*S) { + const char *End = ::strchr(S, ' '); + if (!End) + End = S + strlen(S); + if (End != S) + ApplyOneQAOverride(*OS, Args, std::string(S, End), SavedStrings); + S = End; + if (*S != '\0') + ++S; + } +} + +extern int cc1_main(ArrayRef Argv, const char *Argv0, + void *MainAddr); +extern int cc1as_main(ArrayRef Argv, const char *Argv0, + void *MainAddr); +extern int cc1gen_reproducer_main(ArrayRef Argv, + const char *Argv0, void *MainAddr); + +static void insertTargetAndModeArgs(const ParsedClangName &NameParts, + SmallVectorImpl &ArgVector, + std::set &SavedStrings) { + // Put target and mode arguments at the start of argument list so that + // arguments specified in command line could override them. Avoid putting + // them at index 0, as an option like '-cc1' must remain the first. + int InsertionPoint = 0; + if (ArgVector.size() > 0) + ++InsertionPoint; + + if (NameParts.DriverMode) { + // Add the mode flag to the arguments. + ArgVector.insert(ArgVector.begin() + InsertionPoint, + GetStableCStr(SavedStrings, NameParts.DriverMode)); + } + + if (NameParts.TargetIsValid) { + const char *arr[] = {"-target", GetStableCStr(SavedStrings, + NameParts.TargetPrefix)}; + ArgVector.insert(ArgVector.begin() + InsertionPoint, + std::begin(arr), std::end(arr)); + } +} + +static void getCLEnvVarOptions(std::string &EnvValue, llvm::StringSaver &Saver, + SmallVectorImpl &Opts) { + llvm::cl::TokenizeWindowsCommandLine(EnvValue, Saver, Opts); + // The first instance of '#' should be replaced with '=' in each option. + for (const char *Opt : Opts) + if (char *NumberSignPtr = const_cast(::strchr(Opt, '#'))) + *NumberSignPtr = '='; +} + +static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) { + // Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE. + TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS"); + if (TheDriver.CCPrintOptions) + TheDriver.CCPrintOptionsFilename = ::getenv("CC_PRINT_OPTIONS_FILE"); + + // Handle CC_PRINT_HEADERS and CC_PRINT_HEADERS_FILE. + TheDriver.CCPrintHeaders = !!::getenv("CC_PRINT_HEADERS"); + if (TheDriver.CCPrintHeaders) + TheDriver.CCPrintHeadersFilename = ::getenv("CC_PRINT_HEADERS_FILE"); + + // Handle CC_LOG_DIAGNOSTICS and CC_LOG_DIAGNOSTICS_FILE. + TheDriver.CCLogDiagnostics = !!::getenv("CC_LOG_DIAGNOSTICS"); + if (TheDriver.CCLogDiagnostics) + TheDriver.CCLogDiagnosticsFilename = ::getenv("CC_LOG_DIAGNOSTICS_FILE"); +} + +static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient, + const std::string &Path) { + // If the clang binary happens to be named cl.exe for compatibility reasons, + // use clang-cl.exe as the prefix to avoid confusion between clang and MSVC. + StringRef ExeBasename(llvm::sys::path::filename(Path)); + if (ExeBasename.equals_lower("cl.exe")) + ExeBasename = "clang-cl.exe"; + DiagClient->setPrefix(ExeBasename); +} + +// This lets us create the DiagnosticsEngine with a properly-filled-out +// DiagnosticOptions instance. +static DiagnosticOptions * +CreateAndPopulateDiagOpts(ArrayRef argv) { + auto *DiagOpts = new DiagnosticOptions; + std::unique_ptr Opts(createDriverOptTable()); + unsigned MissingArgIndex, MissingArgCount; + InputArgList Args = + Opts->ParseArgs(argv.slice(1), MissingArgIndex, MissingArgCount); + // We ignore MissingArgCount and the return value of ParseDiagnosticArgs. + // Any errors that would be diagnosed here will also be diagnosed later, + // when the DiagnosticsEngine actually exists. + (void)ParseDiagnosticArgs(*DiagOpts, Args); + return DiagOpts; +} + +static void SetInstallDir(SmallVectorImpl &argv, + Driver &TheDriver, bool CanonicalPrefixes) { + // Attempt to find the original path used to invoke the driver, to determine + // the installed path. We do this manually, because we want to support that + // path being a symlink. + SmallString<128> InstalledPath(argv[0]); + + // Do a PATH lookup, if there are no directory components. + if (llvm::sys::path::filename(InstalledPath) == InstalledPath) + if (llvm::ErrorOr Tmp = llvm::sys::findProgramByName( + llvm::sys::path::filename(InstalledPath.str()))) + InstalledPath = *Tmp; + + // FIXME: We don't actually canonicalize this, we just make it absolute. + if (CanonicalPrefixes) + llvm::sys::fs::make_absolute(InstalledPath); + + StringRef InstalledPathParent(llvm::sys::path::parent_path(InstalledPath)); + if (llvm::sys::fs::exists(InstalledPathParent)) + TheDriver.setInstalledDir(InstalledPathParent); +} + +static int ExecuteCC1Tool(ArrayRef argv, StringRef Tool) { + void *GetExecutablePathVP = (void *)(intptr_t) GetExecutablePath; + if (Tool == "") + return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP); + if (Tool == "as") + return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP); + if (Tool == "gen-reproducer") + return cc1gen_reproducer_main(argv.slice(2), argv[0], GetExecutablePathVP); + + // Reject unknown tools. + llvm::errs() << "error: unknown integrated tool '" << Tool << "'. " + << "Valid tools include '-cc1' and '-cc1as'.\n"; + return 1; +} + +int mainEntryClickHouseClang(int argc_, /* const */ char **argv_) { + llvm::InitLLVM X(argc_, argv_); + SmallVector argv(argv_, argv_ + argc_); + + if (llvm::sys::Process::FixupStandardFileDescriptors()) + return 1; + + llvm::InitializeAllTargets(); + auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(argv[0]); + + llvm::BumpPtrAllocator A; + llvm::StringSaver Saver(A); + + // Parse response files using the GNU syntax, unless we're in CL mode. There + // are two ways to put clang in CL compatibility mode: argv[0] is either + // clang-cl or cl, or --driver-mode=cl is on the command line. The normal + // command line parsing can't happen until after response file parsing, so we + // have to manually search for a --driver-mode=cl argument the hard way. + // Finally, our -cc1 tools don't care which tokenization mode we use because + // response files written by clang will tokenize the same way in either mode. + bool ClangCLMode = false; + if (StringRef(TargetAndMode.DriverMode).equals("--driver-mode=cl") || + std::find_if(argv.begin(), argv.end(), [](const char *F) { + return F && strcmp(F, "--driver-mode=cl") == 0; + }) != argv.end()) { + ClangCLMode = true; + } + enum { Default, POSIX, Windows } RSPQuoting = Default; + for (const char *F : argv) { + if (strcmp(F, "--rsp-quoting=posix") == 0) + RSPQuoting = POSIX; + else if (strcmp(F, "--rsp-quoting=windows") == 0) + RSPQuoting = Windows; + } + + // Determines whether we want nullptr markers in argv to indicate response + // files end-of-lines. We only use this for the /LINK driver argument with + // clang-cl.exe on Windows. + bool MarkEOLs = ClangCLMode; + + llvm::cl::TokenizerCallback Tokenizer; + if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode)) + Tokenizer = &llvm::cl::TokenizeWindowsCommandLine; + else + Tokenizer = &llvm::cl::TokenizeGNUCommandLine; + + if (MarkEOLs && argv.size() > 1 && StringRef(argv[1]).startswith("-cc1")) + MarkEOLs = false; + llvm::cl::ExpandResponseFiles(Saver, Tokenizer, argv, MarkEOLs); + + // Handle -cc1 integrated tools, even if -cc1 was expanded from a response + // file. + auto FirstArg = std::find_if(argv.begin() + 1, argv.end(), + [](const char *A) { return A != nullptr; }); + if (FirstArg != argv.end() && StringRef(*FirstArg).startswith("-cc1")) { + // If -cc1 came from a response file, remove the EOL sentinels. + if (MarkEOLs) { + auto newEnd = std::remove(argv.begin(), argv.end(), nullptr); + argv.resize(newEnd - argv.begin()); + } + return ExecuteCC1Tool(argv, argv[1] + 4); + } + + bool CanonicalPrefixes = true; + for (int i = 1, size = argv.size(); i < size; ++i) { + // Skip end-of-line response file markers + if (argv[i] == nullptr) + continue; + if (StringRef(argv[i]) == "-no-canonical-prefixes") { + CanonicalPrefixes = false; + break; + } + } + + // Handle CL and _CL_ which permits additional command line options to be + // prepended or appended. + if (ClangCLMode) { + // Arguments in "CL" are prepended. + llvm::Optional OptCL = llvm::sys::Process::GetEnv("CL"); + if (OptCL.hasValue()) { + SmallVector PrependedOpts; + getCLEnvVarOptions(OptCL.getValue(), Saver, PrependedOpts); + + // Insert right after the program name to prepend to the argument list. + argv.insert(argv.begin() + 1, PrependedOpts.begin(), PrependedOpts.end()); + } + // Arguments in "_CL_" are appended. + llvm::Optional Opt_CL_ = llvm::sys::Process::GetEnv("_CL_"); + if (Opt_CL_.hasValue()) { + SmallVector AppendedOpts; + getCLEnvVarOptions(Opt_CL_.getValue(), Saver, AppendedOpts); + + // Insert at the end of the argument list to append. + argv.append(AppendedOpts.begin(), AppendedOpts.end()); + } + } + + std::set SavedStrings; + // Handle CCC_OVERRIDE_OPTIONS, used for editing a command line behind the + // scenes. + if (const char *OverrideStr = ::getenv("CCC_OVERRIDE_OPTIONS")) { + // FIXME: Driver shouldn't take extra initial argument. + ApplyQAOverride(argv, OverrideStr, SavedStrings); + } + + std::string Path = GetExecutablePath(argv[0], CanonicalPrefixes); + + IntrusiveRefCntPtr DiagOpts = + CreateAndPopulateDiagOpts(argv); + + TextDiagnosticPrinter *DiagClient + = new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts); + FixupDiagPrefixExeName(DiagClient, Path); + + IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); + + DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient); + + if (!DiagOpts->DiagnosticSerializationFile.empty()) { + auto SerializedConsumer = + clang::serialized_diags::create(DiagOpts->DiagnosticSerializationFile, + &*DiagOpts, /*MergeChildRecords=*/true); + Diags.setClient(new ChainedDiagnosticConsumer( + Diags.takeClient(), std::move(SerializedConsumer))); + } + + ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false); + + Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags); + SetInstallDir(argv, TheDriver, CanonicalPrefixes); + TheDriver.setTargetAndMode(TargetAndMode); + + insertTargetAndModeArgs(TargetAndMode, argv, SavedStrings); + + SetBackdoorDriverOutputsFromEnvVars(TheDriver); + + std::unique_ptr C(TheDriver.BuildCompilation(argv)); + int Res = 1; + if (C && !C->containsError()) { + SmallVector, 4> FailingCommands; + Res = TheDriver.ExecuteCompilation(*C, FailingCommands); + + // Force a crash to test the diagnostics. + if (TheDriver.GenReproducer) { + Diags.Report(diag::err_drv_force_crash) + << !::getenv("FORCE_CLANG_DIAGNOSTICS_CRASH"); + + // Pretend that every command failed. + FailingCommands.clear(); + for (const auto &J : C->getJobs()) + if (const Command *C = dyn_cast(&J)) + FailingCommands.push_back(std::make_pair(-1, C)); + } + + for (const auto &P : FailingCommands) { + int CommandRes = P.first; + const Command *FailingCommand = P.second; + if (!Res) + Res = CommandRes; + + // If result status is < 0, then the driver command signalled an error. + // If result status is 70, then the driver command reported a fatal error. + // On Windows, abort will return an exit code of 3. In these cases, + // generate additional diagnostic information if possible. + bool DiagnoseCrash = CommandRes < 0 || CommandRes == 70; +#ifdef _WIN32 + DiagnoseCrash |= CommandRes == 3; +#endif + if (DiagnoseCrash) { + TheDriver.generateCompilationDiagnostics(*C, *FailingCommand); + break; + } + } + } + + Diags.getClient()->finish(); + + // If any timers were active but haven't been destroyed yet, print their + // results now. This happens in -disable-free mode. + llvm::TimerGroup::printAll(llvm::errs()); + +#ifdef _WIN32 + // Exit status should not be negative on Win32, unless abnormal termination. + // Once abnormal termiation was caught, negative status should not be + // propagated. + if (Res < 0) + Res = 1; +#endif + + // If we have multiple failing commands, we return the result of the first + // failing command. + return Res; +} diff --git a/dbms/src/Server/Compiler-6.0.0/lld.cpp b/dbms/programs/clang/Compiler-7.0.0/lld.cpp similarity index 65% rename from dbms/src/Server/Compiler-6.0.0/lld.cpp rename to dbms/programs/clang/Compiler-7.0.0/lld.cpp index aba5ade16ad..8e118b6e24b 100644 --- a/dbms/src/Server/Compiler-6.0.0/lld.cpp +++ b/dbms/programs/clang/Compiler-7.0.0/lld.cpp @@ -7,12 +7,22 @@ // //===----------------------------------------------------------------------===// // -// This is the entry point to the lld driver. This is a thin wrapper which -// dispatches to the given platform specific driver. +// This file contains the main function of the lld executable. The main +// function is a thin wrapper which dispatches to the platform specific +// driver. // -// If there is -flavor option, it is dispatched according to the arguments. -// If the flavor parameter is not present, then it is dispatched according -// to argv[0]. +// lld is a single executable that contains four different linkers for ELF, +// COFF, WebAssembly and Mach-O. The main function dispatches according to +// argv[0] (i.e. command name). The most common name for each target is shown +// below: +// +// - ld.lld: ELF (Unix) +// - ld64: Mach-O (macOS) +// - lld-link: COFF (Windows) +// - ld-wasm: WebAssembly +// +// lld can be invoked as "lld" along with "-flavor" option. This is for +// backward compatibility and not recommended. // //===----------------------------------------------------------------------===// @@ -20,15 +30,15 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/ADT/Twine.h" -#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/InitLLVM.h" #include "llvm/Support/Path.h" -#include "llvm/Support/PrettyStackTrace.h" -#include "llvm/Support/Signals.h" - +#include using namespace lld; using namespace llvm; using namespace llvm::sys; +/* + enum Flavor { Invalid, Gnu, // -flavor gnu @@ -42,8 +52,6 @@ LLVM_ATTRIBUTE_NORETURN static void die(const Twine &S) { exit(1); } -#if 0 - static Flavor getFlavor(StringRef S) { return StringSwitch(S) .CasesLower("ld", "ld.lld", "gnu", Gnu) @@ -103,35 +111,40 @@ static Flavor parseFlavor(std::vector &V) { Arg0 = Arg0.drop_back(4); return parseProgname(Arg0); } +*/ -#endif +// If this function returns true, lld calls _exit() so that it quickly +// exits without invoking destructors of globally allocated objects. +// +// We don't want to do that if we are running tests though, because +// doing that breaks leak sanitizer. So, lit sets this environment variable, +// and we use it to detect whether we are running tests or not. +static bool canExitEarly() { return StringRef(getenv("LLD_IN_TEST")) != "1"; } /// Universal linker main(). This linker emulates the gnu, darwin, or /// windows linker based on the argv[0] or -flavor option. -int mainEntryClickHouseLLD(int Argc, char **Argv) { - // Standard set up, so program fails gracefully. - sys::PrintStackTraceOnErrorSignal(Argv[0]); - PrettyStackTraceProgram StackPrinter(Argc, Argv); - llvm_shutdown_obj Shutdown; +int mainEntryClickHouseLLD(int Argc, /* const */ char **Argv) { + InitLLVM X(Argc, Argv); std::vector Args(Argv, Argv + Argc); -#if 0 +/* switch (parseFlavor(Args)) { case Gnu: if (isPETarget(Args)) return !mingw::link(Args); -#endif - return !elf::link(Args, true); -#if 0 +*/ + return !elf::link(Args, canExitEarly()); +/* case WinLink: - return !coff::link(Args, true); + return !coff::link(Args, canExitEarly()); case Darwin: - return !mach_o::link(Args); + return !mach_o::link(Args, canExitEarly()); case Wasm: - return !wasm::link(Args, true); + return !wasm::link(Args, canExitEarly()); default: die("lld is a generic driver.\n" - "Invoke ld.lld (Unix), ld (macOS) or lld-link (Windows) instead."); + "Invoke ld.lld (Unix), ld64.lld (macOS), lld-link (Windows), wasm-lld" + " (WebAssembly) instead"); } -#endif +*/ } diff --git a/dbms/programs/clang/Compiler-7.0.0bundled/CMakeLists.txt b/dbms/programs/clang/Compiler-7.0.0bundled/CMakeLists.txt new file mode 100644 index 00000000000..d0ccc8d672c --- /dev/null +++ b/dbms/programs/clang/Compiler-7.0.0bundled/CMakeLists.txt @@ -0,0 +1,49 @@ +add_definitions(-Wno-error -Wno-unused-parameter -Wno-non-virtual-dtor -U_LIBCPP_DEBUG) + +link_directories(${LLVM_LIBRARY_DIRS}) + +add_library(clickhouse-compiler-lib ${LINK_MODE} + driver.cpp + cc1_main.cpp + cc1as_main.cpp + lld.cpp) + +target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0) + +string(REPLACE "${INCLUDE_DEBUG_HELPERS}" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) # cant compile with -fno-rtti + +llvm_libs_all(REQUIRED_LLVM_LIBRARIES) + +message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}") + +target_include_directories(clickhouse-compiler-lib SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS}) + +# This is extracted almost directly from CMakeFiles/.../link.txt in LLVM build directory. + +target_link_libraries(clickhouse-compiler-lib PRIVATE + +clangBasic clangCodeGen clangDriver +clangFrontend +clangFrontendTool +clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend +clangParse clangSerialization clangSema clangEdit clangStaticAnalyzerCheckers +clangASTMatchers clangStaticAnalyzerCore clangAnalysis clangAST clangRewrite clangLex clangBasic +clangCrossTU clangIndex + +lldCOFF +lldDriver +lldELF +lldMinGW +lldMachO +lldReaderWriter +lldYAML +lldCommon +lldCore + +${REQUIRED_LLVM_LIBRARIES} + +PUBLIC ${ZLIB_LIBRARIES} ${EXECINFO_LIBRARY} Threads::Threads +${MALLOC_LIBRARIES} +${GLIBC_COMPATIBILITY_LIBRARIES} +${MEMCPY_LIBRARIES} +) diff --git a/dbms/programs/clang/Compiler-7.0.0bundled/cc1_main.cpp b/dbms/programs/clang/Compiler-7.0.0bundled/cc1_main.cpp new file mode 100644 index 00000000000..3686475dd42 --- /dev/null +++ b/dbms/programs/clang/Compiler-7.0.0bundled/cc1_main.cpp @@ -0,0 +1,243 @@ +//===-- cc1_main.cpp - Clang CC1 Compiler Frontend ------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This is the entry point to the clang -cc1 functionality, which implements the +// core compiler functionality along with a number of additional tools for +// demonstration and testing purposes. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Option/Arg.h" +#include "clang/CodeGen/ObjectFilePCHContainerOperations.h" +#include "clang/Config/config.h" +#include "clang/Driver/DriverDiagnostic.h" +#include "clang/Driver/Options.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/CompilerInvocation.h" +#include "clang/Frontend/FrontendDiagnostic.h" +#include "clang/Frontend/TextDiagnosticBuffer.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" +#include "clang/Frontend/Utils.h" +#include "clang/FrontendTool/Utils.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/Config/llvm-config.h" +#include "llvm/LinkAllPasses.h" +#include "llvm/Option/ArgList.h" +#include "llvm/Option/OptTable.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/Signals.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/Timer.h" +#include "llvm/Support/raw_ostream.h" +#include + +#ifdef CLANG_HAVE_RLIMITS +#include +#endif + +// have no .a version in packages +#undef LINK_POLLY_INTO_TOOLS + +using namespace clang; +using namespace llvm::opt; + +//===----------------------------------------------------------------------===// +// Main driver +//===----------------------------------------------------------------------===// + +static void LLVMErrorHandler(void *UserData, const std::string &Message, + bool GenCrashDiag) { + DiagnosticsEngine &Diags = *static_cast(UserData); + + Diags.Report(diag::err_fe_error_backend) << Message; + + // Run the interrupt handlers to make sure any special cleanups get done, in + // particular that we remove files registered with RemoveFileOnSignal. + llvm::sys::RunInterruptHandlers(); + + // We cannot recover from llvm errors. When reporting a fatal error, exit + // with status 70 to generate crash diagnostics. For BSD systems this is + // defined as an internal software error. Otherwise, exit with status 1. + exit(GenCrashDiag ? 70 : 1); +} + +#ifdef LINK_POLLY_INTO_TOOLS +namespace polly { +void initializePollyPasses(llvm::PassRegistry &Registry); +} +#endif + +#ifdef CLANG_HAVE_RLIMITS +// The amount of stack we think is "sufficient". If less than this much is +// available, we may be unable to reach our template instantiation depth +// limit and other similar limits. +// FIXME: Unify this with the stack we request when spawning a thread to build +// a module. +static const int kSufficientStack = 8 << 20; + +#if defined(__linux__) && defined(__PIE__) +static size_t getCurrentStackAllocation() { + // If we can't compute the current stack usage, allow for 512K of command + // line arguments and environment. + size_t Usage = 512 * 1024; + if (FILE *StatFile = fopen("/proc/self/stat", "r")) { + // We assume that the stack extends from its current address to the end of + // the environment space. In reality, there is another string literal (the + // program name) after the environment, but this is close enough (we only + // need to be within 100K or so). + unsigned long StackPtr, EnvEnd; + // Disable silly GCC -Wformat warning that complains about length + // modifiers on ignored format specifiers. We want to retain these + // for documentation purposes even though they have no effect. +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat" +#endif + if (fscanf(StatFile, + "%*d %*s %*c %*d %*d %*d %*d %*d %*u %*lu %*lu %*lu %*lu %*lu " + "%*lu %*ld %*ld %*ld %*ld %*ld %*ld %*llu %*lu %*ld %*lu %*lu " + "%*lu %*lu %lu %*lu %*lu %*lu %*lu %*lu %*llu %*lu %*lu %*d %*d " + "%*u %*u %*llu %*lu %*ld %*lu %*lu %*lu %*lu %*lu %*lu %lu %*d", + &StackPtr, &EnvEnd) == 2) { +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif + Usage = StackPtr < EnvEnd ? EnvEnd - StackPtr : StackPtr - EnvEnd; + } + fclose(StatFile); + } + return Usage; +} + +#include + +LLVM_ATTRIBUTE_NOINLINE +static void ensureStackAddressSpace(int ExtraChunks = 0) { + // Linux kernels prior to 4.1 will sometimes locate the heap of a PIE binary + // relatively close to the stack (they are only guaranteed to be 128MiB + // apart). This results in crashes if we happen to heap-allocate more than + // 128MiB before we reach our stack high-water mark. + // + // To avoid these crashes, ensure that we have sufficient virtual memory + // pages allocated before we start running. + size_t Curr = getCurrentStackAllocation(); + const int kTargetStack = kSufficientStack - 256 * 1024; + if (Curr < kTargetStack) { + volatile char *volatile Alloc = + static_cast(alloca(kTargetStack - Curr)); + Alloc[0] = 0; + Alloc[kTargetStack - Curr - 1] = 0; + } +} +#else +static void ensureStackAddressSpace() {} +#endif + +/// Attempt to ensure that we have at least 8MiB of usable stack space. +static void ensureSufficientStack() { + struct rlimit rlim; + if (getrlimit(RLIMIT_STACK, &rlim) != 0) + return; + + // Increase the soft stack limit to our desired level, if necessary and + // possible. + if (rlim.rlim_cur != RLIM_INFINITY && rlim.rlim_cur < kSufficientStack) { + // Try to allocate sufficient stack. + if (rlim.rlim_max == RLIM_INFINITY || rlim.rlim_max >= kSufficientStack) + rlim.rlim_cur = kSufficientStack; + else if (rlim.rlim_cur == rlim.rlim_max) + return; + else + rlim.rlim_cur = rlim.rlim_max; + + if (setrlimit(RLIMIT_STACK, &rlim) != 0 || + rlim.rlim_cur != kSufficientStack) + return; + } + + // We should now have a stack of size at least kSufficientStack. Ensure + // that we can actually use that much, if necessary. + ensureStackAddressSpace(); +} +#else +static void ensureSufficientStack() {} +#endif + +int cc1_main(ArrayRef Argv, const char *Argv0, void *MainAddr) { + ensureSufficientStack(); + + std::unique_ptr Clang(new CompilerInstance()); + IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); + + // Register the support for object-file-wrapped Clang modules. + auto PCHOps = Clang->getPCHContainerOperations(); + PCHOps->registerWriter(llvm::make_unique()); + PCHOps->registerReader(llvm::make_unique()); + + // Initialize targets first, so that --version shows registered targets. + llvm::InitializeAllTargets(); + llvm::InitializeAllTargetMCs(); + llvm::InitializeAllAsmPrinters(); + llvm::InitializeAllAsmParsers(); + +#ifdef LINK_POLLY_INTO_TOOLS + llvm::PassRegistry &Registry = *llvm::PassRegistry::getPassRegistry(); + polly::initializePollyPasses(Registry); +#endif + + // Buffer diagnostics from argument parsing so that we can output them using a + // well formed diagnostic object. + IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); + TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer; + DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer); + bool Success = CompilerInvocation::CreateFromArgs( + Clang->getInvocation(), Argv.begin(), Argv.end(), Diags); + + // Infer the builtin include path if unspecified. + if (Clang->getHeaderSearchOpts().UseBuiltinIncludes && + Clang->getHeaderSearchOpts().ResourceDir.empty()) + Clang->getHeaderSearchOpts().ResourceDir = + CompilerInvocation::GetResourcesPath(Argv0, MainAddr); + + // Create the actual diagnostics engine. + Clang->createDiagnostics(); + if (!Clang->hasDiagnostics()) + return 1; + + // Set an error handler, so that any LLVM backend diagnostics go through our + // error handler. + llvm::install_fatal_error_handler(LLVMErrorHandler, + static_cast(&Clang->getDiagnostics())); + + DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics()); + if (!Success) + return 1; + + // Execute the frontend actions. + Success = ExecuteCompilerInvocation(Clang.get()); + + // If any timers were active but haven't been destroyed yet, print their + // results now. This happens in -disable-free mode. + llvm::TimerGroup::printAll(llvm::errs()); + + // Our error handler depends on the Diagnostics object, which we're + // potentially about to delete. Uninstall the handler now so that any + // later errors use the default handling behavior instead. + llvm::remove_fatal_error_handler(); + + // When running with -disable-free, don't do any destruction or shutdown. + if (Clang->getFrontendOpts().DisableFree) { + BuryPointer(std::move(Clang)); + return !Success; + } + + return !Success; +} diff --git a/dbms/programs/clang/Compiler-7.0.0bundled/cc1as_main.cpp b/dbms/programs/clang/Compiler-7.0.0bundled/cc1as_main.cpp new file mode 100644 index 00000000000..ce23422077f --- /dev/null +++ b/dbms/programs/clang/Compiler-7.0.0bundled/cc1as_main.cpp @@ -0,0 +1,555 @@ +//===-- cc1as_main.cpp - Clang Assembler ---------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This is the entry point to the clang -cc1as functionality, which implements +// the direct interface to the LLVM MC based assembler. +// +//===----------------------------------------------------------------------===// + +#include "clang/Basic/Diagnostic.h" +#include "clang/Basic/DiagnosticOptions.h" +#include "clang/Driver/DriverDiagnostic.h" +#include "clang/Driver/Options.h" +#include "clang/Frontend/FrontendDiagnostic.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" +#include "clang/Frontend/Utils.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/ADT/Triple.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/MC/MCAsmBackend.h" +#include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCCodeEmitter.h" +#include "llvm/MC/MCContext.h" +#include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCObjectFileInfo.h" +#include "llvm/MC/MCParser/MCAsmParser.h" +#include "llvm/MC/MCParser/MCTargetAsmParser.h" +#include "llvm/MC/MCRegisterInfo.h" +#include "llvm/MC/MCStreamer.h" +#include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/MC/MCTargetOptions.h" +#include "llvm/Option/Arg.h" +#include "llvm/Option/ArgList.h" +#include "llvm/Option/OptTable.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/FormattedStream.h" +#include "llvm/Support/Host.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Signals.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/TargetRegistry.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/Timer.h" +#include "llvm/Support/raw_ostream.h" +#include +#include +using namespace clang; +using namespace clang::driver; +using namespace clang::driver::options; +using namespace llvm; +using namespace llvm::opt; + + +namespace { + +/// \brief Helper class for representing a single invocation of the assembler. +struct AssemblerInvocation { + /// @name Target Options + /// @{ + + /// The name of the target triple to assemble for. + std::string Triple; + + /// If given, the name of the target CPU to determine which instructions + /// are legal. + std::string CPU; + + /// The list of target specific features to enable or disable -- this should + /// be a list of strings starting with '+' or '-'. + std::vector Features; + + /// The list of symbol definitions. + std::vector SymbolDefs; + + /// @} + /// @name Language Options + /// @{ + + std::vector IncludePaths; + unsigned NoInitialTextSection : 1; + unsigned SaveTemporaryLabels : 1; + unsigned GenDwarfForAssembly : 1; + unsigned RelaxELFRelocations : 1; + unsigned DwarfVersion; + std::string DwarfDebugFlags; + std::string DwarfDebugProducer; + std::string DebugCompilationDir; + llvm::DebugCompressionType CompressDebugSections = + llvm::DebugCompressionType::None; + std::string MainFileName; + + /// @} + /// @name Frontend Options + /// @{ + + std::string InputFile; + std::vector LLVMArgs; + std::string OutputPath; + enum FileType { + FT_Asm, ///< Assembly (.s) output, transliterate mode. + FT_Null, ///< No output, for timing purposes. + FT_Obj ///< Object file output. + }; + FileType OutputType; + unsigned ShowHelp : 1; + unsigned ShowVersion : 1; + + /// @} + /// @name Transliterate Options + /// @{ + + unsigned OutputAsmVariant; + unsigned ShowEncoding : 1; + unsigned ShowInst : 1; + + /// @} + /// @name Assembler Options + /// @{ + + unsigned RelaxAll : 1; + unsigned NoExecStack : 1; + unsigned FatalWarnings : 1; + unsigned IncrementalLinkerCompatible : 1; + + /// The name of the relocation model to use. + std::string RelocationModel; + + /// @} + +public: + AssemblerInvocation() { + Triple = ""; + NoInitialTextSection = 0; + InputFile = "-"; + OutputPath = "-"; + OutputType = FT_Asm; + OutputAsmVariant = 0; + ShowInst = 0; + ShowEncoding = 0; + RelaxAll = 0; + NoExecStack = 0; + FatalWarnings = 0; + IncrementalLinkerCompatible = 0; + DwarfVersion = 0; + } + + static bool CreateFromArgs(AssemblerInvocation &Res, + ArrayRef Argv, + DiagnosticsEngine &Diags); +}; + +} + +bool AssemblerInvocation::CreateFromArgs(AssemblerInvocation &Opts, + ArrayRef Argv, + DiagnosticsEngine &Diags) { + bool Success = true; + + // Parse the arguments. + std::unique_ptr OptTbl(createDriverOptTable()); + + const unsigned IncludedFlagsBitmask = options::CC1AsOption; + unsigned MissingArgIndex, MissingArgCount; + InputArgList Args = OptTbl->ParseArgs(Argv, MissingArgIndex, MissingArgCount, + IncludedFlagsBitmask); + + // Check for missing argument error. + if (MissingArgCount) { + Diags.Report(diag::err_drv_missing_argument) + << Args.getArgString(MissingArgIndex) << MissingArgCount; + Success = false; + } + + // Issue errors on unknown arguments. + for (const Arg *A : Args.filtered(OPT_UNKNOWN)) { + auto ArgString = A->getAsString(Args); + std::string Nearest; + if (OptTbl->findNearest(ArgString, Nearest, IncludedFlagsBitmask) > 1) + Diags.Report(diag::err_drv_unknown_argument) << ArgString; + else + Diags.Report(diag::err_drv_unknown_argument_with_suggestion) + << ArgString << Nearest; + Success = false; + } + + // Construct the invocation. + + // Target Options + Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple)); + Opts.CPU = Args.getLastArgValue(OPT_target_cpu); + Opts.Features = Args.getAllArgValues(OPT_target_feature); + + // Use the default target triple if unspecified. + if (Opts.Triple.empty()) + Opts.Triple = llvm::sys::getDefaultTargetTriple(); + + // Language Options + Opts.IncludePaths = Args.getAllArgValues(OPT_I); + Opts.NoInitialTextSection = Args.hasArg(OPT_n); + Opts.SaveTemporaryLabels = Args.hasArg(OPT_msave_temp_labels); + // Any DebugInfoKind implies GenDwarfForAssembly. + Opts.GenDwarfForAssembly = Args.hasArg(OPT_debug_info_kind_EQ); + + if (const Arg *A = Args.getLastArg(OPT_compress_debug_sections, + OPT_compress_debug_sections_EQ)) { + if (A->getOption().getID() == OPT_compress_debug_sections) { + // TODO: be more clever about the compression type auto-detection + Opts.CompressDebugSections = llvm::DebugCompressionType::GNU; + } else { + Opts.CompressDebugSections = + llvm::StringSwitch(A->getValue()) + .Case("none", llvm::DebugCompressionType::None) + .Case("zlib", llvm::DebugCompressionType::Z) + .Case("zlib-gnu", llvm::DebugCompressionType::GNU) + .Default(llvm::DebugCompressionType::None); + } + } + + Opts.RelaxELFRelocations = Args.hasArg(OPT_mrelax_relocations); + Opts.DwarfVersion = getLastArgIntValue(Args, OPT_dwarf_version_EQ, 2, Diags); + Opts.DwarfDebugFlags = Args.getLastArgValue(OPT_dwarf_debug_flags); + Opts.DwarfDebugProducer = Args.getLastArgValue(OPT_dwarf_debug_producer); + Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir); + Opts.MainFileName = Args.getLastArgValue(OPT_main_file_name); + + // Frontend Options + if (Args.hasArg(OPT_INPUT)) { + bool First = true; + for (const Arg *A : Args.filtered(OPT_INPUT)) { + if (First) { + Opts.InputFile = A->getValue(); + First = false; + } else { + Diags.Report(diag::err_drv_unknown_argument) << A->getAsString(Args); + Success = false; + } + } + } + Opts.LLVMArgs = Args.getAllArgValues(OPT_mllvm); + Opts.OutputPath = Args.getLastArgValue(OPT_o); + if (Arg *A = Args.getLastArg(OPT_filetype)) { + StringRef Name = A->getValue(); + unsigned OutputType = StringSwitch(Name) + .Case("asm", FT_Asm) + .Case("null", FT_Null) + .Case("obj", FT_Obj) + .Default(~0U); + if (OutputType == ~0U) { + Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name; + Success = false; + } else + Opts.OutputType = FileType(OutputType); + } + Opts.ShowHelp = Args.hasArg(OPT_help); + Opts.ShowVersion = Args.hasArg(OPT_version); + + // Transliterate Options + Opts.OutputAsmVariant = + getLastArgIntValue(Args, OPT_output_asm_variant, 0, Diags); + Opts.ShowEncoding = Args.hasArg(OPT_show_encoding); + Opts.ShowInst = Args.hasArg(OPT_show_inst); + + // Assemble Options + Opts.RelaxAll = Args.hasArg(OPT_mrelax_all); + Opts.NoExecStack = Args.hasArg(OPT_mno_exec_stack); + Opts.FatalWarnings = Args.hasArg(OPT_massembler_fatal_warnings); + Opts.RelocationModel = Args.getLastArgValue(OPT_mrelocation_model, "pic"); + Opts.IncrementalLinkerCompatible = + Args.hasArg(OPT_mincremental_linker_compatible); + Opts.SymbolDefs = Args.getAllArgValues(OPT_defsym); + + return Success; +} + +static std::unique_ptr +getOutputStream(AssemblerInvocation &Opts, DiagnosticsEngine &Diags, + bool Binary) { + if (Opts.OutputPath.empty()) + Opts.OutputPath = "-"; + + // Make sure that the Out file gets unlinked from the disk if we get a + // SIGINT. + if (Opts.OutputPath != "-") + sys::RemoveFileOnSignal(Opts.OutputPath); + + std::error_code EC; + auto Out = llvm::make_unique( + Opts.OutputPath, EC, (Binary ? sys::fs::F_None : sys::fs::F_Text)); + if (EC) { + Diags.Report(diag::err_fe_unable_to_open_output) << Opts.OutputPath + << EC.message(); + return nullptr; + } + + return Out; +} + +static bool ExecuteAssembler(AssemblerInvocation &Opts, + DiagnosticsEngine &Diags) { + // Get the target specific parser. + std::string Error; + const Target *TheTarget = TargetRegistry::lookupTarget(Opts.Triple, Error); + if (!TheTarget) + return Diags.Report(diag::err_target_unknown_triple) << Opts.Triple; + + ErrorOr> Buffer = + MemoryBuffer::getFileOrSTDIN(Opts.InputFile); + + if (std::error_code EC = Buffer.getError()) { + Error = EC.message(); + return Diags.Report(diag::err_fe_error_reading) << Opts.InputFile; + } + + SourceMgr SrcMgr; + + // Tell SrcMgr about this buffer, which is what the parser will pick up. + SrcMgr.AddNewSourceBuffer(std::move(*Buffer), SMLoc()); + + // Record the location of the include directories so that the lexer can find + // it later. + SrcMgr.setIncludeDirs(Opts.IncludePaths); + + std::unique_ptr MRI(TheTarget->createMCRegInfo(Opts.Triple)); + assert(MRI && "Unable to create target register info!"); + + std::unique_ptr MAI(TheTarget->createMCAsmInfo(*MRI, Opts.Triple)); + assert(MAI && "Unable to create target asm info!"); + + // Ensure MCAsmInfo initialization occurs before any use, otherwise sections + // may be created with a combination of default and explicit settings. + MAI->setCompressDebugSections(Opts.CompressDebugSections); + + MAI->setRelaxELFRelocations(Opts.RelaxELFRelocations); + + bool IsBinary = Opts.OutputType == AssemblerInvocation::FT_Obj; + std::unique_ptr FDOS = getOutputStream(Opts, Diags, IsBinary); + if (!FDOS) + return true; + + // FIXME: This is not pretty. MCContext has a ptr to MCObjectFileInfo and + // MCObjectFileInfo needs a MCContext reference in order to initialize itself. + std::unique_ptr MOFI(new MCObjectFileInfo()); + + MCContext Ctx(MAI.get(), MRI.get(), MOFI.get(), &SrcMgr); + + bool PIC = false; + if (Opts.RelocationModel == "static") { + PIC = false; + } else if (Opts.RelocationModel == "pic") { + PIC = true; + } else { + assert(Opts.RelocationModel == "dynamic-no-pic" && + "Invalid PIC model!"); + PIC = false; + } + + MOFI->InitMCObjectFileInfo(Triple(Opts.Triple), PIC, Ctx); + if (Opts.SaveTemporaryLabels) + Ctx.setAllowTemporaryLabels(false); + if (Opts.GenDwarfForAssembly) + Ctx.setGenDwarfForAssembly(true); + if (!Opts.DwarfDebugFlags.empty()) + Ctx.setDwarfDebugFlags(StringRef(Opts.DwarfDebugFlags)); + if (!Opts.DwarfDebugProducer.empty()) + Ctx.setDwarfDebugProducer(StringRef(Opts.DwarfDebugProducer)); + if (!Opts.DebugCompilationDir.empty()) + Ctx.setCompilationDir(Opts.DebugCompilationDir); + if (!Opts.MainFileName.empty()) + Ctx.setMainFileName(StringRef(Opts.MainFileName)); + Ctx.setDwarfVersion(Opts.DwarfVersion); + + // Build up the feature string from the target feature list. + std::string FS; + if (!Opts.Features.empty()) { + FS = Opts.Features[0]; + for (unsigned i = 1, e = Opts.Features.size(); i != e; ++i) + FS += "," + Opts.Features[i]; + } + + std::unique_ptr Str; + + std::unique_ptr MCII(TheTarget->createMCInstrInfo()); + std::unique_ptr STI( + TheTarget->createMCSubtargetInfo(Opts.Triple, Opts.CPU, FS)); + + raw_pwrite_stream *Out = FDOS.get(); + std::unique_ptr BOS; + + // FIXME: There is a bit of code duplication with addPassesToEmitFile. + if (Opts.OutputType == AssemblerInvocation::FT_Asm) { + MCInstPrinter *IP = TheTarget->createMCInstPrinter( + llvm::Triple(Opts.Triple), Opts.OutputAsmVariant, *MAI, *MCII, *MRI); + + std::unique_ptr CE; + if (Opts.ShowEncoding) + CE.reset(TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx)); + MCTargetOptions MCOptions; + std::unique_ptr MAB( + TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions)); + + auto FOut = llvm::make_unique(*Out); + Str.reset(TheTarget->createAsmStreamer( + Ctx, std::move(FOut), /*asmverbose*/ true, + /*useDwarfDirectory*/ true, IP, std::move(CE), std::move(MAB), + Opts.ShowInst)); + } else if (Opts.OutputType == AssemblerInvocation::FT_Null) { + Str.reset(createNullStreamer(Ctx)); + } else { + assert(Opts.OutputType == AssemblerInvocation::FT_Obj && + "Invalid file type!"); + if (!FDOS->supportsSeeking()) { + BOS = make_unique(*FDOS); + Out = BOS.get(); + } + + std::unique_ptr CE( + TheTarget->createMCCodeEmitter(*MCII, *MRI, Ctx)); + MCTargetOptions MCOptions; + std::unique_ptr MAB( + TheTarget->createMCAsmBackend(*STI, *MRI, MCOptions)); + + Triple T(Opts.Triple); + Str.reset(TheTarget->createMCObjectStreamer( + T, Ctx, std::move(MAB), *Out, std::move(CE), *STI, Opts.RelaxAll, + Opts.IncrementalLinkerCompatible, + /*DWARFMustBeAtTheEnd*/ true)); + Str.get()->InitSections(Opts.NoExecStack); + } + + // Assembly to object compilation should leverage assembly info. + Str->setUseAssemblerInfoForParsing(true); + + bool Failed = false; + + std::unique_ptr Parser( + createMCAsmParser(SrcMgr, Ctx, *Str.get(), *MAI)); + + // FIXME: init MCTargetOptions from sanitizer flags here. + MCTargetOptions Options; + std::unique_ptr TAP( + TheTarget->createMCAsmParser(*STI, *Parser, *MCII, Options)); + if (!TAP) + Failed = Diags.Report(diag::err_target_unknown_triple) << Opts.Triple; + + // Set values for symbols, if any. + for (auto &S : Opts.SymbolDefs) { + auto Pair = StringRef(S).split('='); + auto Sym = Pair.first; + auto Val = Pair.second; + int64_t Value; + // We have already error checked this in the driver. + Val.getAsInteger(0, Value); + Ctx.setSymbolValue(Parser->getStreamer(), Sym, Value); + } + + if (!Failed) { + Parser->setTargetParser(*TAP.get()); + Failed = Parser->Run(Opts.NoInitialTextSection); + } + + // Close Streamer first. + // It might have a reference to the output stream. + Str.reset(); + // Close the output stream early. + BOS.reset(); + FDOS.reset(); + + // Delete output file if there were errors. + if (Failed && Opts.OutputPath != "-") + sys::fs::remove(Opts.OutputPath); + + return Failed; +} + +static void LLVMErrorHandler(void *UserData, const std::string &Message, + bool /*GenCrashDiag*/) { + DiagnosticsEngine &Diags = *static_cast(UserData); + + Diags.Report(diag::err_fe_error_backend) << Message; + + // We cannot recover from llvm errors. + exit(1); +} + +int cc1as_main(ArrayRef Argv, const char */*Argv0*/, void */*MainAddr*/) { + // Initialize targets and assembly printers/parsers. + InitializeAllTargetInfos(); + InitializeAllTargetMCs(); + InitializeAllAsmParsers(); + + // Construct our diagnostic client. + IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); + TextDiagnosticPrinter *DiagClient + = new TextDiagnosticPrinter(errs(), &*DiagOpts); + DiagClient->setPrefix("clang -cc1as"); + IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); + DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient); + + // Set an error handler, so that any LLVM backend diagnostics go through our + // error handler. + ScopedFatalErrorHandler FatalErrorHandler + (LLVMErrorHandler, static_cast(&Diags)); + + // Parse the arguments. + AssemblerInvocation Asm; + if (!AssemblerInvocation::CreateFromArgs(Asm, Argv, Diags)) + return 1; + + if (Asm.ShowHelp) { + std::unique_ptr Opts(driver::createDriverOptTable()); + Opts->PrintHelp(llvm::outs(), "clang -cc1as", "Clang Integrated Assembler", + /*Include=*/driver::options::CC1AsOption, /*Exclude=*/0, + /*ShowAllAliases=*/false); + return 0; + } + + // Honor -version. + // + // FIXME: Use a better -version message? + if (Asm.ShowVersion) { + llvm::cl::PrintVersionMessage(); + return 0; + } + + // Honor -mllvm. + // + // FIXME: Remove this, one day. + if (!Asm.LLVMArgs.empty()) { + unsigned NumArgs = Asm.LLVMArgs.size(); + auto Args = llvm::make_unique(NumArgs + 2); + Args[0] = "clang (LLVM option parsing)"; + for (unsigned i = 0; i != NumArgs; ++i) + Args[i + 1] = Asm.LLVMArgs[i].c_str(); + Args[NumArgs + 1] = nullptr; + llvm::cl::ParseCommandLineOptions(NumArgs + 1, Args.get()); + } + + // Execute the invocation, unless there were parsing errors. + bool Failed = Diags.hasErrorOccurred() || ExecuteAssembler(Asm, Diags); + + // If any timers were active but haven't been destroyed yet, print their + // results now. + TimerGroup::printAll(errs()); + + return !!Failed; +} diff --git a/dbms/programs/clang/Compiler-7.0.0bundled/driver.cpp b/dbms/programs/clang/Compiler-7.0.0bundled/driver.cpp new file mode 100644 index 00000000000..9a061b9d137 --- /dev/null +++ b/dbms/programs/clang/Compiler-7.0.0bundled/driver.cpp @@ -0,0 +1,512 @@ +//===-- driver.cpp - Clang GCC-Compatible Driver --------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This is the entry point to the clang driver; it is a thin wrapper +// for functionality in the Driver clang library. +// +//===----------------------------------------------------------------------===// + +#include "clang/Driver/Driver.h" +#include "clang/Basic/DiagnosticOptions.h" +#include "clang/Driver/Compilation.h" +#include "clang/Driver/DriverDiagnostic.h" +#include "clang/Driver/Options.h" +#include "clang/Driver/ToolChain.h" +#include "clang/Frontend/ChainedDiagnosticConsumer.h" +#include "clang/Frontend/CompilerInvocation.h" +#include "clang/Frontend/SerializedDiagnosticPrinter.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" +#include "clang/Frontend/Utils.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Option/ArgList.h" +#include "llvm/Option/OptTable.h" +#include "llvm/Option/Option.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Host.h" +#include "llvm/Support/InitLLVM.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/Program.h" +#include "llvm/Support/Regex.h" +#include "llvm/Support/Signals.h" +#include "llvm/Support/StringSaver.h" +#include "llvm/Support/TargetSelect.h" +#include "llvm/Support/Timer.h" +#include "llvm/Support/raw_ostream.h" +#include +#include +#include +using namespace clang; +using namespace clang::driver; +using namespace llvm::opt; + +std::string GetExecutablePath(const char *Argv0, bool CanonicalPrefixes) { + if (!CanonicalPrefixes) { + SmallString<128> ExecutablePath(Argv0); + // Do a PATH lookup if Argv0 isn't a valid path. + if (!llvm::sys::fs::exists(ExecutablePath)) + if (llvm::ErrorOr P = + llvm::sys::findProgramByName(ExecutablePath)) + ExecutablePath = *P; + return ExecutablePath.str(); + } + + // This just needs to be some symbol in the binary; C++ doesn't + // allow taking the address of ::main however. + void *P = (void*) (intptr_t) GetExecutablePath; + return llvm::sys::fs::getMainExecutable(Argv0, P); +} + +static const char *GetStableCStr(std::set &SavedStrings, + StringRef S) { + return SavedStrings.insert(S).first->c_str(); +} + +/// ApplyQAOverride - Apply a list of edits to the input argument lists. +/// +/// The input string is a space separate list of edits to perform, +/// they are applied in order to the input argument lists. Edits +/// should be one of the following forms: +/// +/// '#': Silence information about the changes to the command line arguments. +/// +/// '^': Add FOO as a new argument at the beginning of the command line. +/// +/// '+': Add FOO as a new argument at the end of the command line. +/// +/// 's/XXX/YYY/': Substitute the regular expression XXX with YYY in the command +/// line. +/// +/// 'xOPTION': Removes all instances of the literal argument OPTION. +/// +/// 'XOPTION': Removes all instances of the literal argument OPTION, +/// and the following argument. +/// +/// 'Ox': Removes all flags matching 'O' or 'O[sz0-9]' and adds 'Ox' +/// at the end of the command line. +/// +/// \param OS - The stream to write edit information to. +/// \param Args - The vector of command line arguments. +/// \param Edit - The override command to perform. +/// \param SavedStrings - Set to use for storing string representations. +static void ApplyOneQAOverride(raw_ostream &OS, + SmallVectorImpl &Args, + StringRef Edit, + std::set &SavedStrings) { + // This does not need to be efficient. + + if (Edit[0] == '^') { + const char *Str = + GetStableCStr(SavedStrings, Edit.substr(1)); + OS << "### Adding argument " << Str << " at beginning\n"; + Args.insert(Args.begin() + 1, Str); + } else if (Edit[0] == '+') { + const char *Str = + GetStableCStr(SavedStrings, Edit.substr(1)); + OS << "### Adding argument " << Str << " at end\n"; + Args.push_back(Str); + } else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") && + Edit.slice(2, Edit.size()-1).find('/') != StringRef::npos) { + StringRef MatchPattern = Edit.substr(2).split('/').first; + StringRef ReplPattern = Edit.substr(2).split('/').second; + ReplPattern = ReplPattern.slice(0, ReplPattern.size()-1); + + for (unsigned i = 1, e = Args.size(); i != e; ++i) { + // Ignore end-of-line response file markers + if (Args[i] == nullptr) + continue; + std::string Repl = llvm::Regex(MatchPattern).sub(ReplPattern, Args[i]); + + if (Repl != Args[i]) { + OS << "### Replacing '" << Args[i] << "' with '" << Repl << "'\n"; + Args[i] = GetStableCStr(SavedStrings, Repl); + } + } + } else if (Edit[0] == 'x' || Edit[0] == 'X') { + auto Option = Edit.substr(1); + for (unsigned i = 1; i < Args.size();) { + if (Option == Args[i]) { + OS << "### Deleting argument " << Args[i] << '\n'; + Args.erase(Args.begin() + i); + if (Edit[0] == 'X') { + if (i < Args.size()) { + OS << "### Deleting argument " << Args[i] << '\n'; + Args.erase(Args.begin() + i); + } else + OS << "### Invalid X edit, end of command line!\n"; + } + } else + ++i; + } + } else if (Edit[0] == 'O') { + for (unsigned i = 1; i < Args.size();) { + const char *A = Args[i]; + // Ignore end-of-line response file markers + if (A == nullptr) + continue; + if (A[0] == '-' && A[1] == 'O' && + (A[2] == '\0' || + (A[3] == '\0' && (A[2] == 's' || A[2] == 'z' || + ('0' <= A[2] && A[2] <= '9'))))) { + OS << "### Deleting argument " << Args[i] << '\n'; + Args.erase(Args.begin() + i); + } else + ++i; + } + OS << "### Adding argument " << Edit << " at end\n"; + Args.push_back(GetStableCStr(SavedStrings, '-' + Edit.str())); + } else { + OS << "### Unrecognized edit: " << Edit << "\n"; + } +} + +/// ApplyQAOverride - Apply a comma separate list of edits to the +/// input argument lists. See ApplyOneQAOverride. +static void ApplyQAOverride(SmallVectorImpl &Args, + const char *OverrideStr, + std::set &SavedStrings) { + raw_ostream *OS = &llvm::errs(); + + if (OverrideStr[0] == '#') { + ++OverrideStr; + OS = &llvm::nulls(); + } + + *OS << "### CCC_OVERRIDE_OPTIONS: " << OverrideStr << "\n"; + + // This does not need to be efficient. + + const char *S = OverrideStr; + while (*S) { + const char *End = ::strchr(S, ' '); + if (!End) + End = S + strlen(S); + if (End != S) + ApplyOneQAOverride(*OS, Args, std::string(S, End), SavedStrings); + S = End; + if (*S != '\0') + ++S; + } +} + +extern int cc1_main(ArrayRef Argv, const char *Argv0, + void *MainAddr); +extern int cc1as_main(ArrayRef Argv, const char *Argv0, + void *MainAddr); +extern int cc1gen_reproducer_main(ArrayRef Argv, + const char *Argv0, void *MainAddr); + +static void insertTargetAndModeArgs(const ParsedClangName &NameParts, + SmallVectorImpl &ArgVector, + std::set &SavedStrings) { + // Put target and mode arguments at the start of argument list so that + // arguments specified in command line could override them. Avoid putting + // them at index 0, as an option like '-cc1' must remain the first. + int InsertionPoint = 0; + if (ArgVector.size() > 0) + ++InsertionPoint; + + if (NameParts.DriverMode) { + // Add the mode flag to the arguments. + ArgVector.insert(ArgVector.begin() + InsertionPoint, + GetStableCStr(SavedStrings, NameParts.DriverMode)); + } + + if (NameParts.TargetIsValid) { + const char *arr[] = {"-target", GetStableCStr(SavedStrings, + NameParts.TargetPrefix)}; + ArgVector.insert(ArgVector.begin() + InsertionPoint, + std::begin(arr), std::end(arr)); + } +} + +static void getCLEnvVarOptions(std::string &EnvValue, llvm::StringSaver &Saver, + SmallVectorImpl &Opts) { + llvm::cl::TokenizeWindowsCommandLine(EnvValue, Saver, Opts); + // The first instance of '#' should be replaced with '=' in each option. + for (const char *Opt : Opts) + if (char *NumberSignPtr = const_cast(::strchr(Opt, '#'))) + *NumberSignPtr = '='; +} + +static void SetBackdoorDriverOutputsFromEnvVars(Driver &TheDriver) { + // Handle CC_PRINT_OPTIONS and CC_PRINT_OPTIONS_FILE. + TheDriver.CCPrintOptions = !!::getenv("CC_PRINT_OPTIONS"); + if (TheDriver.CCPrintOptions) + TheDriver.CCPrintOptionsFilename = ::getenv("CC_PRINT_OPTIONS_FILE"); + + // Handle CC_PRINT_HEADERS and CC_PRINT_HEADERS_FILE. + TheDriver.CCPrintHeaders = !!::getenv("CC_PRINT_HEADERS"); + if (TheDriver.CCPrintHeaders) + TheDriver.CCPrintHeadersFilename = ::getenv("CC_PRINT_HEADERS_FILE"); + + // Handle CC_LOG_DIAGNOSTICS and CC_LOG_DIAGNOSTICS_FILE. + TheDriver.CCLogDiagnostics = !!::getenv("CC_LOG_DIAGNOSTICS"); + if (TheDriver.CCLogDiagnostics) + TheDriver.CCLogDiagnosticsFilename = ::getenv("CC_LOG_DIAGNOSTICS_FILE"); +} + +static void FixupDiagPrefixExeName(TextDiagnosticPrinter *DiagClient, + const std::string &Path) { + // If the clang binary happens to be named cl.exe for compatibility reasons, + // use clang-cl.exe as the prefix to avoid confusion between clang and MSVC. + StringRef ExeBasename(llvm::sys::path::filename(Path)); + if (ExeBasename.equals_lower("cl.exe")) + ExeBasename = "clang-cl.exe"; + DiagClient->setPrefix(ExeBasename); +} + +// This lets us create the DiagnosticsEngine with a properly-filled-out +// DiagnosticOptions instance. +static DiagnosticOptions * +CreateAndPopulateDiagOpts(ArrayRef argv) { + auto *DiagOpts = new DiagnosticOptions; + std::unique_ptr Opts(createDriverOptTable()); + unsigned MissingArgIndex, MissingArgCount; + InputArgList Args = + Opts->ParseArgs(argv.slice(1), MissingArgIndex, MissingArgCount); + // We ignore MissingArgCount and the return value of ParseDiagnosticArgs. + // Any errors that would be diagnosed here will also be diagnosed later, + // when the DiagnosticsEngine actually exists. + (void)ParseDiagnosticArgs(*DiagOpts, Args); + return DiagOpts; +} + +static void SetInstallDir(SmallVectorImpl &argv, + Driver &TheDriver, bool CanonicalPrefixes) { + // Attempt to find the original path used to invoke the driver, to determine + // the installed path. We do this manually, because we want to support that + // path being a symlink. + SmallString<128> InstalledPath(argv[0]); + + // Do a PATH lookup, if there are no directory components. + if (llvm::sys::path::filename(InstalledPath) == InstalledPath) + if (llvm::ErrorOr Tmp = llvm::sys::findProgramByName( + llvm::sys::path::filename(InstalledPath.str()))) + InstalledPath = *Tmp; + + // FIXME: We don't actually canonicalize this, we just make it absolute. + if (CanonicalPrefixes) + llvm::sys::fs::make_absolute(InstalledPath); + + StringRef InstalledPathParent(llvm::sys::path::parent_path(InstalledPath)); + if (llvm::sys::fs::exists(InstalledPathParent)) + TheDriver.setInstalledDir(InstalledPathParent); +} + +static int ExecuteCC1Tool(ArrayRef argv, StringRef Tool) { + void *GetExecutablePathVP = (void *)(intptr_t) GetExecutablePath; + if (Tool == "") + return cc1_main(argv.slice(2), argv[0], GetExecutablePathVP); + if (Tool == "as") + return cc1as_main(argv.slice(2), argv[0], GetExecutablePathVP); + + // Reject unknown tools. + llvm::errs() << "error: unknown integrated tool '" << Tool << "'. " + << "Valid tools include '-cc1' and '-cc1as'.\n"; + return 1; +} + +int mainEntryClickHouseClang(int argc_, char **argv_) { + llvm::InitLLVM X(argc_, argv_); + SmallVector argv(argv_, argv_ + argc_); + + if (llvm::sys::Process::FixupStandardFileDescriptors()) + return 1; + + llvm::InitializeAllTargets(); + auto TargetAndMode = ToolChain::getTargetAndModeFromProgramName(argv[0]); + + llvm::BumpPtrAllocator A; + llvm::StringSaver Saver(A); + + // Parse response files using the GNU syntax, unless we're in CL mode. There + // are two ways to put clang in CL compatibility mode: argv[0] is either + // clang-cl or cl, or --driver-mode=cl is on the command line. The normal + // command line parsing can't happen until after response file parsing, so we + // have to manually search for a --driver-mode=cl argument the hard way. + // Finally, our -cc1 tools don't care which tokenization mode we use because + // response files written by clang will tokenize the same way in either mode. + bool ClangCLMode = false; + if (StringRef(TargetAndMode.DriverMode).equals("--driver-mode=cl") || + std::find_if(argv.begin(), argv.end(), [](const char *F) { + return F && strcmp(F, "--driver-mode=cl") == 0; + }) != argv.end()) { + ClangCLMode = true; + } + enum { Default, POSIX, Windows } RSPQuoting = Default; + for (const char *F : argv) { + if (strcmp(F, "--rsp-quoting=posix") == 0) + RSPQuoting = POSIX; + else if (strcmp(F, "--rsp-quoting=windows") == 0) + RSPQuoting = Windows; + } + + // Determines whether we want nullptr markers in argv to indicate response + // files end-of-lines. We only use this for the /LINK driver argument with + // clang-cl.exe on Windows. + bool MarkEOLs = ClangCLMode; + + llvm::cl::TokenizerCallback Tokenizer; + if (RSPQuoting == Windows || (RSPQuoting == Default && ClangCLMode)) + Tokenizer = &llvm::cl::TokenizeWindowsCommandLine; + else + Tokenizer = &llvm::cl::TokenizeGNUCommandLine; + + if (MarkEOLs && argv.size() > 1 && StringRef(argv[1]).startswith("-cc1")) + MarkEOLs = false; + llvm::cl::ExpandResponseFiles(Saver, Tokenizer, argv, MarkEOLs); + + // Handle -cc1 integrated tools, even if -cc1 was expanded from a response + // file. + auto FirstArg = std::find_if(argv.begin() + 1, argv.end(), + [](const char *A) { return A != nullptr; }); + if (FirstArg != argv.end() && StringRef(*FirstArg).startswith("-cc1")) { + // If -cc1 came from a response file, remove the EOL sentinels. + if (MarkEOLs) { + auto newEnd = std::remove(argv.begin(), argv.end(), nullptr); + argv.resize(newEnd - argv.begin()); + } + return ExecuteCC1Tool(argv, argv[1] + 4); + } + + bool CanonicalPrefixes = true; + for (int i = 1, size = argv.size(); i < size; ++i) { + // Skip end-of-line response file markers + if (argv[i] == nullptr) + continue; + if (StringRef(argv[i]) == "-no-canonical-prefixes") { + CanonicalPrefixes = false; + break; + } + } + + // Handle CL and _CL_ which permits additional command line options to be + // prepended or appended. + if (ClangCLMode) { + // Arguments in "CL" are prepended. + llvm::Optional OptCL = llvm::sys::Process::GetEnv("CL"); + if (OptCL.hasValue()) { + SmallVector PrependedOpts; + getCLEnvVarOptions(OptCL.getValue(), Saver, PrependedOpts); + + // Insert right after the program name to prepend to the argument list. + argv.insert(argv.begin() + 1, PrependedOpts.begin(), PrependedOpts.end()); + } + // Arguments in "_CL_" are appended. + llvm::Optional Opt_CL_ = llvm::sys::Process::GetEnv("_CL_"); + if (Opt_CL_.hasValue()) { + SmallVector AppendedOpts; + getCLEnvVarOptions(Opt_CL_.getValue(), Saver, AppendedOpts); + + // Insert at the end of the argument list to append. + argv.append(AppendedOpts.begin(), AppendedOpts.end()); + } + } + + std::set SavedStrings; + // Handle CCC_OVERRIDE_OPTIONS, used for editing a command line behind the + // scenes. + if (const char *OverrideStr = ::getenv("CCC_OVERRIDE_OPTIONS")) { + // FIXME: Driver shouldn't take extra initial argument. + ApplyQAOverride(argv, OverrideStr, SavedStrings); + } + + std::string Path = GetExecutablePath(argv[0], CanonicalPrefixes); + + IntrusiveRefCntPtr DiagOpts = + CreateAndPopulateDiagOpts(argv); + + TextDiagnosticPrinter *DiagClient + = new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts); + FixupDiagPrefixExeName(DiagClient, Path); + + IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); + + DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagClient); + + if (!DiagOpts->DiagnosticSerializationFile.empty()) { + auto SerializedConsumer = + clang::serialized_diags::create(DiagOpts->DiagnosticSerializationFile, + &*DiagOpts, /*MergeChildRecords=*/true); + Diags.setClient(new ChainedDiagnosticConsumer( + Diags.takeClient(), std::move(SerializedConsumer))); + } + + ProcessWarningOptions(Diags, *DiagOpts, /*ReportDiags=*/false); + + Driver TheDriver(Path, llvm::sys::getDefaultTargetTriple(), Diags); + SetInstallDir(argv, TheDriver, CanonicalPrefixes); + TheDriver.setTargetAndMode(TargetAndMode); + + insertTargetAndModeArgs(TargetAndMode, argv, SavedStrings); + + SetBackdoorDriverOutputsFromEnvVars(TheDriver); + + std::unique_ptr C(TheDriver.BuildCompilation(argv)); + int Res = 1; + if (C && !C->containsError()) { + SmallVector, 4> FailingCommands; + Res = TheDriver.ExecuteCompilation(*C, FailingCommands); + + // Force a crash to test the diagnostics. + if (TheDriver.GenReproducer) { + Diags.Report(diag::err_drv_force_crash) + << !::getenv("FORCE_CLANG_DIAGNOSTICS_CRASH"); + + // Pretend that every command failed. + FailingCommands.clear(); + for (const auto &J : C->getJobs()) + if (const Command *C = dyn_cast(&J)) + FailingCommands.push_back(std::make_pair(-1, C)); + } + + for (const auto &P : FailingCommands) { + int CommandRes = P.first; + const Command *FailingCommand = P.second; + if (!Res) + Res = CommandRes; + + // If result status is < 0, then the driver command signalled an error. + // If result status is 70, then the driver command reported a fatal error. + // On Windows, abort will return an exit code of 3. In these cases, + // generate additional diagnostic information if possible. + bool DiagnoseCrash = CommandRes < 0 || CommandRes == 70; +#ifdef _WIN32 + DiagnoseCrash |= CommandRes == 3; +#endif + if (DiagnoseCrash) { + TheDriver.generateCompilationDiagnostics(*C, *FailingCommand); + break; + } + } + } + + Diags.getClient()->finish(); + + // If any timers were active but haven't been destroyed yet, print their + // results now. This happens in -disable-free mode. + llvm::TimerGroup::printAll(llvm::errs()); + +#ifdef _WIN32 + // Exit status should not be negative on Win32, unless abnormal termination. + // Once abnormal termiation was caught, negative status should not be + // propagated. + if (Res < 0) + Res = 1; +#endif + + // If we have multiple failing commands, we return the result of the first + // failing command. + return Res; +} diff --git a/dbms/programs/clang/Compiler-7.0.0bundled/lld.cpp b/dbms/programs/clang/Compiler-7.0.0bundled/lld.cpp new file mode 100644 index 00000000000..203e50d42a9 --- /dev/null +++ b/dbms/programs/clang/Compiler-7.0.0bundled/lld.cpp @@ -0,0 +1,10 @@ +#include "lld/Common/Driver.h" +#include "llvm/Support/InitLLVM.h" +#include + +int mainEntryClickHouseLLD(int argc, char ** argv) +{ + llvm::InitLLVM X(argc, argv); + std::vector args(argv, argv + argc); + return !lld::elf::link(args, false); +} diff --git a/dbms/src/Server/Compiler-7.0.0svn b/dbms/programs/clang/Compiler-7.0.0svn similarity index 100% rename from dbms/src/Server/Compiler-7.0.0svn rename to dbms/programs/clang/Compiler-7.0.0svn diff --git a/dbms/programs/clang/Compiler-7.0.1 b/dbms/programs/clang/Compiler-7.0.1 new file mode 120000 index 00000000000..eeeb5bbc2c0 --- /dev/null +++ b/dbms/programs/clang/Compiler-7.0.1 @@ -0,0 +1 @@ +Compiler-7.0.0 \ No newline at end of file diff --git a/dbms/src/Server/clickhouse-clang.cpp b/dbms/programs/clang/clickhouse-clang.cpp similarity index 100% rename from dbms/src/Server/clickhouse-clang.cpp rename to dbms/programs/clang/clickhouse-clang.cpp diff --git a/dbms/src/Server/clickhouse-lld.cpp b/dbms/programs/clang/clickhouse-lld.cpp similarity index 100% rename from dbms/src/Server/clickhouse-lld.cpp rename to dbms/programs/clang/clickhouse-lld.cpp diff --git a/dbms/programs/client/CMakeLists.txt b/dbms/programs/client/CMakeLists.txt new file mode 100644 index 00000000000..f3dd518e891 --- /dev/null +++ b/dbms/programs/client/CMakeLists.txt @@ -0,0 +1,12 @@ +add_library (clickhouse-client-lib ${LINK_MODE} Client.cpp) +target_link_libraries (clickhouse-client-lib PRIVATE clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY}) +if (READLINE_INCLUDE_DIR) + target_include_directories (clickhouse-client-lib SYSTEM PRIVATE ${READLINE_INCLUDE_DIR}) +endif () + +if (CLICKHOUSE_SPLIT_BINARY) + add_executable (clickhouse-client clickhouse-client.cpp) + target_link_libraries (clickhouse-client PRIVATE clickhouse-client-lib) +endif () + +install (FILES clickhouse-client.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-client COMPONENT clickhouse-client RENAME config.xml) diff --git a/dbms/src/Server/Client.cpp b/dbms/programs/client/Client.cpp similarity index 76% rename from dbms/src/Server/Client.cpp rename to dbms/programs/client/Client.cpp index eb0e4482a37..1b7e48ac55f 100644 --- a/dbms/src/Server/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -1,3 +1,6 @@ +#include "TestHint.h" +#include "ConnectionParameters.h" + #include #include #include @@ -11,15 +14,15 @@ #include #include #include +#include #include #include #include -#include +#include #include #include #include #include -#include #include #include #include @@ -27,8 +30,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -36,7 +41,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -49,9 +56,18 @@ #include #include #include -#include "InterruptListener.h" +#include #include #include +#include + +#if USE_READLINE +#include "Suggest.h" // Y_IGNORE +#endif + +#ifndef __clang__ +#pragma GCC optimize("-fno-var-tracking-assignments") +#endif /// http://en.wikipedia.org/wiki/ANSI_escape_code @@ -66,15 +82,11 @@ #define DISABLE_LINE_WRAPPING "\033[?7l" #define ENABLE_LINE_WRAPPING "\033[?7h" - namespace DB { namespace ErrorCodes { - extern const int POCO_EXCEPTION; - extern const int STD_EXCEPTION; - extern const int UNKNOWN_EXCEPTION; extern const int NETWORK_ERROR; extern const int NO_DATA_TO_INSERT; extern const int BAD_ARGUMENTS; @@ -83,6 +95,9 @@ namespace ErrorCodes extern const int UNKNOWN_PACKET_FROM_SERVER; extern const int UNEXPECTED_PACKET_FROM_SERVER; extern const int CLIENT_OUTPUT_FORMAT_SPECIFIED; + extern const int LOGICAL_ERROR; + extern const int CANNOT_SET_SIGNAL_HANDLER; + extern const int CANNOT_READLINE; } @@ -104,6 +119,7 @@ private: bool is_interactive = true; /// Use either readline interface or batch mode. bool need_render_progress = true; /// Render query execution progress. bool echo_queries = false; /// Print queries before execution in batch mode. + bool ignore_error = false; /// In case of errors, don't print error message, continue to next query. Only applicable for non-interactive mode. bool print_time_to_stderr = false; /// Output execution time to stderr in batch mode. bool stdin_is_not_tty = false; /// stdin is not a terminal. @@ -134,6 +150,11 @@ private: std::optional out_file_buf; BlockOutputStreamPtr block_out_stream; + /// The user could specify special file for server logs (stderr by default) + std::unique_ptr out_logs_buf; + String server_logs_file; + BlockOutputStreamPtr logs_out_stream; + String home_path; String current_profile; @@ -154,6 +175,12 @@ private: /// If the last query resulted in exception. bool got_exception = false; + int expected_server_error = 0; + int expected_client_error = 0; + int actual_server_error = 0; + int actual_client_error = 0; + + UInt64 server_revision = 0; String server_version; String server_display_name; @@ -169,47 +196,6 @@ private: /// External tables info. std::list external_tables; - - struct ConnectionParameters - { - String host; - UInt16 port; - String default_database; - String user; - String password; - Protocol::Encryption security; - Protocol::Compression compression; - ConnectionTimeouts timeouts; - - ConnectionParameters() {} - - ConnectionParameters(const Poco::Util::AbstractConfiguration & config) - { - bool is_secure = config.getBool("secure", false); - security = is_secure - ? Protocol::Encryption::Enable - : Protocol::Encryption::Disable; - - host = config.getString("host", "localhost"); - port = config.getInt("port", - config.getInt(is_secure ? "tcp_secure_port" : "tcp_port", - is_secure ? DBMS_DEFAULT_SECURE_PORT : DBMS_DEFAULT_PORT)); - - default_database = config.getString("database", ""); - user = config.getString("user", ""); - password = config.getString("password", ""); - - compression = config.getBool("compression", true) - ? Protocol::Compression::Enable - : Protocol::Compression::Disable; - - timeouts = ConnectionTimeouts( - Poco::Timespan(config.getInt("connect_timeout", DBMS_DEFAULT_CONNECT_TIMEOUT_SEC), 0), - Poco::Timespan(config.getInt("receive_timeout", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC), 0), - Poco::Timespan(config.getInt("send_timeout", DBMS_DEFAULT_SEND_TIMEOUT_SEC), 0)); - } - }; - ConnectionParameters connection_parameters; @@ -221,22 +207,7 @@ private: if (home_path_cstr) home_path = home_path_cstr; - std::string config_path; - if (config().has("config-file")) - config_path = config().getString("config-file"); - else if (Poco::File("./clickhouse-client.xml").exists()) - config_path = "./clickhouse-client.xml"; - else if (!home_path.empty() && Poco::File(home_path + "/.clickhouse-client/config.xml").exists()) - config_path = home_path + "/.clickhouse-client/config.xml"; - else if (Poco::File("/etc/clickhouse-client/config.xml").exists()) - config_path = "/etc/clickhouse-client/config.xml"; - - if (!config_path.empty()) - { - ConfigProcessor config_processor(config_path); - auto loaded_config = config_processor.loadConfig(); - config().add(loaded_config.configuration); - } + configReadClient(config(), home_path); context.setApplicationType(Context::ApplicationType::CLIENT); @@ -285,20 +256,10 @@ private: /// If exception code isn't zero, we should return non-zero return code anyway. return e.code() ? e.code() : -1; } - catch (const Poco::Exception & e) - { - std::cerr << "Poco::Exception: " << e.displayText() << std::endl; - return ErrorCodes::POCO_EXCEPTION; - } - catch (const std::exception & e) - { - std::cerr << "std::exception: " << e.what() << std::endl; - return ErrorCodes::STD_EXCEPTION; - } catch (...) { - std::cerr << "Unknown exception" << std::endl; - return ErrorCodes::UNKNOWN_EXCEPTION; + std::cerr << getCurrentExceptionMessage(false) << std::endl; + return getCurrentExceptionCode(); } } @@ -316,9 +277,10 @@ private: || (now.month() == 1 && now.day() <= 5); } - int mainImpl() { + UseSSL use_ssl; + registerFunctions(); registerAggregateFunctions(); @@ -346,15 +308,20 @@ private: format_max_block_size = config().getInt("format_max_block_size", context.getSettingsRef().max_block_size); insert_format = "Values"; - insert_format_max_block_size = config().getInt("insert_format_max_block_size", context.getSettingsRef().max_insert_block_size); + + /// Setting value from cmd arg overrides one from config + if (context.getSettingsRef().max_insert_block_size.changed) + insert_format_max_block_size = context.getSettingsRef().max_insert_block_size; + else + insert_format_max_block_size = config().getInt("insert_format_max_block_size", context.getSettingsRef().max_insert_block_size); if (!is_interactive) { need_render_progress = config().getBool("progress", false); echo_queries = config().getBool("echo", false); + ignore_error = config().getBool("ignore-error", false); } - connection_parameters = ConnectionParameters(config()); connect(); /// Initialize DateLUT here to avoid counting time spent here as query execution time. @@ -427,9 +394,25 @@ private: if (print_time_to_stderr) throw Exception("time option could be specified only in non-interactive mode", ErrorCodes::BAD_ARGUMENTS); - /// Turn tab completion off. - rl_bind_key('\t', rl_insert); +#if USE_READLINE + if (server_revision >= Suggest::MIN_SERVER_REVISION + && !config().getBool("disable_suggestion", false)) + { + /// Load suggestion data from the server. + Suggest::instance().load(connection_parameters, config().getInt("suggestion_limit")); + /// Added '.' to the default list. Because it is used to separate database and table. + rl_basic_word_break_characters = " \t\n\r\"\\'`@$><=;|&{(."; + + /// Not append whitespace after single suggestion. Because whitespace after function name is meaningless. + rl_completion_append_character = '\0'; + + rl_completion_entry_function = Suggest::generator; + } + else + /// Turn tab completion off. + rl_bind_key('\t', rl_insert); +#endif /// Load command history if present. if (config().has("history_file")) history_file = config().getString("history_file"); @@ -450,10 +433,40 @@ private: Poco::File(history_file).createFile(); } +#if USE_READLINE + /// Install Ctrl+C signal handler that will be used in interactive mode. + + if (rl_initialize()) + throw Exception("Cannot initialize readline", ErrorCodes::CANNOT_READLINE); + + auto clear_prompt_or_exit = [](int) + { + /// This is signal safe. + ssize_t res = write(STDOUT_FILENO, "\n", 1); + + /// Allow to quit client while query is in progress by pressing Ctrl+C twice. + /// (First press to Ctrl+C will try to cancel query by InterruptListener). + if (res == 1 && rl_line_buffer[0] && !RL_ISSTATE(RL_STATE_DONE)) + { + rl_replace_line("", 0); + if (rl_forced_update_display()) + _exit(0); + } + else + { + /// A little dirty, but we struggle to find better way to correctly + /// force readline to exit after returning from the signal handler. + _exit(0); + } + }; + + if (signal(SIGINT, clear_prompt_or_exit) == SIG_ERR) + throwFromErrno("Cannot set signal handler.", ErrorCodes::CANNOT_SET_SIGNAL_HANDLER); +#endif + loop(); std::cout << (isNewYearMode() ? "Happy new year." : "Bye.") << std::endl; - return 0; } else @@ -461,8 +474,9 @@ private: query_id = config().getString("query_id", ""); nonInteractive(); + /// If exception code isn't zero, we should return non-zero return code anyway. if (last_exception) - return last_exception->code(); + return last_exception->code() != 0 ? last_exception->code() : -1; return 0; } @@ -471,6 +485,8 @@ private: void connect() { + connection_parameters = ConnectionParameters(config()); + if (is_interactive) std::cout << "Connecting to " << (!connection_parameters.default_database.empty() ? "database " + connection_parameters.default_database + " at " : "") @@ -492,17 +508,17 @@ private: String server_name; UInt64 server_version_major = 0; UInt64 server_version_minor = 0; - UInt64 server_revision = 0; + UInt64 server_version_patch = 0; if (max_client_network_bandwidth) { - ThrottlerPtr throttler = std::make_shared(max_client_network_bandwidth, 0, ""); + ThrottlerPtr throttler = std::make_shared(max_client_network_bandwidth, 0, ""); connection->setThrottler(throttler); } - connection->getServerVersion(server_name, server_version_major, server_version_minor, server_revision); + connection->getServerVersion(server_name, server_version_major, server_version_minor, server_version_patch, server_revision); - server_version = toString(server_version_major) + "." + toString(server_version_minor) + "." + toString(server_revision); + server_version = toString(server_version_major) + "." + toString(server_version_minor) + "." + toString(server_version_patch); if (server_display_name = connection->getServerDisplayName(); server_display_name.length() == 0) { @@ -513,6 +529,7 @@ private: { std::cout << "Connected to " << server_name << " server version " << server_version + << " revision " << server_revision << "." << std::endl << std::endl; } } @@ -526,7 +543,7 @@ private: fd_set fds; FD_ZERO(&fds); FD_SET(STDIN_FILENO, &fds); - return select(1, &fds, 0, 0, &timeout) == 1; + return select(1, &fds, nullptr, nullptr, &timeout) == 1; } inline const String prompt() const @@ -594,10 +611,14 @@ private: } catch (const Exception & e) { - std::cerr << std::endl - << "Exception on client:" << std::endl - << "Code: " << e.code() << ". " << e.displayText() << std::endl - << std::endl; + actual_client_error = e.code(); + if (!actual_client_error || actual_client_error != expected_client_error) + { + std::cerr << std::endl + << "Exception on client:" << std::endl + << "Code: " << e.code() << ". " << e.displayText() << std::endl + << std::endl; + } /// Client-side exception during query execution can result in the loss of /// sync in the connection protocol. @@ -635,7 +656,7 @@ private: bool process(const String & text) { - const auto ignore_error = config().getBool("ignore-error", false); + const bool test_mode = config().has("testmode"); if (config().has("multiquery")) { /// Several queries separated by ';'. @@ -650,6 +671,7 @@ private: { const char * pos = begin; ASTPtr ast = parseQuery(pos, end, true); + if (!ast) { if (ignore_error) @@ -665,7 +687,7 @@ private: return true; } - ASTInsertQuery * insert = typeid_cast(&*ast); + ASTInsertQuery * insert = typeid_cast(ast.get()); if (insert && insert->data) { @@ -679,6 +701,10 @@ private: while (isWhitespaceASCII(*begin) || *begin == ';') ++begin; + TestHint test_hint(test_mode, query); + expected_client_error = test_hint.clientError(); + expected_server_error = test_hint.serverError(); + try { if (!processSingleQuery(query, ast) && !ignore_error) @@ -686,10 +712,16 @@ private: } catch (...) { - std::cerr << "Error on processing query: " << query << std::endl << getCurrentExceptionMessage(true); + last_exception = std::make_unique(getCurrentExceptionMessage(true), getCurrentExceptionCode()); + actual_client_error = last_exception->code(); + if (!ignore_error && (!actual_client_error || actual_client_error != expected_client_error)) + std::cerr << "Error on processing query: " << query << std::endl << last_exception->message(); got_exception = true; } + if (!test_hint.checkActual(actual_server_error, actual_client_error, got_exception, last_exception)) + connection->forceConnected(); + if (got_exception && !ignore_error) { if (is_interactive) @@ -710,7 +742,7 @@ private: bool processSingleQuery(const String & line, ASTPtr parsed_query_ = nullptr) { - if (exit_strings.end() != exit_strings.find(line)) + if (exit_strings.end() != exit_strings.find(trim(line, [](char c){ return isWhitespaceASCII(c) || c == ';'; }))) return false; resetOutput(); @@ -848,14 +880,14 @@ private: /// If structure was received (thus, server has not thrown an exception), /// send our data with that structure. sendData(sample); - receivePacket(); + receiveEndOfQuery(); } } ASTPtr parseQuery(const char * & pos, const char * end, bool allow_multi_statements) { - ParserQuery parser(end); + ParserQuery parser(end, true); ASTPtr res; const auto ignore_error = config().getBool("ignore-error", false); @@ -863,7 +895,7 @@ private: if (is_interactive || ignore_error) { String message; - res = tryParseQuery(parser, pos, end, message, true, "", allow_multi_statements); + res = tryParseQuery(parser, pos, end, message, true, "", allow_multi_statements, 0); if (!res) { @@ -872,7 +904,7 @@ private: } } else - res = parseQueryAndMovePosition(parser, pos, end, "", allow_multi_statements); + res = parseQueryAndMovePosition(parser, pos, end, "", allow_multi_statements, 0); if (is_interactive) { @@ -930,6 +962,11 @@ private: connection->sendData(block); processed_rows += block.rows(); + /// Check if server send Log packet + auto packet_type = connection->checkPacket(); + if (packet_type && *packet_type == Protocol::Server::Log) + receiveAndProcessPacket(); + if (!block) break; } @@ -941,18 +978,28 @@ private: /// Flush all buffers. void resetOutput() { - block_out_stream = nullptr; + block_out_stream.reset(); + logs_out_stream.reset(); + if (pager_cmd) { pager_cmd->in.close(); pager_cmd->wait(); } pager_cmd = nullptr; + if (out_file_buf) { out_file_buf->next(); out_file_buf.reset(); } + + if (out_logs_buf) + { + out_logs_buf->next(); + out_logs_buf.reset(); + } + std_out.next(); } @@ -985,7 +1032,7 @@ private: continue; /// If there is no new data, continue checking whether the query was cancelled after a timeout. } - if (!receivePacket()) + if (!receiveAndProcessPacket()) break; } @@ -996,7 +1043,7 @@ private: /// Receive a part of the result, or progress info or an exception and process it. /// Returns true if one should continue receiving packets. - bool receivePacket() + bool receiveAndProcessPacket() { Connection::Packet packet = connection->receivePacket(); @@ -1027,6 +1074,10 @@ private: last_exception = std::move(packet.exception); return false; + case Protocol::Server::Log: + onLogData(packet.block); + return true; + case Protocol::Server::EndOfStream: onEndOfStream(); return false; @@ -1040,22 +1091,59 @@ private: /// Receive the block that serves as an example of the structure of table where data will be inserted. bool receiveSampleBlock(Block & out) { - Connection::Packet packet = connection->receivePacket(); - - switch (packet.type) + while (true) { - case Protocol::Server::Data: - out = packet.block; - return true; + Connection::Packet packet = connection->receivePacket(); - case Protocol::Server::Exception: - onException(*packet.exception); - last_exception = std::move(packet.exception); - return false; + switch (packet.type) + { + case Protocol::Server::Data: + out = packet.block; + return true; - default: - throw NetException("Unexpected packet from server (expected Data, got " - + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER); + case Protocol::Server::Exception: + onException(*packet.exception); + last_exception = std::move(packet.exception); + return false; + + case Protocol::Server::Log: + onLogData(packet.block); + break; + + default: + throw NetException("Unexpected packet from server (expected Data, Exception or Log, got " + + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER); + } + } + } + + + /// Process Log packets, exit when recieve Exception or EndOfStream + bool receiveEndOfQuery() + { + while (true) + { + Connection::Packet packet = connection->receivePacket(); + + switch (packet.type) + { + case Protocol::Server::EndOfStream: + onEndOfStream(); + return true; + + case Protocol::Server::Exception: + onException(*packet.exception); + last_exception = std::move(packet.exception); + return false; + + case Protocol::Server::Log: + onLogData(packet.block); + break; + + default: + throw NetException("Unexpected packet from server (expected Exception, EndOfStream or Log, got " + + String(Protocol::Server::toString(packet.type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER); + } } } @@ -1111,6 +1199,38 @@ private: } + void initLogsOutputStream() + { + if (!logs_out_stream) + { + WriteBuffer * wb = out_logs_buf.get(); + + if (!out_logs_buf) + { + if (server_logs_file.empty()) + { + /// Use stderr by default + out_logs_buf = std::make_unique(STDERR_FILENO); + wb = out_logs_buf.get(); + } + else if (server_logs_file == "-") + { + /// Use stdout if --server_logs_file=- specified + wb = &std_out; + } + else + { + out_logs_buf = std::make_unique(server_logs_file, DBMS_DEFAULT_BUFFER_SIZE, O_WRONLY | O_APPEND | O_CREAT); + wb = out_logs_buf.get(); + } + } + + logs_out_stream = std::make_shared(*wb); + logs_out_stream->writePrefix(); + } + } + + void onData(Block & block) { if (written_progress_chars) @@ -1134,6 +1254,14 @@ private: } + void onLogData(Block & block) + { + initLogsOutputStream(); + logs_out_stream->write(block); + logs_out_stream->flush(); + } + + void onTotals(Block & block) { initBlockOutputStream(block); @@ -1263,6 +1391,14 @@ private: resetOutput(); got_exception = true; + actual_server_error = e.code(); + if (expected_server_error) + { + if (actual_server_error == expected_server_error) + return; + std::cerr << "Expected error code: " << expected_server_error << " but got: " << actual_server_error << "." << std::endl; + } + std::string text = e.displayText(); auto embedded_stack_trace_pos = text.find("Stack trace"); @@ -1286,6 +1422,9 @@ private: if (block_out_stream) block_out_stream->writeSuffix(); + if (logs_out_stream) + logs_out_stream->writeSuffix(); + resetOutput(); if (is_interactive && !written_first_block) @@ -1294,10 +1433,7 @@ private: void showClientVersion() { - std::cout << "ClickHouse client version " << DBMS_VERSION_MAJOR - << "." << DBMS_VERSION_MINOR - << "." << ClickHouseRevision::get() - << "." << std::endl; + std::cout << DBMS_NAME << " client version " << VERSION_STRING << "." << std::endl; } public: @@ -1362,53 +1498,73 @@ public: } } -#define DECLARE_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) (#NAME, boost::program_options::value (), DESCRIPTION) + ioctl(0, TIOCGWINSZ, &terminal_size); + + namespace po = boost::program_options; + + unsigned line_length = po::options_description::m_default_line_length; + unsigned min_description_length = line_length / 2; + if (!stdin_is_not_tty) + { + line_length = std::max(3U, static_cast(terminal_size.ws_col)); + min_description_length = std::min(min_description_length, line_length - 2); + } + +#define DECLARE_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) (#NAME, po::value(), DESCRIPTION) /// Main commandline options related to client functionality and all parameters from Settings. - boost::program_options::options_description main_description("Main options"); + po::options_description main_description("Main options", line_length, min_description_length); main_description.add_options() ("help", "produce help message") - ("config-file,c", boost::program_options::value(), "config-file path") - ("host,h", boost::program_options::value()->default_value("localhost"), "server host") - ("port", boost::program_options::value()->default_value(9000), "server port") - ("ssl,s", "ssl") - ("user,u", boost::program_options::value(), "user") - ("password", boost::program_options::value(), "password") - ("query_id", boost::program_options::value(), "query_id") - ("query,q", boost::program_options::value(), "query") - ("database,d", boost::program_options::value(), "database") - ("pager", boost::program_options::value(), "pager") + ("config-file,c", po::value(), "config-file path") + ("host,h", po::value()->default_value("localhost"), "server host") + ("port", po::value()->default_value(9000), "server port") + ("secure,s", "Use TLS connection") + ("user,u", po::value()->default_value("default"), "user") + ("password", po::value(), "password") + ("ask-password", "ask-password") + ("query_id", po::value(), "query_id") + ("query,q", po::value(), "query") + ("database,d", po::value(), "database") + ("pager", po::value(), "pager") + ("disable_suggestion,A", "Disable loading suggestion data. Note that suggestion data is loaded asynchronously through a second connection to ClickHouse server. Also it is reasonable to disable suggestion if you want to paste a query with TAB characters. Shorthand option -A is for those who get used to mysql client.") + ("suggestion_limit", po::value()->default_value(10000), + "Suggestion limit for how many databases, tables and columns to fetch.") ("multiline,m", "multiline") ("multiquery,n", "multiquery") - ("ignore-error", "Do not stop processing in multiquery mode") - ("format,f", boost::program_options::value(), "default output format") + ("format,f", po::value(), "default output format") + ("testmode,T", "enable test hints in comments") + ("ignore-error", "do not stop processing in multiquery mode") ("vertical,E", "vertical output format, same as --format=Vertical or FORMAT Vertical or \\G at end of command") ("time,t", "print query execution time to stderr in non-interactive mode (for benchmarks)") ("stacktrace", "print stack traces of exceptions") ("progress", "print progress even in non-interactive mode") ("version,V", "print version information and exit") + ("version-clean", "print version in machine-readable format and exit") ("echo", "in batch mode, print query before execution") - ("max_client_network_bandwidth", boost::program_options::value(), "the maximum speed of data exchange over the network for the client in bytes per second.") - ("compression", boost::program_options::value(), "enable or disable compression") + ("max_client_network_bandwidth", po::value(), "the maximum speed of data exchange over the network for the client in bytes per second.") + ("compression", po::value(), "enable or disable compression") + ("log-level", po::value(), "client log level") + ("server_logs_file", po::value(), "put server logs into specified file") APPLY_FOR_SETTINGS(DECLARE_SETTING) ; #undef DECLARE_SETTING /// Commandline options related to external tables. - boost::program_options::options_description external_description("External tables options"); + po::options_description external_description("External tables options"); external_description.add_options() - ("file", boost::program_options::value(), "data file or - for stdin") - ("name", boost::program_options::value()->default_value("_data"), "name of the table") - ("format", boost::program_options::value()->default_value("TabSeparated"), "data format") - ("structure", boost::program_options::value(), "structure") - ("types", boost::program_options::value(), "types") + ("file", po::value(), "data file or - for stdin") + ("name", po::value()->default_value("_data"), "name of the table") + ("format", po::value()->default_value("TabSeparated"), "data format") + ("structure", po::value(), "structure") + ("types", po::value(), "types") ; /// Parse main commandline options. - boost::program_options::parsed_options parsed = boost::program_options::command_line_parser( + po::parsed_options parsed = po::command_line_parser( common_arguments.size(), common_arguments.data()).options(main_description).run(); - boost::program_options::variables_map options; - boost::program_options::store(parsed, options); + po::variables_map options; + po::store(parsed, options); if (options.count("version") || options.count("V")) { @@ -1416,6 +1572,12 @@ public: exit(0); } + if (options.count("version-clean")) + { + std::cout << VERSION_STRING; + exit(0); + } + /// Output of help message. if (options.count("help") || (options.count("host") && options["host"].as() == "elp")) /// If user writes -help instead of --help. @@ -1425,14 +1587,17 @@ public: exit(0); } + if (options.count("log-level")) + Poco::Logger::root().setLevel(options["log-level"].as()); + size_t number_of_external_tables_with_stdin_source = 0; for (size_t i = 0; i < external_tables_arguments.size(); ++i) { /// Parse commandline options related to external tables. - boost::program_options::parsed_options parsed = boost::program_options::command_line_parser( + po::parsed_options parsed = po::command_line_parser( external_tables_arguments[i].size(), external_tables_arguments[i].data()).options(external_description).run(); - boost::program_options::variables_map external_options; - boost::program_options::store(parsed, external_options); + po::variables_map external_options; + po::store(parsed, external_options); try { @@ -1451,7 +1616,7 @@ public: } } - /// Extract settings and limits from the options. + /// Extract settings from the options. #define EXTRACT_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) \ if (options.count(#NAME)) \ context.setSetting(#NAME, options[#NAME].as()); @@ -1474,17 +1639,20 @@ public: if (options.count("port") && !options["port"].defaulted()) config().setInt("port", options["port"].as()); - if (options.count("ssl")) - config().setBool("ssl", true); - if (options.count("user")) + if (options.count("secure")) + config().setBool("secure", true); + if (options.count("user") && !options["user"].defaulted()) config().setString("user", options["user"].as()); if (options.count("password")) config().setString("password", options["password"].as()); - + if (options.count("ask-password")) + config().setBool("ask-password", true); if (options.count("multiline")) config().setBool("multiline", true); if (options.count("multiquery")) config().setBool("multiquery", true); + if (options.count("testmode")) + config().setBool("testmode", true); if (options.count("ignore-error")) config().setBool("ignore-error", true); if (options.count("format")) @@ -1503,12 +1671,17 @@ public: max_client_network_bandwidth = options["max_client_network_bandwidth"].as(); if (options.count("compression")) config().setBool("compression", options["compression"].as()); + if (options.count("server_logs_file")) + server_logs_file = options["server_logs_file"].as(); + if (options.count("disable_suggestion")) + config().setBool("disable_suggestion", true); + if (options.count("suggestion_limit")) + config().setInt("suggestion_limit", options["suggestion_limit"].as()); } }; } - int mainEntryClickHouseClient(int argc, char ** argv) { DB::Client client; @@ -1522,6 +1695,11 @@ int mainEntryClickHouseClient(int argc, char ** argv) std::cerr << "Bad arguments: " << e.what() << std::endl; return 1; } + catch (...) + { + std::cerr << DB::getCurrentExceptionMessage(true) << std::endl; + return 1; + } return client.run(); } diff --git a/dbms/programs/client/ConnectionParameters.h b/dbms/programs/client/ConnectionParameters.h new file mode 100644 index 00000000000..68bc3728349 --- /dev/null +++ b/dbms/programs/client/ConnectionParameters.h @@ -0,0 +1,84 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + +struct ConnectionParameters +{ + String host; + UInt16 port; + String default_database; + String user; + String password; + Protocol::Secure security; + Protocol::Compression compression; + ConnectionTimeouts timeouts; + + ConnectionParameters() {} + + ConnectionParameters(const Poco::Util::AbstractConfiguration & config) + { + bool is_secure = config.getBool("secure", false); + security = is_secure + ? Protocol::Secure::Enable + : Protocol::Secure::Disable; + + host = config.getString("host", "localhost"); + port = config.getInt("port", + config.getInt(is_secure ? "tcp_port_secure" : "tcp_port", + is_secure ? DBMS_DEFAULT_SECURE_PORT : DBMS_DEFAULT_PORT)); + + default_database = config.getString("database", ""); + user = config.getString("user", ""); + + if (config.getBool("ask-password", false)) + { + if (config.has("password")) + throw Exception("Specified both --password and --ask-password. Remove one of them", ErrorCodes::BAD_ARGUMENTS); + + std::cout << "Password for user " << user << ": "; + SetTerminalEcho(false); + + SCOPE_EXIT({ + SetTerminalEcho(true); + }); + std::getline(std::cin, password); + std::cout << std::endl; + } + else + { + password = config.getString("password", ""); + } + + compression = config.getBool("compression", true) + ? Protocol::Compression::Enable + : Protocol::Compression::Disable; + + timeouts = ConnectionTimeouts( + Poco::Timespan(config.getInt("connect_timeout", DBMS_DEFAULT_CONNECT_TIMEOUT_SEC), 0), + Poco::Timespan(config.getInt("receive_timeout", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC), 0), + Poco::Timespan(config.getInt("send_timeout", DBMS_DEFAULT_SEND_TIMEOUT_SEC), 0), + Poco::Timespan(config.getInt("tcp_keep_alive_timeout", 0), 0)); + } +}; + +} diff --git a/dbms/programs/client/Suggest.h b/dbms/programs/client/Suggest.h new file mode 100644 index 00000000000..617e2bb520e --- /dev/null +++ b/dbms/programs/client/Suggest.h @@ -0,0 +1,219 @@ +#pragma once + +#include "ConnectionParameters.h" + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int UNKNOWN_PACKET_FROM_SERVER; +} + +class Suggest : public ext::singleton +{ +private: + /// The vector will be filled with completion words from the server and sorted. + using Words = std::vector; + + /// Keywords may be not up to date with ClickHouse parser. + Words words + { + "CREATE", "DATABASE", "IF", "NOT", "EXISTS", "TEMPORARY", "TABLE", "ON", "CLUSTER", "DEFAULT", "MATERIALIZED", "ALIAS", "ENGINE", + "AS", "VIEW", "POPULATE", "SETTINGS", "ATTACH", "DETACH", "DROP", "RENAME", "TO", "ALTER", "ADD", "MODIFY", "CLEAR", "COLUMN", "AFTER", + "COPY", "PROJECT", "PRIMARY", "KEY", "CHECK", "PARTITION", "PART", "FREEZE", "FETCH", "FROM", "SHOW", "INTO", "OUTFILE", "FORMAT", "TABLES", + "DATABASES", "LIKE", "PROCESSLIST", "CASE", "WHEN", "THEN", "ELSE", "END", "DESCRIBE", "DESC", "USE", "SET", "OPTIMIZE", "FINAL", "DEDUPLICATE", + "INSERT", "VALUES", "SELECT", "DISTINCT", "SAMPLE", "ARRAY", "JOIN", "GLOBAL", "LOCAL", "ANY", "ALL", "INNER", "LEFT", "RIGHT", "FULL", "OUTER", + "CROSS", "USING", "PREWHERE", "WHERE", "GROUP", "BY", "WITH", "TOTALS", "HAVING", "ORDER", "COLLATE", "LIMIT", "UNION", "AND", "OR", "ASC", "IN", + "KILL", "QUERY", "SYNC", "ASYNC", "TEST" + }; + + /// Words are fetched asynchonously. + std::thread loading_thread; + std::atomic ready{false}; + + /// Points to current word to suggest. + Words::const_iterator pos; + /// Points after the last possible match. + Words::const_iterator end; + + /// Set iterators to the matched range of words if any. + void findRange(const char * prefix, size_t prefix_length) + { + std::string prefix_str(prefix); + std::tie(pos, end) = std::equal_range(words.begin(), words.end(), prefix_str, + [prefix_length](const std::string & s, const std::string & prefix) { return strncmp(s.c_str(), prefix.c_str(), prefix_length) < 0; }); + } + + /// Iterates through matched range. + char * nextMatch() + { + if (pos >= end) + return nullptr; + + /// readline will free memory by itself. + char * word = strdup(pos->c_str()); + ++pos; + return word; + } + + void loadImpl(Connection & connection, size_t suggestion_limit) + { + std::stringstream query; + query << "SELECT DISTINCT arrayJoin(extractAll(name, '[\\\\w_]{2,}')) AS res FROM (" + "SELECT name FROM system.functions" + " UNION ALL " + "SELECT name FROM system.table_engines" + " UNION ALL " + "SELECT name FROM system.formats" + " UNION ALL " + "SELECT name FROM system.table_functions" + " UNION ALL " + "SELECT name FROM system.data_type_families" + " UNION ALL " + "SELECT name FROM system.settings" + " UNION ALL " + "SELECT concat(func.name, comb.name) FROM system.functions AS func CROSS JOIN system.aggregate_function_combinators AS comb WHERE is_aggregate"; + + /// The user may disable loading of databases, tables, columns by setting suggestion_limit to zero. + if (suggestion_limit > 0) + { + String limit_str = toString(suggestion_limit); + query << + " UNION ALL " + "SELECT name FROM system.databases LIMIT " << limit_str + << " UNION ALL " + "SELECT DISTINCT name FROM system.tables LIMIT " << limit_str + << " UNION ALL " + "SELECT DISTINCT name FROM system.columns LIMIT " << limit_str; + } + + query << ") WHERE notEmpty(res)"; + + fetch(connection, query.str()); + } + + void fetch(Connection & connection, const std::string & query) + { + connection.sendQuery(query); + + while (true) + { + Connection::Packet packet = connection.receivePacket(); + switch (packet.type) + { + case Protocol::Server::Data: + fillWordsFromBlock(packet.block); + continue; + + case Protocol::Server::Progress: + continue; + case Protocol::Server::ProfileInfo: + continue; + case Protocol::Server::Totals: + continue; + case Protocol::Server::Extremes: + continue; + case Protocol::Server::Log: + continue; + + case Protocol::Server::Exception: + packet.exception->rethrow(); + return; + + case Protocol::Server::EndOfStream: + return; + + default: + throw Exception("Unknown packet from server", ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); + } + } + } + + void fillWordsFromBlock(const Block & block) + { + if (!block) + return; + + if (block.columns() != 1) + throw Exception("Wrong number of columns received for query to read words for suggestion", ErrorCodes::LOGICAL_ERROR); + + const ColumnString & column = typeid_cast(*block.getByPosition(0).column); + + size_t rows = block.rows(); + for (size_t i = 0; i < rows; ++i) + words.emplace_back(column.getDataAt(i).toString()); + } + +public: + /// More old server versions cannot execute the query above. + static constexpr int MIN_SERVER_REVISION = 54406; + + void load(const ConnectionParameters & connection_parameters, size_t suggestion_limit) + { + loading_thread = std::thread([connection_parameters, suggestion_limit, this] + { + try + { + Connection connection( + connection_parameters.host, + connection_parameters.port, + connection_parameters.default_database, + connection_parameters.user, + connection_parameters.password, + connection_parameters.timeouts, + "client", + connection_parameters.compression, + connection_parameters.security); + + loadImpl(connection, suggestion_limit); + } + catch (...) + { + std::cerr << "Cannot load data for command line suggestions: " << getCurrentExceptionMessage(false, true) << "\n"; + } + + /// Note that keyword suggestions are available even if we cannot load data from server. + + std::sort(words.begin(), words.end()); + ready = true; + }); + } + + /// A function for readline. + static char * generator(const char * text, int state) + { + Suggest & suggest = Suggest::instance(); + if (!suggest.ready) + return nullptr; + if (state == 0) + suggest.findRange(text, strlen(text)); + + /// Do not append whitespace after word. For unknown reason, rl_completion_append_character = '\0' does not work. + rl_completion_suppress_append = 1; + + return suggest.nextMatch(); + } + + ~Suggest() + { + if (loading_thread.joinable()) + loading_thread.join(); + } +}; + +} diff --git a/dbms/programs/client/TestHint.h b/dbms/programs/client/TestHint.h new file mode 100644 index 00000000000..790e58ee7fe --- /dev/null +++ b/dbms/programs/client/TestHint.h @@ -0,0 +1,118 @@ +#pragma once + +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + + +/// Checks expected server and client error codes in testmode. +/// To enable it add special comment after the query: "-- { serverError 60 }" or "-- { clientError 20 }". +class TestHint +{ +public: + TestHint(bool enabled_, const String & query) + : enabled(enabled_) + { + if (!enabled_) + return; + + /// TODO: This is absolutely wrong. Fragment may be contained inside string literal. + size_t pos = query.find("--"); + + if (pos != String::npos && query.find("--", pos + 2) != String::npos) + return; /// It's not last comment. Hint belongs to commented query. /// TODO Absolutely wrong: there maybe the following comment for the next query. + + if (pos != String::npos) + { + /// TODO: This is also wrong. Comment may already have ended by line break. + pos = query.find('{', pos + 2); + + if (pos != String::npos) + { + String hint = query.substr(pos + 1); + + /// TODO: And this is wrong for the same reason. + pos = hint.find('}'); + hint.resize(pos); + parse(hint); + } + } + } + + /// @returns true if it's possible to continue without reconnect + bool checkActual(int & actual_server_error, int & actual_client_error, + bool & got_exception, std::unique_ptr & last_exception) const + { + if (!enabled) + return true; + + if (allErrorsExpected(actual_server_error, actual_client_error)) + { + got_exception = false; + last_exception.reset(); + actual_server_error = 0; + actual_client_error = 0; + return false; + } + + if (lostExpectedError(actual_server_error, actual_client_error)) + { + std::cerr << "Success when error expected. It expects server error " + << server_error << ", client error " << client_error << "." << std::endl; + got_exception = true; + last_exception = std::make_unique("Success when error expected", ErrorCodes::LOGICAL_ERROR); /// return error to OS + return false; + } + + return true; + } + + int serverError() const { return server_error; } + int clientError() const { return client_error; } + +private: + bool enabled = false; + int server_error = 0; + int client_error = 0; + + void parse(const String & hint) + { + std::stringstream ss; + ss << hint; + while (!ss.eof()) + { + String item; + ss >> item; + if (item.empty()) + break; + + if (item == "serverError") + ss >> server_error; + else if (item == "clientError") + ss >> client_error; + } + } + + bool allErrorsExpected(int actual_server_error, int actual_client_error) const + { + return (server_error || client_error) && (server_error == actual_server_error) && (client_error == actual_client_error); + } + + bool lostExpectedError(int actual_server_error, int actual_client_error) const + { + return (server_error && !actual_server_error) || (client_error && !actual_client_error); + } +}; + +} diff --git a/dbms/src/Server/clickhouse-client.cpp b/dbms/programs/client/clickhouse-client.cpp similarity index 100% rename from dbms/src/Server/clickhouse-client.cpp rename to dbms/programs/client/clickhouse-client.cpp diff --git a/dbms/src/Server/clickhouse-client.xml b/dbms/programs/client/clickhouse-client.xml similarity index 71% rename from dbms/src/Server/clickhouse-client.xml rename to dbms/programs/client/clickhouse-client.xml index 3cd05d7fec9..c073ab38aea 100644 --- a/dbms/src/Server/clickhouse-client.xml +++ b/dbms/programs/client/clickhouse-client.xml @@ -15,17 +15,18 @@ {display_name} :) - {display_name} \e[1;32m:)\e[0m - {display_name} \e[1;31m:)\e[0m + {display_name} \x01\e[1;32m\x02:)\x01\e[0m\x02 + {display_name} \x01\e[1;31m\x02:)\x01\e[0m\x02 diff --git a/dbms/programs/compressor/CMakeLists.txt b/dbms/programs/compressor/CMakeLists.txt new file mode 100644 index 00000000000..7aa2cad5708 --- /dev/null +++ b/dbms/programs/compressor/CMakeLists.txt @@ -0,0 +1,8 @@ +add_library (clickhouse-compressor-lib ${LINK_MODE} Compressor.cpp) +target_link_libraries (clickhouse-compressor-lib PRIVATE clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) + +if (CLICKHOUSE_SPLIT_BINARY) + # Also in utils + add_executable (clickhouse-compressor clickhouse-compressor.cpp) + target_link_libraries (clickhouse-compressor PRIVATE clickhouse-compressor-lib) +endif () diff --git a/dbms/src/Server/Compressor.cpp b/dbms/programs/compressor/Compressor.cpp similarity index 100% rename from dbms/src/Server/Compressor.cpp rename to dbms/programs/compressor/Compressor.cpp diff --git a/dbms/src/Server/clickhouse-compressor.cpp b/dbms/programs/compressor/clickhouse-compressor.cpp similarity index 100% rename from dbms/src/Server/clickhouse-compressor.cpp rename to dbms/programs/compressor/clickhouse-compressor.cpp diff --git a/dbms/src/Server/config_tools.h.in b/dbms/programs/config_tools.h.in similarity index 61% rename from dbms/src/Server/config_tools.h.in rename to dbms/programs/config_tools.h.in index 2e8337c74b9..a7a538d9f1d 100644 --- a/dbms/src/Server/config_tools.h.in +++ b/dbms/programs/config_tools.h.in @@ -7,5 +7,8 @@ #cmakedefine01 ENABLE_CLICKHOUSE_LOCAL #cmakedefine01 ENABLE_CLICKHOUSE_BENCHMARK #cmakedefine01 ENABLE_CLICKHOUSE_PERFORMANCE -#cmakedefine01 ENABLE_CLICKHOUSE_TOOLS #cmakedefine01 ENABLE_CLICKHOUSE_COPIER +#cmakedefine01 ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG +#cmakedefine01 ENABLE_CLICKHOUSE_COMPRESSOR +#cmakedefine01 ENABLE_CLICKHOUSE_FORMAT +#cmakedefine01 ENABLE_CLICKHOUSE_OBFUSCATOR diff --git a/dbms/programs/copier/CMakeLists.txt b/dbms/programs/copier/CMakeLists.txt new file mode 100644 index 00000000000..ed3e55208aa --- /dev/null +++ b/dbms/programs/copier/CMakeLists.txt @@ -0,0 +1,7 @@ +add_library (clickhouse-copier-lib ${LINK_MODE} ClusterCopier.cpp) +target_link_libraries (clickhouse-copier-lib PRIVATE clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions daemon) + +if (CLICKHOUSE_SPLIT_BINARY) + add_executable (clickhouse-copier clickhouse-copier.cpp) + target_link_libraries (clickhouse-copier clickhouse-copier-lib) +endif () diff --git a/dbms/src/Server/ClusterCopier.cpp b/dbms/programs/copier/ClusterCopier.cpp similarity index 89% rename from dbms/src/Server/ClusterCopier.cpp rename to dbms/programs/copier/ClusterCopier.cpp index 4233ce70e3f..02d371957ef 100644 --- a/dbms/src/Server/ClusterCopier.cpp +++ b/dbms/programs/copier/ClusterCopier.cpp @@ -17,25 +17,30 @@ #include #include +#include +#include +#include + #include #include +#include #include #include +#include +#include +#include +#include +#include +#include +#include #include #include #include #include -#include #include #include #include #include - -#include -#include -#include -#include -#include #include #include #include @@ -47,6 +52,7 @@ #include #include #include +#include #include #include #include @@ -60,8 +66,7 @@ #include #include #include -#include -#include +#include #include @@ -318,7 +323,7 @@ struct TaskTable struct TaskCluster { TaskCluster(const String & task_zookeeper_path_, const String & default_local_database_) - : task_zookeeper_path(task_zookeeper_path_), default_local_database(default_local_database_) {} + : task_zookeeper_path(task_zookeeper_path_), default_local_database(default_local_database_) {} void loadTasks(const Poco::Util::AbstractConfiguration & config, const String & base_key = ""); @@ -354,24 +359,26 @@ struct TaskCluster }; +struct MultiTransactionInfo +{ + int32_t code; + Coordination::Requests requests; + Coordination::Responses responses; +}; + + /// Atomically checks that is_dirty node is not exists, and made the remaining op /// Returns relative number of failed operation in the second field (the passed op has 0 index) -static zkutil::MultiTransactionInfo checkNoNodeAndCommit( +static MultiTransactionInfo checkNoNodeAndCommit( const zkutil::ZooKeeperPtr & zookeeper, const String & checking_node_path, - zkutil::OpPtr && op) + Coordination::RequestPtr && op) { - zkutil::Ops ops; - ops.emplace_back(std::make_shared(checking_node_path, "", zookeeper->getDefaultACL(), zkutil::CreateMode::Persistent)); - ops.emplace_back(std::make_shared(checking_node_path, -1)); - ops.emplace_back(std::move(op)); - - zkutil::MultiTransactionInfo info; - zookeeper->tryMultiNoThrow(ops, nullptr, &info); - - if (info.code != ZOK && !zkutil::isUserError(info.code)) - throw zkutil::KeeperException(info.code); - + MultiTransactionInfo info; + info.requests.emplace_back(zkutil::makeCreateRequest(checking_node_path, "", zkutil::CreateMode::Persistent)); + info.requests.emplace_back(zkutil::makeRemoveRequest(checking_node_path, -1)); + info.requests.emplace_back(std::move(op)); + info.code = zookeeper->tryMulti(info.requests, info.responses); return info; } @@ -403,8 +410,7 @@ BlockInputStreamPtr squashStreamIntoOneBlock(const BlockInputStreamPtr & stream) return std::make_shared( stream, std::numeric_limits::max(), - std::numeric_limits::max() - ); + std::numeric_limits::max()); } Block getBlockWithAllStreamData(const BlockInputStreamPtr & stream) @@ -544,14 +550,14 @@ TaskTable::TaskTable(TaskCluster & parent, const Poco::Util::AbstractConfigurati engine_push_str = config.getString(table_prefix + "engine"); { ParserStorage parser_storage; - engine_push_ast = parseQuery(parser_storage, engine_push_str); + engine_push_ast = parseQuery(parser_storage, engine_push_str, 0); engine_push_partition_key_ast = extractPartitionKey(engine_push_ast); } sharding_key_str = config.getString(table_prefix + "sharding_key"); { ParserExpressionWithOptionalAlias parser_expression(false); - sharding_key_ast = parseQuery(parser_expression, sharding_key_str); + sharding_key_ast = parseQuery(parser_expression, sharding_key_str, 0); engine_split_ast = createASTStorageDistributed(cluster_push_name, table_push.first, table_push.second, sharding_key_ast); } @@ -559,7 +565,7 @@ TaskTable::TaskTable(TaskCluster & parent, const Poco::Util::AbstractConfigurati if (!where_condition_str.empty()) { ParserExpressionWithOptionalAlias parser_expression(false); - where_condition_ast = parseQuery(parser_expression, where_condition_str); + where_condition_ast = parseQuery(parser_expression, where_condition_str, 0); // Will use canonical expression form where_condition_str = queryToString(where_condition_ast); @@ -607,7 +613,7 @@ static ShardPriority getReplicasPriority(const Cluster::Addresses & replicas, co res.is_remote = 1; for (auto & replica : replicas) { - if (isLocalAddress(replica.resolved_address)) + if (isLocalAddress(DNSResolver::instance().resolveHost(replica.host_name))) { res.is_remote = 0; break; @@ -694,15 +700,19 @@ void DB::TaskCluster::reloadSettings(const Poco::Util::AbstractConfiguration & c if (config.has(prefix + "settings_push")) settings_push.loadSettingsFromConfig(prefix + "settings_push", config); - /// Override important settings - settings_pull.load_balancing = LoadBalancing::NEAREST_HOSTNAME; - settings_pull.readonly = 1; - settings_pull.max_threads = 1; - settings_pull.max_block_size = settings_pull.max_block_size.changed ? settings_pull.max_block_size.value : 8192UL; - settings_pull.preferred_block_size_bytes = 0; + auto set_default_value = [] (auto && setting, auto && default_value) + { + setting = setting.changed ? setting.value : default_value; + }; - settings_push.insert_distributed_timeout = 0; + /// Override important settings + settings_pull.readonly = 1; settings_push.insert_distributed_sync = 1; + set_default_value(settings_pull.load_balancing, LoadBalancing::NEAREST_HOSTNAME); + set_default_value(settings_pull.max_threads, 1); + set_default_value(settings_pull.max_block_size, 8192UL); + set_default_value(settings_pull.preferred_block_size_bytes, 0); + set_default_value(settings_push.insert_distributed_timeout, 0); } @@ -713,13 +723,11 @@ class ClusterCopier { public: - ClusterCopier(const ConfigurationPtr & zookeeper_config_, - const String & task_path_, + ClusterCopier(const String & task_path_, const String & host_id_, const String & proxy_database_name_, Context & context_) : - zookeeper_config(zookeeper_config_), task_zookeeper_path(task_path_), host_id(host_id_), working_database_name(proxy_database_name_), @@ -730,9 +738,9 @@ public: void init() { - auto zookeeper = getZooKeeper(); + auto zookeeper = context.getZooKeeper(); - task_description_watch_callback = [this] (zkutil::ZooKeeper &, int, int, const char *) + task_description_watch_callback = [this] (const Coordination::WatchResponse &) { UInt64 version = ++task_descprtion_version; LOG_DEBUG(log, "Task description should be updated, local version " << version); @@ -760,8 +768,8 @@ public: /// Do not initialize tables, will make deferred initialization in process() - getZooKeeper()->createAncestors(getWorkersPathVersion() + "/"); - getZooKeeper()->createAncestors(getWorkersPath() + "/"); + zookeeper->createAncestors(getWorkersPathVersion() + "/"); + zookeeper->createAncestors(getWorkersPath() + "/"); } template @@ -787,7 +795,7 @@ public: } std::rethrow_exception(exception); - }; + } void discoverShardPartitions(const TaskShardPtr & task_shard) @@ -809,7 +817,7 @@ public: try { - type->deserializeTextQuoted(*column_dummy, rb); + type->deserializeTextQuoted(*column_dummy, rb, FormatSettings()); } catch (Exception & e) { @@ -888,15 +896,15 @@ public: void reloadTaskDescription() { - auto zookeeper = getZooKeeper(); + auto zookeeper = context.getZooKeeper(); task_description_watch_zookeeper = zookeeper; String task_config_str; - zkutil::Stat stat; + Coordination::Stat stat; int code; zookeeper->tryGetWatch(task_description_path, task_config_str, &stat, task_description_watch_callback, &code); - if (code != ZOK) + if (code) throw Exception("Can't get description node " + task_description_path, ErrorCodes::BAD_ARGUMENTS); LOG_DEBUG(log, "Loading description, zxid=" << task_descprtion_current_stat.czxid); @@ -1036,9 +1044,13 @@ protected: String workers_path = getWorkersPath(); String current_worker_path = getCurrentWorkerNodePath(); + size_t num_bad_version_errors = 0; + while (true) { - zkutil::Stat stat; + updateConfigIfNeeded(); + + Coordination::Stat stat; zookeeper->get(workers_version_path, &stat); auto version = stat.version; zookeeper->get(workers_path, &stat); @@ -1047,31 +1059,40 @@ protected: { LOG_DEBUG(log, "Too many workers (" << stat.numChildren << ", maximum " << task_cluster->max_workers << ")" << ". Postpone processing " << description); + + if (unprioritized) + current_sleep_time = std::min(max_sleep_time, current_sleep_time + default_sleep_time); + + std::this_thread::sleep_for(current_sleep_time); + num_bad_version_errors = 0; } else { - zkutil::Ops ops; - ops.emplace_back(new zkutil::Op::SetData(workers_version_path, description, version)); - ops.emplace_back(new zkutil::Op::Create(current_worker_path, description, zookeeper->getDefaultACL(), zkutil::CreateMode::Ephemeral)); - auto code = zookeeper->tryMulti(ops); + Coordination::Requests ops; + ops.emplace_back(zkutil::makeSetRequest(workers_version_path, description, version)); + ops.emplace_back(zkutil::makeCreateRequest(current_worker_path, description, zkutil::CreateMode::Ephemeral)); + Coordination::Responses responses; + auto code = zookeeper->tryMulti(ops, responses); - if (code == ZOK || code == ZNODEEXISTS) + if (code == Coordination::ZOK || code == Coordination::ZNODEEXISTS) return std::make_shared(current_worker_path, *zookeeper, false, false, description); - if (code == ZBADVERSION) + if (code == Coordination::ZBADVERSION) { - LOG_DEBUG(log, "A concurrent worker has just been added, will check free worker slots again"); + ++num_bad_version_errors; + + /// Try to make fast retries + if (num_bad_version_errors > 3) + { + LOG_DEBUG(log, "A concurrent worker has just been added, will check free worker slots again"); + std::chrono::milliseconds random_sleep_time(std::uniform_int_distribution(1, 1000)(task_cluster->random_engine)); + std::this_thread::sleep_for(random_sleep_time); + num_bad_version_errors = 0; + } } else - throw zkutil::KeeperException(code); + throw Coordination::Exception(code); } - - if (unprioritized) - current_sleep_time = std::min(max_sleep_time, current_sleep_time + default_sleep_time); - - std::this_thread::sleep_for(current_sleep_time); - - updateConfigIfNeeded(); } } @@ -1084,7 +1105,7 @@ protected: { LOG_DEBUG(log, "Check that all shards processed partition " << partition_name << " successfully"); - auto zookeeper = getZooKeeper(); + auto zookeeper = context.getZooKeeper(); Strings status_paths; for (auto & shard : shards_with_partition) @@ -1093,22 +1114,27 @@ protected: status_paths.emplace_back(task_shard_partition.getShardStatusPath()); } - zkutil::Stat stat; std::vector zxid1, zxid2; try { - // Check that state is Finished and remember zxid + std::vector get_futures; for (const String & path : status_paths) + get_futures.emplace_back(zookeeper->asyncGet(path)); + + // Check that state is Finished and remember zxid + for (auto & future : get_futures) { - TaskStateWithOwner status = TaskStateWithOwner::fromString(zookeeper->get(path, &stat)); + auto res = future.get(); + + TaskStateWithOwner status = TaskStateWithOwner::fromString(res.data); if (status.state != TaskState::Finished) { - LOG_INFO(log, "The task " << path << " is being rewritten by " << status.owner - << ". Partition will be rechecked"); + LOG_INFO(log, "The task " << res.data << " is being rewritten by " << status.owner << ". Partition will be rechecked"); return false; } - zxid1.push_back(stat.pzxid); + + zxid1.push_back(res.stat.pzxid); } // Check that partition is not dirty @@ -1118,17 +1144,21 @@ protected: return false; } + get_futures.clear(); + for (const String & path : status_paths) + get_futures.emplace_back(zookeeper->asyncGet(path)); + // Remember zxid of states again - for (const auto & path : status_paths) + for (auto & future : get_futures) { - zookeeper->exists(path, &stat); - zxid2.push_back(stat.pzxid); + auto res = future.get(); + zxid2.push_back(res.stat.pzxid); } } - catch (const zkutil::KeeperException & e) + catch (const Coordination::Exception & e) { LOG_INFO(log, "A ZooKeeper error occurred while checking partition " << partition_name - << ". Will recheck the partition. Error: " << e.what()); + << ". Will recheck the partition. Error: " << e.displayText()); return false; } @@ -1210,9 +1240,9 @@ protected: { cleaner_holder = zkutil::EphemeralNodeHolder::create(dirt_cleaner_path, *zookeeper, host_id); } - catch (zkutil::KeeperException & e) + catch (const Coordination::Exception & e) { - if (e.code == ZNODEEXISTS) + if (e.code == Coordination::ZNODEEXISTS) { LOG_DEBUG(log, "Partition " << task_partition.name << " is cleaning now by somebody, sleep"); std::this_thread::sleep_for(default_sleep_time); @@ -1222,7 +1252,7 @@ protected: throw; } - zkutil::Stat stat; + Coordination::Stat stat; if (zookeeper->exists(current_partition_active_workers_dir, &stat)) { if (stat.numChildren != 0) @@ -1259,8 +1289,10 @@ protected: } /// Remove the locking node - cleaner_holder.reset(); - zookeeper->remove(is_dirty_flag_path); + Coordination::Requests requests; + requests.emplace_back(zkutil::makeRemoveRequest(dirt_cleaner_path, -1)); + requests.emplace_back(zkutil::makeRemoveRequest(is_dirty_flag_path, -1)); + zookeeper->multi(requests); LOG_INFO(log, "Partition " << task_partition.name << " was dropped on cluster " << task_table.cluster_push_name); return true; @@ -1272,6 +1304,9 @@ protected: bool tryProcessTable(TaskTable & task_table) { + /// An heuristic: if previous shard is already done, then check next one without sleeps due to max_workers constraint + bool previous_shard_is_instantly_finished = false; + /// Process each partition that is present in cluster for (const String & partition_name : task_table.ordered_partition_names) { @@ -1317,6 +1352,7 @@ protected: else { /// We have already checked that partition, but did not discover it + previous_shard_is_instantly_finished = true; continue; } } @@ -1328,15 +1364,20 @@ protected: expected_shards.emplace_back(shard); + /// Do not sleep if there is a sequence of already processed shards to increase startup + bool is_unprioritized_task = !previous_shard_is_instantly_finished && shard->priority.is_remote; PartitionTaskStatus task_status = PartitionTaskStatus::Error; + bool was_error = false; for (size_t try_num = 0; try_num < max_shard_partition_tries; ++try_num) { - task_status = tryProcessPartitionTask(partition); + task_status = tryProcessPartitionTask(partition, is_unprioritized_task); /// Exit if success if (task_status == PartitionTaskStatus::Finished) break; + was_error = true; + /// Skip if the task is being processed by someone if (task_status == PartitionTaskStatus::Active) break; @@ -1347,6 +1388,8 @@ protected: if (task_status == PartitionTaskStatus::Error) ++num_failed_shards; + + previous_shard_is_instantly_finished = !was_error; } cluster_partition.elapsed_time_seconds += watch.elapsedSeconds(); @@ -1413,13 +1456,13 @@ protected: Error, }; - PartitionTaskStatus tryProcessPartitionTask(ShardPartition & task_partition) + PartitionTaskStatus tryProcessPartitionTask(ShardPartition & task_partition, bool is_unprioritized_task) { PartitionTaskStatus res; try { - res = processPartitionTaskImpl(task_partition); + res = processPartitionTaskImpl(task_partition, is_unprioritized_task); } catch (...) { @@ -1440,14 +1483,13 @@ protected: return res; } - PartitionTaskStatus processPartitionTaskImpl(ShardPartition & task_partition) + PartitionTaskStatus processPartitionTaskImpl(ShardPartition & task_partition, bool is_unprioritized_task) { TaskShard & task_shard = task_partition.task_shard; TaskTable & task_table = task_shard.task_table; ClusterPartition & cluster_partition = task_table.getClusterPartition(task_partition.name); - auto zookeeper = getZooKeeper(); - auto acl = zookeeper->getDefaultACL(); + auto zookeeper = context.getZooKeeper(); String is_dirty_flag_path = task_partition.getCommonPartitionIsDirtyPath(); String current_task_is_active_path = task_partition.getActiveWorkerPath(); @@ -1459,8 +1501,8 @@ protected: auto create_is_dirty_node = [&] () { auto code = zookeeper->tryCreate(is_dirty_flag_path, current_task_status_path, zkutil::CreateMode::Persistent); - if (code != ZOK && code != ZNODEEXISTS) - throw zkutil::KeeperException(code, is_dirty_flag_path); + if (code && code != Coordination::ZNODEEXISTS) + throw Coordination::Exception(code, is_dirty_flag_path); }; /// Returns SELECT query filtering current partition and applying user filter @@ -1469,18 +1511,18 @@ protected: String query; query += "SELECT " + fields + " FROM " + getDatabaseDotTable(from_table); /// TODO: Bad, it is better to rewrite with ASTLiteral(partition_key_field) - query += " WHERE (" + queryToString(task_table.engine_push_partition_key_ast) + " = " + task_partition.name + ")"; + query += " WHERE (" + queryToString(task_table.engine_push_partition_key_ast) + " = (" + task_partition.name + " AS partition_key))"; if (!task_table.where_condition_str.empty()) query += " AND (" + task_table.where_condition_str + ")"; if (!limit.empty()) query += " LIMIT " + limit; ParserQuery p_query(query.data() + query.size()); - return parseQuery(p_query, query); + return parseQuery(p_query, query, 0); }; /// Load balancing - auto worker_node_holder = createTaskWorkerNodeAndWaitIfNeed(zookeeper, current_task_status_path, task_shard.priority.is_remote); + auto worker_node_holder = createTaskWorkerNodeAndWaitIfNeed(zookeeper, current_task_status_path, is_unprioritized_task); LOG_DEBUG(log, "Processing " << current_task_status_path); @@ -1495,7 +1537,7 @@ protected: } catch (...) { - tryLogCurrentException(log, "An error occurred while clean partition"); + tryLogCurrentException(log, "An error occurred when clean partition"); } return PartitionTaskStatus::Error; @@ -1508,9 +1550,9 @@ protected: { partition_task_node_holder = zkutil::EphemeralNodeHolder::create(current_task_is_active_path, *zookeeper, host_id); } - catch (const zkutil::KeeperException & e) + catch (const Coordination::Exception & e) { - if (e.code == ZNODEEXISTS) + if (e.code == Coordination::ZNODEEXISTS) { LOG_DEBUG(log, "Someone is already processing " << current_task_is_active_path); return PartitionTaskStatus::Active; @@ -1561,7 +1603,7 @@ protected: if (count != 0) { - zkutil::Stat stat_shards; + Coordination::Stat stat_shards; zookeeper->get(task_partition.getPartitionShardsPath(), &stat_shards); if (stat_shards.numChildren == 0) @@ -1579,18 +1621,20 @@ protected: /// Try start processing, create node about it { String start_state = TaskStateWithOwner::getData(TaskState::Started, host_id); - auto op_create = std::make_shared(current_task_status_path, start_state, acl, zkutil::CreateMode::Persistent); - zkutil::MultiTransactionInfo info = checkNoNodeAndCommit(zookeeper, is_dirty_flag_path, std::move(op_create)); + auto op_create = zkutil::makeCreateRequest(current_task_status_path, start_state, zkutil::CreateMode::Persistent); + MultiTransactionInfo info = checkNoNodeAndCommit(zookeeper, is_dirty_flag_path, std::move(op_create)); - if (info.code != ZOK) + if (info.code) { - if (info.getFailedOp().getPath() == is_dirty_flag_path) + zkutil::KeeperMultiException exception(info.code, info.requests, info.responses); + + if (exception.getPathForFirstFailedOp() == is_dirty_flag_path) { LOG_INFO(log, "Partition " << task_partition.name << " is dirty and will be dropped and refilled"); return PartitionTaskStatus::Error; } - throw zkutil::KeeperException(info.code, current_task_status_path); + throw exception; } } @@ -1612,8 +1656,7 @@ protected: bool inject_fault = false; if (copy_fault_probability > 0) { - std::uniform_real_distribution<> get_urand(0, 1); - double value = get_urand(task_table.task_cluster.random_engine); + double value = std::uniform_real_distribution<>(0, 1)(task_table.task_cluster.random_engine); inject_fault = value < copy_fault_probability; } @@ -1629,7 +1672,7 @@ protected: query += "INSERT INTO " + getDatabaseDotTable(task_shard.table_split_shard) + " VALUES "; ParserQuery p_query(query.data() + query.size()); - query_insert_ast = parseQuery(p_query, query); + query_insert_ast = parseQuery(p_query, query, 0); LOG_DEBUG(log, "Executing INSERT query: " << query); } @@ -1649,12 +1692,11 @@ protected: BlockIO io_select = InterpreterFactory::get(query_select_ast, context_select)->execute(); BlockIO io_insert = InterpreterFactory::get(query_insert_ast, context_insert)->execute(); - input = std::make_shared(io_select.in); + input = io_select.in; output = io_insert.out; } - using ExistsFuture = zkutil::ZooKeeper::ExistsFuture; - auto future_is_dirty_checker = std::make_unique(zookeeper->asyncExists(is_dirty_flag_path)); + std::future future_is_dirty_checker; Stopwatch watch(CLOCK_MONOTONIC_COARSE); constexpr size_t check_period_milliseconds = 500; @@ -1665,32 +1707,17 @@ protected: if (zookeeper->expired()) throw Exception("ZooKeeper session is expired, cancel INSERT SELECT", ErrorCodes::UNFINISHED); - if (future_is_dirty_checker != nullptr) - { - zkutil::ZooKeeper::StatAndExists status; - try - { - status = future_is_dirty_checker->get(); - future_is_dirty_checker.reset(); - } - catch (zkutil::KeeperException & e) - { - future_is_dirty_checker.reset(); - - if (e.isTemporaryError()) - LOG_INFO(log, "ZooKeeper is lagging: " << e.displayText()); - else - throw; - } - - if (status.exists) - throw Exception("Partition is dirty, cancel INSERT SELECT", ErrorCodes::UNFINISHED); - } + if (!future_is_dirty_checker.valid()) + future_is_dirty_checker = zookeeper->asyncExists(is_dirty_flag_path); + /// check_period_milliseconds should less than average insert time of single block + /// Otherwise, the insertion will slow a little bit if (watch.elapsedMilliseconds() >= check_period_milliseconds) { - watch.restart(); - future_is_dirty_checker = std::make_unique(zookeeper->asyncExists(is_dirty_flag_path)); + Coordination::ExistsResponse status = future_is_dirty_checker.get(); + + if (status.error != Coordination::ZNONODE) + throw Exception("Partition is dirty, cancel INSERT SELECT", ErrorCodes::UNFINISHED); } return false; @@ -1709,7 +1736,7 @@ protected: copyData(*input, *output, cancel_check, update_stats); // Just in case - if (future_is_dirty_checker != nullptr) + if (future_is_dirty_checker.valid()) future_is_dirty_checker.get(); if (inject_fault) @@ -1725,12 +1752,14 @@ protected: /// Finalize the processing, change state of current partition task (and also check is_dirty flag) { String state_finished = TaskStateWithOwner::getData(TaskState::Finished, host_id); - auto op_set = std::make_shared(current_task_status_path, state_finished, 0); - zkutil::MultiTransactionInfo info = checkNoNodeAndCommit(zookeeper, is_dirty_flag_path, std::move(op_set)); + auto op_set = zkutil::makeSetRequest(current_task_status_path, state_finished, 0); + MultiTransactionInfo info = checkNoNodeAndCommit(zookeeper, is_dirty_flag_path, std::move(op_set)); - if (info.code != ZOK) + if (info.code) { - if (info.getFailedOp().getPath() == is_dirty_flag_path) + zkutil::KeeperMultiException exception(info.code, info.requests, info.responses); + + if (exception.getPathForFirstFailedOp() == is_dirty_flag_path) LOG_INFO(log, "Partition " << task_partition.name << " became dirty and will be dropped and refilled"); else LOG_INFO(log, "Someone made the node abandoned. Will refill partition. " << zkutil::ZooKeeper::error2string(info.code)); @@ -1780,7 +1809,7 @@ protected: &task_cluster->settings_pull); ParserCreateQuery parser_create_query; - return parseQuery(parser_create_query, create_query_pull_str); + return parseQuery(parser_create_query, create_query_pull_str, 0); } void createShardInternalTables(TaskShard & task_shard, bool create_split = true) @@ -1831,7 +1860,7 @@ protected: } ParserQuery parser_query(query.data() + query.size()); - ASTPtr query_ast = parseQuery(parser_query, query); + ASTPtr query_ast = parseQuery(parser_query, query, 0); LOG_DEBUG(log, "Computing destination partition set, executing query: " << query); @@ -1848,7 +1877,7 @@ protected: for (size_t i = 0; i < column.column->size(); ++i) { WriteBufferFromOwnString wb; - column.type->serializeTextQuoted(*column.column, i, wb); + column.type->serializeTextQuoted(*column.column, i, wb, FormatSettings()); res.emplace(wb.str()); } } @@ -1878,7 +1907,7 @@ protected: << partition_quoted_name << " existence, executing query: " << query); ParserQuery parser_query(query.data() + query.size()); - ASTPtr query_ast = parseQuery(parser_query, query); + ASTPtr query_ast = parseQuery(parser_query, query, 0); Context local_context = context; local_context.setSettings(task_cluster->settings_pull); @@ -1903,7 +1932,7 @@ protected: if (query_ast_ == nullptr) { ParserQuery p_query(query.data() + query.size()); - query_ast = parseQuery(p_query, query); + query_ast = parseQuery(p_query, query, 0); } else query_ast = query_ast_; @@ -1959,7 +1988,7 @@ protected: if (increment_and_check_exit()) return; } - catch (const Exception & e) + catch (const Exception &) { LOG_INFO(log, getCurrentExceptionMessage(false, true)); } @@ -1983,21 +2012,7 @@ protected: return successful_shards; } - zkutil::ZooKeeperPtr getZooKeeper() - { - auto zookeeper = context.getZooKeeper(); - - if (!zookeeper) - { - context.setZooKeeper(std::make_shared(*zookeeper_config, "zookeeper")); - zookeeper = context.getZooKeeper(); - } - - return zookeeper; - } - private: - ConfigurationPtr zookeeper_config; String task_zookeeper_path; String task_description_path; String host_id; @@ -2006,13 +2021,13 @@ private: /// Auto update config stuff UInt64 task_descprtion_current_version = 1; std::atomic task_descprtion_version{1}; - zkutil::WatchCallback task_description_watch_callback; + Coordination::WatchCallback task_description_watch_callback; /// ZooKeeper session used to set the callback zkutil::ZooKeeperPtr task_description_watch_zookeeper; ConfigurationPtr task_cluster_initial_config; ConfigurationPtr task_cluster_current_config; - zkutil::Stat task_descprtion_current_stat; + Coordination::Stat task_descprtion_current_stat; std::unique_ptr task_cluster; @@ -2031,8 +2046,6 @@ private: void ClusterCopierApp::initialize(Poco::Util::Application & self) { - Poco::Util::ServerApplication::initialize(self); - is_help = config().has("help"); if (is_help) return; @@ -2054,11 +2067,17 @@ void ClusterCopierApp::initialize(Poco::Util::Application & self) process_path = Poco::Path(base_dir + "/clickhouse-copier_" + process_id).absolute().toString(); Poco::File(process_path).createDirectories(); - setupLogging(); + /// Override variables for BaseDaemon + if (config().has("log-level")) + config().setString("logger.level", config().getString("log-level")); - std::string stderr_path = process_path + "/stderr"; - if (!freopen(stderr_path.c_str(), "a+", stderr)) - throw Poco::OpenFileException("Cannot attach stderr to " + stderr_path); + if (config().has("base-dir") || !config().has("logger.log")) + config().setString("logger.log", process_path + "/log.log"); + + if (config().has("base-dir") || !config().has("logger.errorlog")) + config().setString("logger.errorlog", process_path + "/log.err.log"); + + Base::initialize(self); } @@ -2076,10 +2095,8 @@ void ClusterCopierApp::handleHelp(const std::string &, const std::string &) void ClusterCopierApp::defineOptions(Poco::Util::OptionSet & options) { - Poco::Util::ServerApplication::defineOptions(options); + Base::defineOptions(options); - options.addOption(Poco::Util::Option("config-file", "c", "path to config file with ZooKeeper config", true) - .argument("config-file").binding("config-file")); options.addOption(Poco::Util::Option("task-path", "", "path to task in ZooKeeper") .argument("task-path").binding("task-path")); options.addOption(Poco::Util::Option("safe-mode", "", "disables ALTER DROP PARTITION in case of errors") @@ -2097,40 +2114,11 @@ void ClusterCopierApp::defineOptions(Poco::Util::OptionSet & options) } -void ClusterCopierApp::setupLogging() -{ - Poco::AutoPtr split_channel(new Poco::SplitterChannel); - - Poco::AutoPtr log_file_channel(new Poco::FileChannel); - log_file_channel->setProperty("path", process_path + "/log.log"); - split_channel->addChannel(log_file_channel); - log_file_channel->open(); - - if (!config().getBool("application.runAsDaemon", true)) - { - Poco::AutoPtr console_channel(new Poco::ConsoleChannel); - split_channel->addChannel(console_channel); - console_channel->open(); - } - - Poco::AutoPtr formatter = new OwnPatternFormatter(nullptr); - formatter->setProperty("times", "local"); - Poco::AutoPtr formatting_channel(new Poco::FormattingChannel(formatter)); - formatting_channel->setChannel(split_channel); - split_channel->open(); - - Poco::Logger::root().setChannel(formatting_channel); - Poco::Logger::root().setLevel(log_level); -} - - void ClusterCopierApp::mainImpl() { - ConfigurationPtr zookeeper_configuration(new Poco::Util::XMLConfiguration(config_xml_path)); - auto log = &logger(); - StatusFile status_file(process_path + "/status"); + auto log = &logger(); LOG_INFO(log, "Starting clickhouse-copier (" << "id " << process_id << ", " << "host_id " << host_id << ", " @@ -2140,6 +2128,7 @@ void ClusterCopierApp::mainImpl() auto context = std::make_unique(Context::createGlobal()); SCOPE_EXIT(context->shutdown()); + context->setConfig(loaded_config.configuration); context->setGlobalContext(*context); context->setApplicationType(Context::ApplicationType::LOCAL); context->setPath(process_path); @@ -2153,9 +2142,10 @@ void ClusterCopierApp::mainImpl() context->addDatabase(default_database, std::make_shared(default_database)); context->setCurrentDatabase(default_database); - std::unique_ptr copier(new ClusterCopier( - zookeeper_configuration, task_path, host_id, default_database, *context)); + /// Initialize query scope just in case. + CurrentThread::QueryScope query_scope(*context); + auto copier = std::make_unique(task_path, host_id, default_database, *context); copier->setSafeMode(is_safe_mode); copier->setCopyFaultProbability(copy_fault_probability); copier->init(); diff --git a/dbms/src/Server/ClusterCopier.h b/dbms/programs/copier/ClusterCopier.h similarity index 97% rename from dbms/src/Server/ClusterCopier.h rename to dbms/programs/copier/ClusterCopier.h index 347a1d8f645..89f45df8686 100644 --- a/dbms/src/Server/ClusterCopier.h +++ b/dbms/programs/copier/ClusterCopier.h @@ -1,5 +1,6 @@ #pragma once #include +#include /* clickhouse cluster copier util * Copies tables data from one cluster to new tables of other (possibly the same) cluster in distributed fault-tolerant manner. @@ -52,7 +53,7 @@ namespace DB { -class ClusterCopierApp : public Poco::Util::ServerApplication +class ClusterCopierApp : public BaseDaemon { public: @@ -66,6 +67,8 @@ public: private: + using Base = BaseDaemon; + void mainImpl(); void setupLogging(); diff --git a/dbms/src/Server/clickhouse-copier.cpp b/dbms/programs/copier/clickhouse-copier.cpp similarity index 100% rename from dbms/src/Server/clickhouse-copier.cpp rename to dbms/programs/copier/clickhouse-copier.cpp diff --git a/dbms/programs/extract-from-config/CMakeLists.txt b/dbms/programs/extract-from-config/CMakeLists.txt new file mode 100644 index 00000000000..62253649368 --- /dev/null +++ b/dbms/programs/extract-from-config/CMakeLists.txt @@ -0,0 +1,7 @@ +add_library (clickhouse-extract-from-config-lib ${LINK_MODE} ExtractFromConfig.cpp) +target_link_libraries (clickhouse-extract-from-config-lib PRIVATE clickhouse_common_config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY}) + +if (CLICKHOUSE_SPLIT_BINARY) + add_executable (clickhouse-extract-from-config clickhouse-extract-from-config.cpp) + target_link_libraries (clickhouse-extract-from-config PRIVATE clickhouse-extract-from-config-lib) +endif () diff --git a/dbms/src/Server/ExtractFromConfig.cpp b/dbms/programs/extract-from-config/ExtractFromConfig.cpp similarity index 90% rename from dbms/src/Server/ExtractFromConfig.cpp rename to dbms/programs/extract-from-config/ExtractFromConfig.cpp index 7f0122256dd..af9550e4547 100644 --- a/dbms/src/Server/ExtractFromConfig.cpp +++ b/dbms/programs/extract-from-config/ExtractFromConfig.cpp @@ -26,18 +26,18 @@ static void setupLogging(const std::string & log_level) static std::string extractFromConfig( const std::string & config_path, const std::string & key, bool process_zk_includes, bool try_get = false) { - ConfigProcessor processor(config_path, /* throw_on_bad_incl = */ false, /* log_to_console = */ false); + DB::ConfigProcessor processor(config_path, /* throw_on_bad_incl = */ false, /* log_to_console = */ false); bool has_zk_includes; - XMLDocumentPtr config_xml = processor.processConfig(&has_zk_includes); + DB::XMLDocumentPtr config_xml = processor.processConfig(&has_zk_includes); if (has_zk_includes && process_zk_includes) { - ConfigurationPtr bootstrap_configuration(new Poco::Util::XMLConfiguration(config_xml)); + DB::ConfigurationPtr bootstrap_configuration(new Poco::Util::XMLConfiguration(config_xml)); zkutil::ZooKeeperPtr zookeeper = std::make_shared( *bootstrap_configuration, "zookeeper"); zkutil::ZooKeeperNodeCache zk_node_cache([&] { return zookeeper; }); config_xml = processor.processConfig(&has_zk_includes, &zk_node_cache); } - ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml)); + DB::ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml)); // do not throw exception if not found if (try_get) return configuration->getString(key, ""); diff --git a/dbms/src/Server/clickhouse-extract-from-config.cpp b/dbms/programs/extract-from-config/clickhouse-extract-from-config.cpp similarity index 100% rename from dbms/src/Server/clickhouse-extract-from-config.cpp rename to dbms/programs/extract-from-config/clickhouse-extract-from-config.cpp diff --git a/dbms/programs/format/CMakeLists.txt b/dbms/programs/format/CMakeLists.txt new file mode 100644 index 00000000000..53d09e82621 --- /dev/null +++ b/dbms/programs/format/CMakeLists.txt @@ -0,0 +1,6 @@ +add_library (clickhouse-format-lib ${LINK_MODE} Format.cpp) +target_link_libraries (clickhouse-format-lib PRIVATE dbms clickhouse_common_io clickhouse_parsers ${Boost_PROGRAM_OPTIONS_LIBRARY}) +if (CLICKHOUSE_SPLIT_BINARY) + add_executable (clickhouse-format clickhouse-format.cpp) + target_link_libraries (clickhouse-format PRIVATE clickhouse-format-lib) +endif () diff --git a/dbms/src/Server/Format.cpp b/dbms/programs/format/Format.cpp similarity index 96% rename from dbms/src/Server/Format.cpp rename to dbms/programs/format/Format.cpp index 2cd6be1efe7..b7e2629df16 100644 --- a/dbms/src/Server/Format.cpp +++ b/dbms/programs/format/Format.cpp @@ -50,7 +50,7 @@ int mainEntryClickHouseFormat(int argc, char ** argv) const char * end = pos + query.size(); ParserQuery parser(end); - ASTPtr res = parseQuery(parser, pos, end, "query"); + ASTPtr res = parseQuery(parser, pos, end, "query", 0); if (!quiet) { diff --git a/dbms/src/Server/clickhouse-format.cpp b/dbms/programs/format/clickhouse-format.cpp similarity index 100% rename from dbms/src/Server/clickhouse-format.cpp rename to dbms/programs/format/clickhouse-format.cpp diff --git a/dbms/programs/local/CMakeLists.txt b/dbms/programs/local/CMakeLists.txt new file mode 100644 index 00000000000..07729d68563 --- /dev/null +++ b/dbms/programs/local/CMakeLists.txt @@ -0,0 +1,7 @@ +add_library (clickhouse-local-lib ${LINK_MODE} LocalServer.cpp) +target_link_libraries (clickhouse-local-lib PRIVATE clickhouse_common_io clickhouse-server-lib clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Boost_PROGRAM_OPTIONS_LIBRARY}) + +if (CLICKHOUSE_SPLIT_BINARY) + add_executable (clickhouse-local clickhouse-local.cpp) + target_link_libraries (clickhouse-local PRIVATE clickhouse-local-lib) +endif () diff --git a/dbms/src/Server/LocalServer.cpp b/dbms/programs/local/LocalServer.cpp similarity index 51% rename from dbms/src/Server/LocalServer.cpp rename to dbms/programs/local/LocalServer.cpp index 1196c62f06f..40e26438afc 100644 --- a/dbms/src/Server/LocalServer.cpp +++ b/dbms/programs/local/LocalServer.cpp @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include #include @@ -16,17 +16,23 @@ #include #include #include +#include +#include #include #include #include +#include #include #include #include -#include "StatusFile.h" +#include #include #include #include #include +#include +#include +#include namespace DB @@ -53,178 +59,42 @@ void LocalServer::initialize(Poco::Util::Application & self) Poco::Util::Application::initialize(self); // Turn off server logging to stderr - if (config().has("silent")) + if (!config().has("verbose")) { Poco::Logger::root().setLevel("none"); Poco::Logger::root().setChannel(Poco::AutoPtr(new Poco::NullChannel())); } } - -void LocalServer::defineOptions(Poco::Util::OptionSet& _options) +void LocalServer::applyCmdSettings(Context & context) { - Poco::Util::Application::defineOptions (_options); - - _options.addOption( - Poco::Util::Option("config-file", "", "Load configuration from a given file") - .required(false) - .repeatable(false) - .argument("[config.xml]") - .binding("config-file")); - - /// Arguments that define first query creating initial table: - /// (If structure argument is omitted then initial query is not generated) - _options.addOption( - Poco::Util::Option("structure", "S", "Structure of initial table(list columns names with their types)") - .required(false) - .repeatable(false) - .argument("[name Type]") - .binding("table-structure")); - - /// Turn off logging - _options.addOption( - Poco::Util::Option("silent", "s", "Quiet mode, print only errors") - .required(false) - .repeatable(false) - .binding("silent")); - - _options.addOption( - Poco::Util::Option("table", "N", "Name of initial table") - .required(false) - .repeatable(false) - .argument("[table]") - .binding("table-name")); - - _options.addOption( - Poco::Util::Option("file", "f", "Path to file with data of initial table (stdin if not specified)") - .required(false) - .repeatable(false) - .argument(" stdin") - .binding("table-file")); - - _options.addOption( - Poco::Util::Option("input-format", "if", "Input format of initial table data") - .required(false) - .repeatable(false) - .argument("") - .binding("table-data-format")); - - /// List of queries to execute - _options.addOption( - Poco::Util::Option("query", "q", "Queries to execute") - .required(false) - .repeatable(false) - .argument("") - .binding("query")); - - /// Default Output format - _options.addOption( - Poco::Util::Option("output-format", "of", "Default output format") - .required(false) - .repeatable(false) - .argument("[TSV]", true) - .binding("output-format")); - - /// Alias for previous one, required for clickhouse-client compatibility - _options.addOption( - Poco::Util::Option("format", "", "Default output format") - .required(false) - .repeatable(false) - .argument("[TSV]", true) - .binding("format")); - - _options.addOption( - Poco::Util::Option("stacktrace", "", "Print stack traces of exceptions") - .required(false) - .repeatable(false) - .binding("stacktrace")); - - _options.addOption( - Poco::Util::Option("verbose", "", "Print info about execution of queries") - .required(false) - .repeatable(false) - .noArgument() - .binding("verbose")); - - _options.addOption( - Poco::Util::Option("help", "", "Display help information") - .required(false) - .repeatable(false) - .noArgument() - .binding("help") - .callback(Poco::Util::OptionCallback(this, &LocalServer::handleHelp))); - - /// These arrays prevent "variable tracking size limit exceeded" compiler notice. - static const char * settings_names[] = { -#define DECLARE_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) #NAME, - APPLY_FOR_SETTINGS(DECLARE_SETTING) -#undef DECLARE_SETTING - nullptr}; - - for (const char ** name = settings_names; *name; ++name) - _options.addOption(Poco::Util::Option(*name, "", "Settings.h").required(false).argument("") - .repeatable(false).binding(*name)); -} - - -void LocalServer::applyOptions() -{ - context->setDefaultFormat(config().getString("output-format", config().getString("format", "TSV"))); - - /// settings and limits could be specified in config file, but passed settings has higher priority #define EXTRACT_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) \ - if (config().has(#NAME) && !context->getSettingsRef().NAME.changed) \ - context->setSetting(#NAME, config().getString(#NAME)); + if (cmd_settings.NAME.changed) \ + context.getSettingsRef().NAME = cmd_settings.NAME; APPLY_FOR_SETTINGS(EXTRACT_SETTING) #undef EXTRACT_SETTING } - -void LocalServer::displayHelp() -{ - Poco::Util::HelpFormatter helpFormatter(options()); - helpFormatter.setCommand(commandName()); - helpFormatter.setUsage("[initial table definition] [--query ]"); - helpFormatter.setHeader("\n" - "clickhouse-local allows to execute SQL queries on your data files via single command line call.\n" - "To do so, intially you need to define your data source and its format.\n" - "After you can execute your SQL queries in the usual manner.\n" - "There are two ways to define initial table keeping your data:\n" - "either just in first query like this:\n" - " CREATE TABLE () ENGINE = File(, );\n" - "either through corresponding command line parameters." - ); - helpFormatter.setWidth(132); /// 80 is ugly due to wide settings params - - helpFormatter.format(std::cerr); - std::cerr << "Example printing memory used by each Unix user:\n" - "ps aux | tail -n +2 | awk '{ printf(\"%s\\t%s\\n\", $1, $4) }' | " - "clickhouse-local -S \"user String, mem Float64\" -q \"SELECT user, round(sum(mem), 2) as memTotal FROM table GROUP BY user ORDER BY memTotal DESC FORMAT Pretty\"\n"; -} - - -void LocalServer::handleHelp(const std::string & /*name*/, const std::string & /*value*/) -{ - displayHelp(); - stopOptionsProcessing(); -} - - /// If path is specified and not empty, will try to setup server environment and load existing metadata void LocalServer::tryInitPath() { - if (!config().has("path") || (path = config().getString("path")).empty()) - return; - + std::string path = config().getString("path", ""); Poco::trimInPlace(path); - if (path.empty()) + + if (!path.empty()) + { + if (path.back() != '/') + path += '/'; + + context->setPath(path); return; - if (path.back() != '/') - path += '/'; + } - context->setPath(path); - - StatusFile status{path + "status"}; + /// In case of empty path set paths to helpful directories + std::string cd = Poco::Path::current(); + context->setTemporaryPath(cd + "tmp"); + context->setFlagsPath(cd + "flags"); + context->setUserFilesPath(""); // user's files are everywhere } @@ -233,13 +103,12 @@ try { Logger * log = &logger(); + UseSSL use_ssl; + if (!config().has("query") && !config().has("table-structure")) /// Nothing to process { - if (!config().hasOption("help")) - { - std::cerr << "There are no queries to process." << std::endl; - displayHelp(); - } + if (config().hasOption("verbose")) + std::cerr << "There are no queries to process." << '\n'; return Application::EXIT_OK; } @@ -247,9 +116,11 @@ try /// Load config files if exists if (config().has("config-file") || Poco::File("config.xml").exists()) { - ConfigProcessor config_processor(config().getString("config-file", "config.xml"), false, true); + const auto config_path = config().getString("config-file", "config.xml"); + ConfigProcessor config_processor(config_path, false, true); + config_processor.setConfigPath(Poco::Path(config_path).makeParent().toString()); auto loaded_config = config_processor.loadConfig(); - config_processor.savePreprocessedConfig(loaded_config); + config_processor.savePreprocessedConfig(loaded_config, loaded_config.configuration->getString("path", DBMS_DEFAULT_PATH)); config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); } @@ -258,7 +129,7 @@ try context->setApplicationType(Context::ApplicationType::LOCAL); tryInitPath(); - applyOptions(); + std::optional status; /// Skip temp path installation @@ -272,6 +143,7 @@ try registerAggregateFunctions(); registerTableFunctions(); registerStorages(); + registerDictionaries(); /// Maybe useless if (config().has("macros")) @@ -304,13 +176,17 @@ try * Otherwise, metadata of temporary File(format, EXPLICIT_PATH) tables will pollute metadata/ directory; * if such tables will not be dropped, clickhouse-server will not be able to load them due to security reasons. */ - const std::string default_database = "_local"; + std::string default_database = config().getString("default_database", "_local"); context->addDatabase(default_database, std::make_shared(default_database)); context->setCurrentDatabase(default_database); + applyCmdOptions(*context); - if (!path.empty()) + if (!context->getPath().empty()) { - LOG_DEBUG(log, "Loading metadata from " << path); + /// Lock path directory before read + status.emplace(context->getPath() + "status"); + + LOG_DEBUG(log, "Loading metadata from " << context->getPath()); loadMetadataSystem(*context); attachSystemTables(); loadMetadata(*context); @@ -330,20 +206,7 @@ try } catch (const Exception & e) { - bool print_stack_trace = config().has("stacktrace"); - - std::string text = e.displayText(); - - auto embedded_stack_trace_pos = text.find("Stack trace"); - if (std::string::npos != embedded_stack_trace_pos && !print_stack_trace) - text.resize(embedded_stack_trace_pos); - - std::cerr << "Code: " << e.code() << ". " << text << std::endl << std::endl; - - if (print_stack_trace && std::string::npos == embedded_stack_trace_pos) - { - std::cerr << "Stack trace:" << std::endl << e.getStackTrace().toString(); - } + std::cerr << getCurrentExceptionMessage(config().hasOption("stacktrace")) << '\n'; /// If exception code isn't zero, we should return non-zero return code anyway. return e.code() ? e.code() : -1; @@ -397,32 +260,58 @@ void LocalServer::attachSystemTables() void LocalServer::processQueries() { - Logger * log = &logger(); - String initial_create_query = getInitialCreateTableQuery(); String queries_str = initial_create_query + config().getString("query"); - bool verbose = config().hasOption("verbose"); - std::vector queries; auto parse_res = splitMultipartQuery(queries_str, queries); if (!parse_res.second) throw Exception("Cannot parse and execute the following part of query: " + String(parse_res.first), ErrorCodes::SYNTAX_ERROR); + context->setSessionContext(*context); + context->setQueryContext(*context); + context->setUser("default", "", Poco::Net::SocketAddress{}, ""); context->setCurrentQueryId(""); + applyCmdSettings(*context); + + /// Use the same query_id (and thread group) for all queries + CurrentThread::QueryScope query_scope_holder(*context); + + bool echo_queries = config().hasOption("echo") || config().hasOption("verbose"); + std::exception_ptr exception; for (const auto & query : queries) { ReadBufferFromString read_buf(query); WriteBufferFromFileDescriptor write_buf(STDOUT_FILENO); - if (verbose) - LOG_INFO(log, "Executing query: " << query); + if (echo_queries) + { + writeString(query, write_buf); + writeChar('\n', write_buf); + write_buf.next(); + } - executeQuery(read_buf, write_buf, /* allow_into_outfile = */ true, *context, {}); + try + { + executeQuery(read_buf, write_buf, /* allow_into_outfile = */ true, *context, {}); + } + catch (...) + { + if (!config().hasOption("ignore-error")) + throw; + + if (!exception) + exception = std::current_exception(); + + std::cerr << getCurrentExceptionMessage(config().hasOption("stacktrace")) << '\n'; + } } + + if (exception) + std::rethrow_exception(exception); } static const char * minimal_default_user_xml = @@ -463,7 +352,7 @@ void LocalServer::setupUsers() const auto users_config_path = config().getString("users_config", config().getString("config-file", "config.xml")); ConfigProcessor config_processor(users_config_path); const auto loaded_config = config_processor.loadConfig(); - config_processor.savePreprocessedConfig(loaded_config); + config_processor.savePreprocessedConfig(loaded_config, config().getString("path", DBMS_DEFAULT_PATH)); users_config = loaded_config.configuration; } else @@ -477,6 +366,140 @@ void LocalServer::setupUsers() throw Exception("Can't load config for users", ErrorCodes::CANNOT_LOAD_CONFIG); } +static void showClientVersion() +{ + std::cout << DBMS_NAME << " client version " << VERSION_STRING << "." << '\n'; +} + +std::string LocalServer::getHelpHeader() const +{ + return + "usage: clickhouse-local [initial table definition] [--query ]\n" + + "clickhouse-local allows to execute SQL queries on your data files via single command line call." + " To do so, initially you need to define your data source and its format." + " After you can execute your SQL queries in usual manner.\n" + + "There are two ways to define initial table keeping your data." + " Either just in first query like this:\n" + " CREATE TABLE
() ENGINE = File(, );\n" + "Either through corresponding command line parameters --table --structure --input-format and --file."; +} + +std::string LocalServer::getHelpFooter() const +{ + return + "Example printing memory used by each Unix user:\n" + "ps aux | tail -n +2 | awk '{ printf(\"%s\\t%s\\n\", $1, $4) }' | " + "clickhouse-local -S \"user String, mem Float64\" -q" + " \"SELECT user, round(sum(mem), 2) as mem_total FROM table GROUP BY user ORDER" + " BY mem_total DESC FORMAT PrettyCompact\""; +} + +void LocalServer::init(int argc, char ** argv) +{ + namespace po = boost::program_options; + + /// Don't parse options with Poco library, we prefer neat boost::program_options + stopOptionsProcessing(); + + unsigned line_length = po::options_description::m_default_line_length; + unsigned min_description_length = line_length / 2; + if (isatty(STDIN_FILENO)) + { + winsize terminal_size{}; + ioctl(0, TIOCGWINSZ, &terminal_size); + line_length = std::max(3U, static_cast(terminal_size.ws_col)); + min_description_length = std::min(min_description_length, line_length - 2); + } + +#define DECLARE_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) (#NAME, po::value (), DESCRIPTION) + po::options_description description("Main options", line_length, min_description_length); + description.add_options() + ("help", "produce help message") + ("config-file,c", po::value(), "config-file path") + ("query,q", po::value(), "query") + ("database,d", po::value(), "database") + + ("table,N", po::value(), "name of the initial table") + /// If structure argument is omitted then initial query is not generated + ("structure,S", po::value(), "structure of the initial table (list of column and type names)") + ("file,f", po::value(), "path to file with data of the initial table (stdin if not specified)") + ("input-format", po::value(), "input format of the initial table data") + ("format,f", po::value(), "default output format (clickhouse-client compatibility)") + ("output-format", po::value(), "default output format") + + ("stacktrace", "print stack traces of exceptions") + ("echo", "print query before execution") + ("verbose", "print query and other debugging info") + ("ignore-error", "do not stop processing if a query failed") + ("version,V", "print version information and exit") + APPLY_FOR_SETTINGS(DECLARE_SETTING); +#undef DECLARE_SETTING + + /// Parse main commandline options. + po::parsed_options parsed = po::command_line_parser(argc, argv).options(description).run(); + po::variables_map options; + po::store(parsed, options); + + if (options.count("version") || options.count("V")) + { + showClientVersion(); + exit(0); + } + + if (options.count("help")) + { + std::cout << getHelpHeader() << "\n"; + std::cout << description << "\n"; + std::cout << getHelpFooter() << "\n"; + exit(0); + } + + /// Extract settings and limits from the options. +#define EXTRACT_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) \ + if (options.count(#NAME)) \ + cmd_settings.set(#NAME, options[#NAME].as()); + APPLY_FOR_SETTINGS(EXTRACT_SETTING) +#undef EXTRACT_SETTING + + /// Save received data into the internal config. + if (options.count("config-file")) + config().setString("config-file", options["config-file"].as()); + if (options.count("query")) + config().setString("query", options["query"].as()); + if (options.count("database")) + config().setString("default_database", options["database"].as()); + + if (options.count("table")) + config().setString("table-name", options["table"].as()); + if (options.count("file")) + config().setString("table-file", options["file"].as()); + if (options.count("structure")) + config().setString("table-structure", options["structure"].as()); + if (options.count("input-format")) + config().setString("table-data-format", options["input-format"].as()); + if (options.count("format")) + config().setString("format", options["format"].as()); + if (options.count("output-format")) + config().setString("output-format", options["output-format"].as()); + + if (options.count("stacktrace")) + config().setBool("stacktrace", true); + if (options.count("echo")) + config().setBool("echo", true); + if (options.count("verbose")) + config().setBool("verbose", true); + if (options.count("ignore-error")) + config().setBool("ignore-error", true); +} + +void LocalServer::applyCmdOptions(Context & context) +{ + context.setDefaultFormat(config().getString("output-format", config().getString("format", "TSV"))); + applyCmdSettings(context); +} + } int mainEntryClickHouseLocal(int argc, char ** argv) @@ -489,7 +512,7 @@ int mainEntryClickHouseLocal(int argc, char ** argv) } catch (...) { - std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; + std::cerr << DB::getCurrentExceptionMessage(true) << '\n'; auto code = DB::getCurrentExceptionCode(); return code ? code : 1; } diff --git a/dbms/src/Server/LocalServer.h b/dbms/programs/local/LocalServer.h similarity index 73% rename from dbms/src/Server/LocalServer.h rename to dbms/programs/local/LocalServer.h index 5a4bb28233b..244c7bda0b9 100644 --- a/dbms/src/Server/LocalServer.h +++ b/dbms/programs/local/LocalServer.h @@ -1,8 +1,10 @@ #pragma once +#include #include #include + namespace DB { @@ -14,19 +16,17 @@ class Context; class LocalServer : public Poco::Util::Application { public: - LocalServer(); void initialize(Poco::Util::Application & self) override; - void defineOptions(Poco::Util::OptionSet& _options) override; - int main(const std::vector & args) override; - ~LocalServer(); + void init(int argc, char ** argv); + + ~LocalServer() override; private: - /** Composes CREATE subquery based on passed arguments (--structure --file --table and --input-format) * This query will be executed first, before queries passed through --query argument * Returns empty string if it cannot compose that query. @@ -34,17 +34,20 @@ private: std::string getInitialCreateTableQuery(); void tryInitPath(); - void applyOptions(); + void applyCmdOptions(Context & context); + void applyCmdSettings(Context & context); void attachSystemTables(); void processQueries(); void setupUsers(); - void displayHelp(); - void handleHelp(const std::string & name, const std::string & value); + + std::string getHelpHeader() const; + std::string getHelpFooter() const; protected: - std::unique_ptr context; - std::string path; + + /// Settings specified via command line args + Settings cmd_settings; }; } diff --git a/dbms/src/Server/clickhouse-local.cpp b/dbms/programs/local/clickhouse-local.cpp similarity index 100% rename from dbms/src/Server/clickhouse-local.cpp rename to dbms/programs/local/clickhouse-local.cpp diff --git a/dbms/src/Server/main.cpp b/dbms/programs/main.cpp similarity index 63% rename from dbms/src/Server/main.cpp rename to dbms/programs/main.cpp index c174dc1a3d0..29d64213d9c 100644 --- a/dbms/src/Server/main.cpp +++ b/dbms/programs/main.cpp @@ -1,45 +1,65 @@ - #include +#include +#include +#include /// pair + +#if __has_include("config_tools.h") +#include "config_tools.h" +#endif +#if __has_include() /// "Arcadia" build system lacks configure files. #include +#endif +#if __has_include() #include -#include +#endif #if USE_TCMALLOC -#include +#include // Y_IGNORE #endif #if ENABLE_CLICKHOUSE_SERVER -#include "Server.h" +#include "server/Server.h" #endif #if ENABLE_CLICKHOUSE_LOCAL -#include "LocalServer.h" +#include "local/LocalServer.h" #endif #include /// Universal executable for various clickhouse applications -#if ENABLE_CLICKHOUSE_SERVER +#if ENABLE_CLICKHOUSE_SERVER || !defined(ENABLE_CLICKHOUSE_SERVER) int mainEntryClickHouseServer(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_CLIENT +#if ENABLE_CLICKHOUSE_CLIENT || !defined(ENABLE_CLICKHOUSE_CLIENT) int mainEntryClickHouseClient(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_LOCAL +#if ENABLE_CLICKHOUSE_LOCAL || !defined(ENABLE_CLICKHOUSE_LOCAL) int mainEntryClickHouseLocal(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_BENCHMARK +#if ENABLE_CLICKHOUSE_BENCHMARK || !defined(ENABLE_CLICKHOUSE_BENCHMARK) int mainEntryClickHouseBenchmark(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_PERFORMANCE +#if ENABLE_CLICKHOUSE_PERFORMANCE || !defined(ENABLE_CLICKHOUSE_PERFORMANCE) int mainEntryClickHousePerformanceTest(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_TOOLS +#if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG || !defined(ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG) int mainEntryClickHouseExtractFromConfig(int argc, char ** argv); +#endif +#if ENABLE_CLICKHOUSE_COMPRESSOR || !defined(ENABLE_CLICKHOUSE_COMPRESSOR) int mainEntryClickHouseCompressor(int argc, char ** argv); +#endif +#if ENABLE_CLICKHOUSE_FORMAT || !defined(ENABLE_CLICKHOUSE_FORMAT) int mainEntryClickHouseFormat(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_COPIER +#if ENABLE_CLICKHOUSE_COPIER || !defined(ENABLE_CLICKHOUSE_COPIER) int mainEntryClickHouseClusterCopier(int argc, char ** argv); #endif +#if ENABLE_CLICKHOUSE_OBFUSCATOR +int mainEntryClickHouseObfuscator(int argc, char ** argv); +#endif +#if ENABLE_CLICKHOUSE_ODBC_BRIDGE || !defined(ENABLE_CLICKHOUSE_ODBC_BRIDGE) +int mainEntryClickHouseODBCBridge(int argc, char ** argv); +#endif + #if USE_EMBEDDED_COMPILER int mainEntryClickHouseClang(int argc, char ** argv); @@ -55,29 +75,40 @@ using MainFunc = int (*)(int, char**); /// Add an item here to register new application std::pair clickhouse_applications[] = { -#if ENABLE_CLICKHOUSE_LOCAL +#if ENABLE_CLICKHOUSE_LOCAL || !defined(ENABLE_CLICKHOUSE_LOCAL) {"local", mainEntryClickHouseLocal}, #endif -#if ENABLE_CLICKHOUSE_CLIENT +#if ENABLE_CLICKHOUSE_CLIENT || !defined(ENABLE_CLICKHOUSE_CLIENT) {"client", mainEntryClickHouseClient}, #endif -#if ENABLE_CLICKHOUSE_BENCHMARK +#if ENABLE_CLICKHOUSE_BENCHMARK || !defined(ENABLE_CLICKHOUSE_BENCHMARK) {"benchmark", mainEntryClickHouseBenchmark}, #endif -#if ENABLE_CLICKHOUSE_SERVER +#if ENABLE_CLICKHOUSE_SERVER || !defined(ENABLE_CLICKHOUSE_SERVER) {"server", mainEntryClickHouseServer}, #endif -#if ENABLE_CLICKHOUSE_PERFORMANCE +#if ENABLE_CLICKHOUSE_PERFORMANCE || !defined(ENABLE_CLICKHOUSE_PERFORMANCE) {"performance-test", mainEntryClickHousePerformanceTest}, #endif -#if ENABLE_CLICKHOUSE_TOOLS +#if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG || !defined(ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG) {"extract-from-config", mainEntryClickHouseExtractFromConfig}, +#endif +#if ENABLE_CLICKHOUSE_COMPRESSOR || !defined(ENABLE_CLICKHOUSE_COMPRESSOR) {"compressor", mainEntryClickHouseCompressor}, +#endif +#if ENABLE_CLICKHOUSE_FORMAT || !defined(ENABLE_CLICKHOUSE_FORMAT) {"format", mainEntryClickHouseFormat}, #endif -#if ENABLE_CLICKHOUSE_COPIER +#if ENABLE_CLICKHOUSE_COPIER || !defined(ENABLE_CLICKHOUSE_COPIER) {"copier", mainEntryClickHouseClusterCopier}, #endif +#if ENABLE_CLICKHOUSE_OBFUSCATOR + {"obfuscator", mainEntryClickHouseObfuscator}, +#endif +#if ENABLE_CLICKHOUSE_ODBC_BRIDGE || !defined(ENABLE_CLICKHOUSE_ODBC_BRIDGE) + {"odbc-bridge", mainEntryClickHouseODBCBridge}, +#endif + #if USE_EMBEDDED_COMPILER {"clang", mainEntryClickHouseClang}, {"clang++", mainEntryClickHouseClang}, @@ -92,7 +123,7 @@ int printHelp(int, char **) for (auto & application : clickhouse_applications) std::cerr << "clickhouse " << application.first << " [args] " << std::endl; return -1; -}; +} bool isClickhouseApp(const std::string & app_suffix, std::vector & argv) diff --git a/dbms/programs/obfuscator/CMakeLists.txt b/dbms/programs/obfuscator/CMakeLists.txt new file mode 100644 index 00000000000..73c3f01e9cb --- /dev/null +++ b/dbms/programs/obfuscator/CMakeLists.txt @@ -0,0 +1,8 @@ +add_library (clickhouse-obfuscator-lib ${LINK_MODE} Obfuscator.cpp) +target_link_libraries (clickhouse-obfuscator-lib PRIVATE dbms ${Boost_PROGRAM_OPTIONS_LIBRARY}) + +if (CLICKHOUSE_SPLIT_BINARY) + add_executable (clickhouse-obfuscator clickhouse-obfuscator.cpp) + set_target_properties(clickhouse-obfuscator PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) + target_link_libraries (clickhouse-obfuscator PRIVATE clickhouse-obfuscator-lib) +endif () diff --git a/dbms/programs/obfuscator/Obfuscator.cpp b/dbms/programs/obfuscator/Obfuscator.cpp new file mode 100644 index 00000000000..f7eb7c5222d --- /dev/null +++ b/dbms/programs/obfuscator/Obfuscator.cpp @@ -0,0 +1,1098 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +static const char * documantation = R"( +Simple tool for table data obfuscation. + +It reads input table and produces output table, that retain some properties of input, but contains different data. +It allows to publish almost real production data for usage in benchmarks. + +It is designed to retain the following properties of data: +- cardinalities of values (number of distinct values) for every column and for every tuple of columns; +- conditional cardinalities: number of distinct values of one column under condition on value of another column; +- probability distributions of absolute value of integers; sign of signed integers; exponent and sign for floats; +- probability distributions of length of strings; +- probability of zero values of numbers; empty strings and arrays, NULLs; +- data compression ratio when compressed with LZ77 and entropy family of codecs; +- continuouty (magnitude of difference) of time values across table; continuouty of floating point values. +- date component of DateTime values; +- UTF-8 validity of string values; +- string values continue to look somewhat natural. + +Most of the properties above are viable for performance testing: +- reading data, filtering, aggregation and sorting will work at almost the same speed + as on original data due to saved cardinalities, magnitudes, compression ratios, etc. + +It works in deterministic fashion: you define a seed value and transform is totally determined by input data and by seed. +Some transforms are one to one and could be reversed, so you need to have large enough seed and keep it in secret. + +It use some cryptographic primitives to transform data, but from the cryptographic point of view, + it doesn't do anything properly and you should never consider the result as secure, unless you have other reasons for it. + +It may retain some data you don't want to publish. + +It always leave numbers 0, 1, -1 as is. Also it leaves dates, lengths of arrays and null flags exactly as in source data. +For example, you have a column IsMobile in your table with values 0 and 1. In transformed data, it will have the same value. +So, the user will be able to count exact ratio of mobile traffic. + +Another example, suppose you have some private data in your table, like user email and you don't want to publish any single email address. +If your table is large enough and contain multiple different emails and there is no email that have very high frequency than all others, + it will perfectly anonymize all data. But if you have small amount of different values in a column, it can possibly reproduce some of them. +And you should take care and look at exact algorithm, how this tool works, and probably fine tune some of it command line parameters. + +This tool works fine only with reasonable amount of data (at least 1000s of rows). +)"; + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; + extern const int CANNOT_SEEK_THROUGH_FILE; +} + + +/// Model is used to transform columns with source data to columns +/// with similar by structure and by probability distributions but anonymized data. +class IModel +{ +public: + /// Call train iteratively for each block to train a model. + virtual void train(const IColumn & column); + + /// Call finalize one time after training before generating. + virtual void finalize(); + + /// Call generate: pass source data column to obtain a column with anonymized data as a result. + virtual ColumnPtr generate(const IColumn & column); + + virtual ~IModel() {} +}; + +using ModelPtr = std::unique_ptr; + + +template +UInt64 hash(Ts... xs) +{ + SipHash hash; + (hash.update(xs), ...); + return hash.get64(); +} + + +UInt64 maskBits(UInt64 x, size_t num_bits) +{ + return x & ((1 << num_bits) - 1); +} + + +/// Apply Feistel network round to least significant num_bits part of x. +UInt64 feistelRound(UInt64 x, size_t num_bits, UInt64 seed, size_t round) +{ + size_t num_bits_left_half = num_bits / 2; + size_t num_bits_right_half = num_bits - num_bits_left_half; + + UInt64 left_half = maskBits(x >> num_bits_right_half, num_bits_left_half); + UInt64 right_half = maskBits(x, num_bits_right_half); + + UInt64 new_left_half = right_half; + UInt64 new_right_half = left_half ^ maskBits(hash(right_half, seed, round), num_bits_left_half); + + return (new_left_half << num_bits_left_half) ^ new_right_half; +} + + +/// Apply Feistel network with num_rounds to least significant num_bits part of x. +UInt64 feistelNetwork(UInt64 x, size_t num_bits, UInt64 seed, size_t num_rounds = 4) +{ + UInt64 bits = maskBits(x, num_bits); + for (size_t i = 0; i < num_rounds; ++i) + bits = feistelRound(bits, num_bits, seed, i); + return (x & ~((1 << num_bits) - 1)) ^ bits; +} + + +/// Pseudorandom permutation within set of numbers with the same log2(x). +UInt64 transform(UInt64 x, UInt64 seed) +{ + /// Keep 0 and 1 as is. + if (x == 0 || x == 1) + return x; + + /// Pseudorandom permutation of two elements. + if (x == 2 || x == 3) + return x ^ (seed & 1); + + size_t num_leading_zeros = __builtin_clzll(x); + + return feistelNetwork(x, 64 - num_leading_zeros - 1, seed); +} + + +class UnsignedIntegerModel : public IModel +{ +private: + const UInt64 seed; + +public: + UnsignedIntegerModel(UInt64 seed) : seed(seed) {} + + void train(const IColumn &) override {} + void finalize() override {} + + ColumnPtr generate(const IColumn & column) override + { + MutableColumnPtr res = column.cloneEmpty(); + + size_t size = column.size(); + res->reserve(size); + + for (size_t i = 0; i < size; ++i) + res->insert(transform(column.getUInt(i), seed)); + + return res; + } +}; + + +/// Keep sign and apply pseudorandom permutation after converting to unsigned as above. +Int64 transformSigned(Int64 x, UInt64 seed) +{ + if (x >= 0) + return transform(x, seed); + else + return -transform(-x, seed); /// It works Ok even for minimum signed number. +} + + +class SignedIntegerModel : public IModel +{ +private: + const UInt64 seed; + +public: + SignedIntegerModel(UInt64 seed) : seed(seed) {} + + void train(const IColumn &) override {} + void finalize() override {} + + ColumnPtr generate(const IColumn & column) override + { + MutableColumnPtr res = column.cloneEmpty(); + + size_t size = column.size(); + res->reserve(size); + + for (size_t i = 0; i < size; ++i) + res->insert(transformSigned(column.getInt(i), seed)); + + return res; + } +}; + + +/// Pseudorandom permutation of mantissa. +template +Float transformFloatMantissa(Float x, UInt64 seed) +{ + using UInt = std::conditional_t, UInt32, UInt64>; + constexpr size_t mantissa_num_bits = std::is_same_v ? 23 : 52; + + UInt x_uint = ext::bit_cast(x); + x_uint = feistelNetwork(x_uint, mantissa_num_bits, seed); + return ext::bit_cast(x_uint); +} + + +/// Transform difference from previous number by applying pseudorandom permutation to mantissa part of it. +/// It allows to retain some continuouty property of source data. +template +class FloatModel : public IModel +{ +private: + const UInt64 seed; + Float src_prev_value = 0; + Float res_prev_value = 0; + +public: + FloatModel(UInt64 seed) : seed(seed) {} + + void train(const IColumn &) override {} + void finalize() override {} + + ColumnPtr generate(const IColumn & column) override + { + const auto & src_data = static_cast &>(column).getData(); + size_t size = src_data.size(); + + auto res_column = ColumnVector::create(size); + auto & res_data = static_cast &>(*res_column).getData(); + + for (size_t i = 0; i < size; ++i) + { + res_data[i] = res_prev_value + transformFloatMantissa(src_data[i] - src_prev_value, seed); + src_prev_value = src_data[i]; + res_prev_value = res_data[i]; + } + + return res_column; + } +}; + + +/// Leave all data as is. For example, it is used for columns of type Date. +class IdentityModel : public IModel +{ +public: + void train(const IColumn &) override {} + void finalize() override {} + + ColumnPtr generate(const IColumn & column) override + { + return column.cloneResized(column.size()); + } +}; + + +/// Pseudorandom function, but keep word characters as word characters. +void transformFixedString(const UInt8 * src, UInt8 * dst, size_t size, UInt64 seed) +{ + { + SipHash hash; + hash.update(seed); + hash.update(reinterpret_cast(src), size); + seed = hash.get64(); + } + + UInt8 * pos = dst; + UInt8 * end = dst + size; + + size_t i = 0; + while (pos < end) + { + SipHash hash; + hash.update(seed); + hash.update(i); + + if (size >= 16) + { + char * dst = reinterpret_cast(std::min(pos, end - 16)); + hash.get128(dst); + } + else + { + char value[16]; + hash.get128(value); + memcpy(dst, value, end - dst); + } + + pos += 16; + ++i; + } + + for (size_t j = 0; j < size; ++j) + { + if (isWordCharASCII(src[j])) + { + static constexpr char word_chars[] = "_01234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + dst[j] = word_chars[dst[j] % (sizeof(word_chars) - 1)]; + } + } +} + + +class FixedStringModel : public IModel +{ +private: + const UInt64 seed; + +public: + FixedStringModel(UInt64 seed) : seed(seed) {} + + void train(const IColumn &) override {} + void finalize() override {} + + ColumnPtr generate(const IColumn & column) override + { + const ColumnFixedString & column_fixed_string = static_cast(column); + const size_t string_size = column_fixed_string.getN(); + + const auto & src_data = column_fixed_string.getChars(); + size_t size = column_fixed_string.size(); + + auto res_column = ColumnFixedString::create(string_size); + auto & res_data = res_column->getChars(); + + res_data.resize(src_data.size()); + + for (size_t i = 0; i < size; ++i) + transformFixedString(&src_data[i * string_size], &res_data[i * string_size], string_size, seed); + + return res_column; + } +}; + + +/// Leave date part as is and apply pseudorandom permutation to time difference with previous value within the same log2 class. +class DateTimeModel : public IModel +{ +private: + const UInt64 seed; + UInt32 src_prev_value = 0; + UInt32 res_prev_value = 0; + + const DateLUTImpl & date_lut; + +public: + DateTimeModel(UInt64 seed) : seed(seed), date_lut(DateLUT::instance()) {} + + void train(const IColumn &) override {} + void finalize() override {} + + ColumnPtr generate(const IColumn & column) override + { + const auto & src_data = static_cast &>(column).getData(); + size_t size = src_data.size(); + + auto res_column = ColumnVector::create(size); + auto & res_data = static_cast &>(*res_column).getData(); + + for (size_t i = 0; i < size; ++i) + { + UInt32 src_datetime = src_data[i]; + UInt32 src_date = date_lut.toDate(src_datetime); + + Int32 src_diff = src_datetime - src_prev_value; + Int32 res_diff = transformSigned(src_diff, seed); + + UInt32 new_datetime = res_prev_value + res_diff; + UInt32 new_time = new_datetime - date_lut.toDate(new_datetime); + res_data[i] = src_date + new_time; + + src_prev_value = src_datetime; + res_prev_value = res_data[i]; + } + + return res_column; + } +}; + + +struct MarkovModelParameters +{ + size_t order; + size_t frequency_cutoff; + size_t num_buckets_cutoff; + size_t frequency_add; + double frequency_desaturate; + size_t determinator_sliding_window_size; +}; + + +/** Actually it's not an order-N model, but a mix of order-{0..N} models. + * + * We calculate code point counts for every context of 0..N previous code points. + * Then throw off some context with low amount of statistics. + * + * When generating data, we try to find statistics for a context of maximum order. + * And if not found - use context of smaller order, up to 0. + */ +class MarkovModel +{ +private: + using CodePoint = UInt32; + using NGramHash = UInt32; + + struct Histogram + { + UInt64 total = 0; /// Not including count_end. + UInt64 count_end = 0; + using Buckets = boost::container::flat_map; + Buckets buckets; + + void add(CodePoint code) + { + ++total; + ++buckets[code]; + } + + void addEnd() + { + ++count_end; + } + + CodePoint sample(UInt64 random, double end_multiplier) const + { + UInt64 range = total + UInt64(count_end * end_multiplier); + if (range == 0) + return END; + + random %= range; + + UInt64 sum = 0; + for (const auto & elem : buckets) + { + sum += elem.second; + if (sum > random) + return elem.first; + } + + return END; + } + }; + + using Table = HashMap; + Table table; + + MarkovModelParameters params; + + std::vector code_points; + + /// Special code point to form context before beginning of string. + static constexpr CodePoint BEGIN = -1; + /// Special code point to indicate end of string. + static constexpr CodePoint END = -2; + + + NGramHash hashContext(const CodePoint * begin, const CodePoint * end) const + { + return CRC32Hash()(StringRef(reinterpret_cast(begin), (end - begin) * sizeof(CodePoint))); + } + + /// By the way, we don't have to use actual Unicode numbers. We use just arbitrary bijective mapping. + CodePoint readCodePoint(const char *& pos, const char * end) + { + size_t length = UTF8::seqLength(*pos); + + if (pos + length > end) + length = end - pos; + if (length > sizeof(CodePoint)) + length = sizeof(CodePoint); + + CodePoint res = 0; + memcpy(&res, pos, length); + pos += length; + return res; + } + + bool writeCodePoint(CodePoint code, char *& pos, char * end) + { + size_t length + = (code & 0xFF000000) ? 4 + : (code & 0xFFFF0000) ? 3 + : (code & 0xFFFFFF00) ? 2 + : 1; + + if (pos + length > end) + return false; + + memcpy(pos, &code, length); + pos += length; + return true; + } + +public: + MarkovModel(MarkovModelParameters params) + : params(std::move(params)), code_points(params.order, BEGIN) {} + + void consume(const char * data, size_t size) + { + /// First 'order' number of code points are pre-filled with BEGIN. + code_points.resize(params.order); + + const char * pos = data; + const char * end = data + size; + + while (true) + { + const bool inside = pos < end; + + CodePoint next_code_point {}; + + if (inside) + next_code_point = readCodePoint(pos, end); + + for (size_t context_size = 0; context_size < params.order; ++context_size) + { + NGramHash context_hash = hashContext(code_points.data() + code_points.size() - context_size, code_points.data() + code_points.size()); + + if (inside) + table[context_hash].add(next_code_point); + else /// if (context_size != 0 || order == 0) /// Don't allow to break string without context (except order-0 model). + table[context_hash].addEnd(); + } + + if (inside) + code_points.push_back(next_code_point); + else + break; + } + } + + + void finalize() + { + if (params.num_buckets_cutoff) + { + for (auto & elem : table) + { + Histogram & histogram = elem.second; + + if (histogram.buckets.size() < params.num_buckets_cutoff) + { + histogram.buckets.clear(); + histogram.total = 0; + } + } + } + + if (params.frequency_cutoff) + { + for (auto & elem : table) + { + Histogram & histogram = elem.second; + if (!histogram.total) + continue; + + if (histogram.total + histogram.count_end < params.frequency_cutoff) + { + histogram.buckets.clear(); + histogram.total = 0; + } + else + { + Histogram::Buckets new_buckets; + UInt64 erased_count = 0; + + for (const auto & bucket : histogram.buckets) + { + if (bucket.second >= params.frequency_cutoff) + new_buckets.emplace(bucket); + else + erased_count += bucket.second; + } + + histogram.buckets.swap(new_buckets); + histogram.total -= erased_count; + } + } + } + + if (params.frequency_add) + { + for (auto & elem : table) + { + Histogram & histogram = elem.second; + if (!histogram.total) + continue; + + for (auto & bucket : histogram.buckets) + bucket.second += params.frequency_add; + + histogram.count_end += params.frequency_add; + histogram.total += params.frequency_add * histogram.buckets.size(); + } + } + + if (params.frequency_desaturate) + { + for (auto & elem : table) + { + Histogram & histogram = elem.second; + if (!histogram.total) + continue; + + double average = histogram.total / histogram.buckets.size(); + + UInt64 new_total = 0; + for (auto & bucket : histogram.buckets) + { + bucket.second = bucket.second * (1.0 - params.frequency_desaturate) + average * params.frequency_desaturate; + new_total += bucket.second; + } + + histogram.total = new_total; + } + } + } + + + size_t generate(char * data, size_t desired_size, size_t buffer_size, + UInt64 seed, const char * determinator_data, size_t determinator_size) + { + code_points.resize(params.order); + + char * pos = data; + char * end = data + buffer_size; + + while (pos < end) + { + Table::iterator it = table.end(); + + size_t context_size = params.order; + while (true) + { + it = table.find(hashContext(code_points.data() + code_points.size() - context_size, code_points.data() + code_points.size())); + if (table.end() != it && it->second.total + it->second.count_end != 0) + break; + + if (context_size == 0) + break; + --context_size; + } + + if (table.end() == it) + throw Exception("Logical error in markov model", ErrorCodes::LOGICAL_ERROR); + + size_t offset_from_begin_of_string = pos - data; + size_t determinator_sliding_window_size = params.determinator_sliding_window_size; + if (determinator_sliding_window_size > determinator_size) + determinator_sliding_window_size = determinator_size; + + size_t determinator_sliding_window_overflow = offset_from_begin_of_string + determinator_sliding_window_size > determinator_size + ? offset_from_begin_of_string + determinator_sliding_window_size - determinator_size : 0; + + const char * determinator_sliding_window_begin = determinator_data + offset_from_begin_of_string - determinator_sliding_window_overflow; + + SipHash hash; + hash.update(seed); + hash.update(determinator_sliding_window_begin, determinator_sliding_window_size); + hash.update(determinator_sliding_window_overflow); + UInt64 determinator = hash.get64(); + + /// If string is greater than desired_size, increase probability of end. + double end_probability_multiplier = 0; + Int64 num_bytes_after_desired_size = (pos - data) - desired_size; + + if (num_bytes_after_desired_size > 0) + end_probability_multiplier = std::pow(1.25, num_bytes_after_desired_size); + + CodePoint code = it->second.sample(determinator, end_probability_multiplier); + + if (code == END) + break; + + if (num_bytes_after_desired_size > 0) + { + /// Heuristic: break at ASCII non-alnum code point. + /// This allows to be close to desired_size but not break natural looking words. + if (code < 128 && !isAlphaNumericASCII(code)) + break; + } + + if (!writeCodePoint(code, pos, end)) + break; + + code_points.push_back(code); + } + + return pos - data; + } +}; + + +/// Generate length of strings as above. +/// To generate content of strings, use +/// order-N Markov model on Unicode code points, +/// and to generate next code point use deterministic RNG +/// determined by hash of a sliding window (default 8 bytes) of source string. +/// This is intended to generate locally-similar strings from locally-similar sources. +class StringModel : public IModel +{ +private: + UInt64 seed; + MarkovModel markov_model; + +public: + StringModel(UInt64 seed, MarkovModelParameters params) : seed(seed), markov_model(std::move(params)) {} + + void train(const IColumn & column) override + { + const ColumnString & column_string = static_cast(column); + size_t size = column_string.size(); + + for (size_t i = 0; i < size; ++i) + { + StringRef string = column_string.getDataAt(i); + markov_model.consume(string.data, string.size); + } + } + + void finalize() override + { + markov_model.finalize(); + } + + ColumnPtr generate(const IColumn & column) override + { + const ColumnString & column_string = static_cast(column); + size_t size = column_string.size(); + + auto res_column = ColumnString::create(); + res_column->reserve(size); + + std::string new_string; + for (size_t i = 0; i < size; ++i) + { + StringRef src_string = column_string.getDataAt(i); + size_t desired_string_size = transform(src_string.size, seed); + new_string.resize(desired_string_size * 2); + + size_t actual_size = 0; + if (desired_string_size != 0) + actual_size = markov_model.generate(new_string.data(), desired_string_size, new_string.size(), seed, src_string.data, src_string.size); + + res_column->insertData(new_string.data(), actual_size); + } + + return res_column; + } +}; + + +class ArrayModel : public IModel +{ +private: + ModelPtr nested_model; + +public: + ArrayModel(ModelPtr nested_model) : nested_model(std::move(nested_model)) {} + + void train(const IColumn & column) override + { + const ColumnArray & column_array = static_cast(column); + const IColumn & nested_column = column_array.getData(); + + nested_model->train(nested_column); + } + + void finalize() override + { + nested_model->finalize(); + } + + ColumnPtr generate(const IColumn & column) override + { + const ColumnArray & column_array = static_cast(column); + const IColumn & nested_column = column_array.getData(); + + ColumnPtr new_nested_column = nested_model->generate(nested_column); + + return ColumnArray::create((*std::move(new_nested_column)).mutate(), (*std::move(column_array.getOffsetsPtr())).mutate()); + } +}; + + +class NullableModel : public IModel +{ +private: + ModelPtr nested_model; + +public: + NullableModel(ModelPtr nested_model) : nested_model(std::move(nested_model)) {} + + void train(const IColumn & column) override + { + const ColumnNullable & column_nullable = static_cast(column); + const IColumn & nested_column = column_nullable.getNestedColumn(); + + nested_model->train(nested_column); + } + + void finalize() override + { + nested_model->finalize(); + } + + ColumnPtr generate(const IColumn & column) override + { + const ColumnNullable & column_nullable = static_cast(column); + const IColumn & nested_column = column_nullable.getNestedColumn(); + + ColumnPtr new_nested_column = nested_model->generate(nested_column); + + return ColumnNullable::create((*std::move(new_nested_column)).mutate(), (*std::move(column_nullable.getNullMapColumnPtr())).mutate()); + } +}; + + +class ModelFactory +{ +public: + ModelPtr get(const IDataType & data_type, UInt64 seed, MarkovModelParameters markov_model_params) const + { + if (isInteger(data_type)) + { + if (isUnsignedInteger(data_type)) + return std::make_unique(seed); + else + return std::make_unique(seed); + } + + if (typeid_cast(&data_type)) + return std::make_unique>(seed); + + if (typeid_cast(&data_type)) + return std::make_unique>(seed); + + if (typeid_cast(&data_type)) + return std::make_unique(); + + if (typeid_cast(&data_type)) + return std::make_unique(seed); + + if (typeid_cast(&data_type)) + return std::make_unique(seed, markov_model_params); + + if (typeid_cast(&data_type)) + return std::make_unique(seed); + + if (auto type = typeid_cast(&data_type)) + return std::make_unique(get(*type->getNestedType(), seed, markov_model_params)); + + if (auto type = typeid_cast(&data_type)) + return std::make_unique(get(*type->getNestedType(), seed, markov_model_params)); + + throw Exception("Unsupported data type", ErrorCodes::NOT_IMPLEMENTED); + } +}; + + +class Obfuscator +{ +private: + std::vector models; + +public: + Obfuscator(const Block & header, UInt64 seed, MarkovModelParameters markov_model_params) + { + ModelFactory factory; + + size_t columns = header.columns(); + models.reserve(columns); + + for (size_t i = 0; i < columns; ++i) + models.emplace_back(factory.get(*header.getByPosition(i).type, hash(seed, i), markov_model_params)); + } + + void train(const Columns & columns) + { + size_t size = columns.size(); + for (size_t i = 0; i < size; ++i) + models[i]->train(*columns[i]); + } + + void finalize() + { + for (auto & model : models) + model->finalize(); + } + + Columns generate(const Columns & columns) + { + size_t size = columns.size(); + Columns res(size); + for (size_t i = 0; i < size; ++i) + res[i] = models[i]->generate(*columns[i]); + return res; + } +}; + +} + + +int mainEntryClickHouseObfuscator(int argc, char ** argv) +try +{ + using namespace DB; + namespace po = boost::program_options; + + po::options_description description("Options"); + description.add_options() + ("help", "produce help message") + ("structure,S", po::value(), "structure of the initial table (list of column and type names)") + ("input-format", po::value(), "input format of the initial table data") + ("output-format", po::value(), "default output format") + ("seed", po::value(), "seed (arbitrary string), must be random string with at least 10 bytes length") + ("limit", po::value(), "if specified - stop after generating that number of rows") + ("silent", po::value()->default_value(false), "don't print information messages to stderr") + ("order", po::value()->default_value(5), "order of markov model to generate strings") + ("frequency-cutoff", po::value()->default_value(5), "frequency cutoff for markov model: remove all buckets with count less than specified") + ("num-buckets-cutoff", po::value()->default_value(0), "cutoff for number of different possible continuations for a context: remove all histograms with less than specified number of buckets") + ("frequency-add", po::value()->default_value(0), "add a constant to every count to lower probability distribution skew") + ("frequency-desaturate", po::value()->default_value(0), "0..1 - move every frequency towards average to lower probability distribution skew") + ("determinator-sliding-window-size", po::value()->default_value(8), "size of a sliding window in a source string - its hash is used as a seed for RNG in markov model") + ; + + po::parsed_options parsed = po::command_line_parser(argc, argv).options(description).run(); + po::variables_map options; + po::store(parsed, options); + + if (options.count("help") + || !options.count("seed") + || !options.count("structure") + || !options.count("input-format") + || !options.count("output-format")) + { + std::cout << documantation << "\n" + << "\nUsage: " << argv[0] << " [options] < in > out\n" + << "\nInput must be seekable file (it will be read twice).\n" + << "\n" << description << "\n" + << "\nExample:\n " << argv[0] << " --seed \"$(head -c16 /dev/urandom | base64)\" --input-format TSV --output-format TSV --structure 'CounterID UInt32, URLDomain String, URL String, SearchPhrase String, Title String' < stats.tsv\n"; + return 0; + } + + UInt64 seed = sipHash64(options["seed"].as()); + + std::string structure = options["structure"].as(); + std::string input_format = options["input-format"].as(); + std::string output_format = options["output-format"].as(); + + std::optional limit; + if (options.count("limit")) + limit = options["limit"].as(); + + bool silent = options["silent"].as(); + + MarkovModelParameters markov_model_params; + + markov_model_params.order = options["order"].as(); + markov_model_params.frequency_cutoff = options["frequency-cutoff"].as(); + markov_model_params.num_buckets_cutoff = options["num-buckets-cutoff"].as(); + markov_model_params.frequency_add = options["frequency-add"].as(); + markov_model_params.frequency_desaturate = options["frequency-desaturate"].as(); + markov_model_params.determinator_sliding_window_size = options["determinator-sliding-window-size"].as(); + + // Create header block + std::vector structure_vals; + boost::split(structure_vals, structure, boost::algorithm::is_any_of(" ,"), boost::algorithm::token_compress_on); + + if (structure_vals.size() % 2 != 0) + throw Exception("Odd number of elements in section structure: must be a list of name type pairs", ErrorCodes::LOGICAL_ERROR); + + Block header; + const DataTypeFactory & data_type_factory = DataTypeFactory::instance(); + + for (size_t i = 0, size = structure_vals.size(); i < size; i += 2) + { + ColumnWithTypeAndName column; + column.name = structure_vals[i]; + column.type = data_type_factory.get(structure_vals[i + 1]); + column.column = column.type->createColumn(); + header.insert(std::move(column)); + } + + Context context = Context::createGlobal(); + + ReadBufferFromFileDescriptor file_in(STDIN_FILENO); + WriteBufferFromFileDescriptor file_out(STDOUT_FILENO); + + { + /// stdin must be seekable + auto res = lseek(file_in.getFD(), 0, SEEK_SET); + if (-1 == res) + throwFromErrno("Input must be seekable file (it will be read twice).", ErrorCodes::CANNOT_SEEK_THROUGH_FILE); + } + + Obfuscator obfuscator(header, seed, markov_model_params); + + size_t max_block_size = 8192; + + /// Train step + { + if (!silent) + std::cerr << "Training models\n"; + + BlockInputStreamPtr input = context.getInputFormat(input_format, file_in, header, max_block_size); + + UInt64 processed_rows = 0; + input->readPrefix(); + while (Block block = input->read()) + { + obfuscator.train(block.getColumns()); + processed_rows += block.rows(); + if (!silent) + std::cerr << "Processed " << processed_rows << " rows\n"; + } + input->readSuffix(); + } + + obfuscator.finalize(); + + /// Generation step + { + if (!silent) + std::cerr << "Generating data\n"; + + file_in.seek(0); + + BlockInputStreamPtr input = context.getInputFormat(input_format, file_in, header, max_block_size); + BlockOutputStreamPtr output = context.getOutputFormat(output_format, file_out, header); + + if (limit) + input = std::make_shared(input, *limit, 0); + + UInt64 processed_rows = 0; + input->readPrefix(); + output->writePrefix(); + while (Block block = input->read()) + { + Columns columns = obfuscator.generate(block.getColumns()); + output->write(header.cloneWithColumns(columns)); + processed_rows += block.rows(); + if (!silent) + std::cerr << "Processed " << processed_rows << " rows\n"; + } + output->writeSuffix(); + input->readSuffix(); + } + + return 0; +} +catch (...) +{ + std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; + auto code = DB::getCurrentExceptionCode(); + return code ? code : 1; +} diff --git a/dbms/programs/obfuscator/clickhouse-obfuscator.cpp b/dbms/programs/obfuscator/clickhouse-obfuscator.cpp new file mode 100644 index 00000000000..e57fa6d1b54 --- /dev/null +++ b/dbms/programs/obfuscator/clickhouse-obfuscator.cpp @@ -0,0 +1,3 @@ +int mainEntryClickHouseObfuscator(int argc, char ** argv); +int main(int argc_, char ** argv_) { return mainEntryClickHouseObfuscator(argc_, argv_); } + diff --git a/dbms/programs/odbc-bridge/CMakeLists.txt b/dbms/programs/odbc-bridge/CMakeLists.txt new file mode 100644 index 00000000000..a57c8c9c8cf --- /dev/null +++ b/dbms/programs/odbc-bridge/CMakeLists.txt @@ -0,0 +1,39 @@ +add_library (clickhouse-odbc-bridge-lib ${LINK_MODE} + PingHandler.cpp + MainHandler.cpp + ColumnInfoHandler.cpp + IdentifierQuoteHandler.cpp + HandlerFactory.cpp + ODBCBridge.cpp + getIdentifierQuote.cpp + validateODBCConnectionString.cpp +) + +target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE clickhouse_common_io daemon dbms) +target_include_directories (clickhouse-odbc-bridge-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include) + +if (USE_POCO_SQLODBC) + target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_SQLODBC_LIBRARY}) + target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_SQLODBC_INCLUDE_DIR}) +endif () +if (Poco_SQL_FOUND) + target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_SQL_LIBRARY}) +endif () + +if (USE_POCO_DATAODBC) + target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_DataODBC_LIBRARY}) + target_include_directories (clickhouse-odbc-bridge-lib SYSTEM PRIVATE ${ODBC_INCLUDE_DIRECTORIES} ${Poco_DataODBC_INCLUDE_DIR}) +endif() +if (Poco_Data_FOUND) + target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE ${Poco_Data_LIBRARY}) +endif () + + +if (ENABLE_TESTS) + add_subdirectory (tests) +endif () + +if (CLICKHOUSE_SPLIT_BINARY) + add_executable (clickhouse-odbc-bridge odbc-bridge.cpp) + target_link_libraries (clickhouse-odbc-bridge PRIVATE clickhouse-odbc-bridge-lib) +endif () diff --git a/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp b/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp new file mode 100644 index 00000000000..997ef9cf216 --- /dev/null +++ b/dbms/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -0,0 +1,172 @@ +#include "ColumnInfoHandler.h" +#include "getIdentifierQuote.h" +#if USE_POCO_SQLODBC || USE_POCO_DATAODBC + +#if USE_POCO_SQLODBC +#include // Y_IGNORE +#include // Y_IGNORE +#include // Y_IGNORE +#define POCO_SQL_ODBC_CLASS Poco::SQL::ODBC +#endif +#if USE_POCO_DATAODBC +#include +#include +#include +#define POCO_SQL_ODBC_CLASS Poco::Data::ODBC +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "validateODBCConnectionString.h" + +namespace DB +{ +namespace +{ + DataTypePtr getDataType(SQLSMALLINT type) + { + const auto & factory = DataTypeFactory::instance(); + + switch (type) + { + case SQL_INTEGER: + return factory.get("Int32"); + case SQL_SMALLINT: + return factory.get("Int16"); + case SQL_FLOAT: + return factory.get("Float32"); + case SQL_REAL: + return factory.get("Float32"); + case SQL_DOUBLE: + return factory.get("Float64"); + case SQL_DATETIME: + return factory.get("DateTime"); + case SQL_TYPE_TIMESTAMP: + return factory.get("DateTime"); + case SQL_TYPE_DATE: + return factory.get("Date"); + default: + return factory.get("String"); + } + } +} + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + +void ODBCColumnsInfoHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) +{ + Poco::Net::HTMLForm params(request, request.stream()); + LOG_TRACE(log, "Request URI: " + request.getURI()); + + auto process_error = [&response, this](const std::string & message) + { + response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); + if (!response.sent()) + response.send() << message << std::endl; + LOG_WARNING(log, message); + }; + + if (!params.has("table")) + { + process_error("No 'table' param in request URL"); + return; + } + if (!params.has("connection_string")) + { + process_error("No 'connection_string' in request URL"); + return; + } + std::string schema_name = ""; + std::string table_name = params.get("table"); + std::string connection_string = params.get("connection_string"); + if (params.has("schema")) + { + schema_name = params.get("schema"); + LOG_TRACE(log, "Will fetch info for table '" << schema_name + "." + table_name << "'"); + } + else + LOG_TRACE(log, "Will fetch info for table '" << table_name << "'"); + LOG_TRACE(log, "Got connection str '" << connection_string << "'"); + + try + { + POCO_SQL_ODBC_CLASS::SessionImpl session(validateODBCConnectionString(connection_string), DBMS_DEFAULT_CONNECT_TIMEOUT_SEC); + SQLHDBC hdbc = session.dbc().handle(); + + SQLHSTMT hstmt = nullptr; + + if (POCO_SQL_ODBC_CLASS::Utility::isError(SQLAllocStmt(hdbc, &hstmt))) + throw POCO_SQL_ODBC_CLASS::ODBCException("Could not allocate connection handle."); + + SCOPE_EXIT(SQLFreeStmt(hstmt, SQL_DROP)); + + /// TODO Why not do SQLColumns instead? + std::string name = schema_name.empty() ? table_name : schema_name + "." + table_name; + std::stringstream ss; + std::string input = "SELECT * FROM " + name + " WHERE 1 = 0"; + ParserQueryWithOutput parser; + ASTPtr select = parseQuery(parser, input.data(), input.data() + input.size(), "", 0); + + IAST::FormatSettings settings(ss, true); + settings.always_quote_identifiers = true; + + auto identifier_quote = getIdentifierQuote(hdbc); + if (identifier_quote.length() == 0) + settings.identifier_quoting_style = IdentifierQuotingStyle::None; + else if (identifier_quote[0] == '`') + settings.identifier_quoting_style = IdentifierQuotingStyle::Backticks; + else if (identifier_quote[0] == '"') + settings.identifier_quoting_style = IdentifierQuotingStyle::DoubleQuotes; + else + throw Exception("Can not map quote identifier '" + identifier_quote + "' to IdentifierQuotingStyle value", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + select->format(settings); + std::string query = ss.str(); + + LOG_TRACE(log, "Inferring structure with query '" << query << "'"); + + if (POCO_SQL_ODBC_CLASS::Utility::isError(POCO_SQL_ODBC_CLASS::SQLPrepare(hstmt, reinterpret_cast(query.data()), query.size()))) + throw POCO_SQL_ODBC_CLASS::DescriptorException(session.dbc()); + + if (POCO_SQL_ODBC_CLASS::Utility::isError(SQLExecute(hstmt))) + throw POCO_SQL_ODBC_CLASS::StatementException(hstmt); + + SQLSMALLINT cols = 0; + if (POCO_SQL_ODBC_CLASS::Utility::isError(SQLNumResultCols(hstmt, &cols))) + throw POCO_SQL_ODBC_CLASS::StatementException(hstmt); + + /// TODO cols not checked + + NamesAndTypesList columns; + for (SQLSMALLINT ncol = 1; ncol <= cols; ++ncol) + { + SQLSMALLINT type = 0; + /// TODO Why 301? + SQLCHAR column_name[301]; + /// TODO Result is not checked. + POCO_SQL_ODBC_CLASS::SQLDescribeCol(hstmt, ncol, column_name, sizeof(column_name), nullptr, &type, nullptr, nullptr, nullptr); + columns.emplace_back(reinterpret_cast(column_name), getDataType(type)); + } + + WriteBufferFromHTTPServerResponse out(request, response, keep_alive_timeout); + writeStringBinary(columns.toString(), out); + } + catch (...) + { + process_error("Error getting columns from ODBC '" + getCurrentExceptionMessage(false) + "'"); + tryLogCurrentException(log); + } +} +} +#endif diff --git a/dbms/programs/odbc-bridge/ColumnInfoHandler.h b/dbms/programs/odbc-bridge/ColumnInfoHandler.h new file mode 100644 index 00000000000..426cea15b34 --- /dev/null +++ b/dbms/programs/odbc-bridge/ColumnInfoHandler.h @@ -0,0 +1,30 @@ +#pragma once +#include +#include +#include +#include + +#if USE_POCO_SQLODBC || USE_POCO_DATAODBC +/** The structure of the table is taken from the query "SELECT * FROM table WHERE 1=0". + * TODO: It would be much better to utilize ODBC methods dedicated for columns description. + * If there is no such table, an exception is thrown. + */ +namespace DB +{ +class ODBCColumnsInfoHandler : public Poco::Net::HTTPRequestHandler +{ +public: + ODBCColumnsInfoHandler(size_t keep_alive_timeout_, std::shared_ptr context_) + : log(&Poco::Logger::get("ODBCColumnsInfoHandler")), keep_alive_timeout(keep_alive_timeout_), context(context_) + { + } + + void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override; + +private: + Poco::Logger * log; + size_t keep_alive_timeout; + std::shared_ptr context; +}; +} +#endif diff --git a/dbms/programs/odbc-bridge/HandlerFactory.cpp b/dbms/programs/odbc-bridge/HandlerFactory.cpp new file mode 100644 index 00000000000..a6422db268c --- /dev/null +++ b/dbms/programs/odbc-bridge/HandlerFactory.cpp @@ -0,0 +1,39 @@ +#include "HandlerFactory.h" +#include "PingHandler.h" +#include "ColumnInfoHandler.h" +#include +#include +#include +#include + +namespace DB +{ +Poco::Net::HTTPRequestHandler * HandlerFactory::createRequestHandler(const Poco::Net::HTTPServerRequest & request) +{ + Poco::URI uri{request.getURI()}; + LOG_TRACE(log, "Request URI: " + uri.toString()); + + if (uri.getPath() == "/ping" && request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET) + return new PingHandler(keep_alive_timeout); + + if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST) + { + + if (uri.getPath() == "/columns_info") +#if USE_POCO_SQLODBC || USE_POCO_DATAODBC + return new ODBCColumnsInfoHandler(keep_alive_timeout, context); +#else + return nullptr; +#endif + else if (uri.getPath() == "/identifier_quote") +#if USE_POCO_SQLODBC || USE_POCO_DATAODBC + return new IdentifierQuoteHandler(keep_alive_timeout, context); +#else + return nullptr; +#endif + else + return new ODBCHandler(pool_map, keep_alive_timeout, context); + } + return nullptr; +} +} diff --git a/dbms/programs/odbc-bridge/HandlerFactory.h b/dbms/programs/odbc-bridge/HandlerFactory.h new file mode 100644 index 00000000000..f1a6803f65b --- /dev/null +++ b/dbms/programs/odbc-bridge/HandlerFactory.h @@ -0,0 +1,39 @@ +#pragma once +#include +#include +#include +#include +#include "MainHandler.h" +#include "ColumnInfoHandler.h" +#include "IdentifierQuoteHandler.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" + #include +#pragma GCC diagnostic pop + + +namespace DB +{ +/** Factory for '/ping', '/', '/columns_info', '/identifier_quote' handlers. + * Also stores Session pools for ODBC connections + */ +class HandlerFactory : public Poco::Net::HTTPRequestHandlerFactory +{ +public: + HandlerFactory(const std::string & name_, size_t keep_alive_timeout_, std::shared_ptr context_) + : log(&Poco::Logger::get(name_)), name(name_), keep_alive_timeout(keep_alive_timeout_), context(context_) + { + pool_map = std::make_shared(); + } + + Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest & request) override; + +private: + Poco::Logger * log; + std::string name; + size_t keep_alive_timeout; + std::shared_ptr context; + std::shared_ptr pool_map; +}; +} diff --git a/dbms/programs/odbc-bridge/IdentifierQuoteHandler.cpp b/dbms/programs/odbc-bridge/IdentifierQuoteHandler.cpp new file mode 100644 index 00000000000..77fd32d80b4 --- /dev/null +++ b/dbms/programs/odbc-bridge/IdentifierQuoteHandler.cpp @@ -0,0 +1,69 @@ +#include "IdentifierQuoteHandler.h" +#if USE_POCO_SQLODBC || USE_POCO_DATAODBC + +#if USE_POCO_SQLODBC +#include // Y_IGNORE +#include // Y_IGNORE +#include // Y_IGNORE +#define POCO_SQL_ODBC_CLASS Poco::SQL::ODBC +#endif +#if USE_POCO_DATAODBC +#include +#include +#include +#define POCO_SQL_ODBC_CLASS Poco::Data::ODBC +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "getIdentifierQuote.h" +#include "validateODBCConnectionString.h" + +namespace DB +{ +void IdentifierQuoteHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) +{ + Poco::Net::HTMLForm params(request, request.stream()); + LOG_TRACE(log, "Request URI: " + request.getURI()); + + auto process_error = [&response, this](const std::string & message) + { + response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); + if (!response.sent()) + response.send() << message << std::endl; + LOG_WARNING(log, message); + }; + + if (!params.has("connection_string")) + { + process_error("No 'connection_string' in request URL"); + return; + } + + try + { + std::string connection_string = params.get("connection_string"); + POCO_SQL_ODBC_CLASS::SessionImpl session(validateODBCConnectionString(connection_string), DBMS_DEFAULT_CONNECT_TIMEOUT_SEC); + SQLHDBC hdbc = session.dbc().handle(); + + auto identifier = getIdentifierQuote(hdbc); + + WriteBufferFromHTTPServerResponse out(request, response, keep_alive_timeout); + writeStringBinary(identifier, out); + } + catch (...) + { + process_error("Error getting identifier quote style from ODBC '" + getCurrentExceptionMessage(false) + "'"); + tryLogCurrentException(log); + } +} +} +#endif diff --git a/dbms/programs/odbc-bridge/IdentifierQuoteHandler.h b/dbms/programs/odbc-bridge/IdentifierQuoteHandler.h new file mode 100644 index 00000000000..2d4cf0277be --- /dev/null +++ b/dbms/programs/odbc-bridge/IdentifierQuoteHandler.h @@ -0,0 +1,28 @@ +#pragma once +#include +#include +#include +#include + +#if USE_POCO_SQLODBC || USE_POCO_DATAODBC +/** This handler establish connection to database, and retrieve quote style identifier + */ +namespace DB +{ +class IdentifierQuoteHandler : public Poco::Net::HTTPRequestHandler +{ +public: + IdentifierQuoteHandler(size_t keep_alive_timeout_, std::shared_ptr context_) + : log(&Poco::Logger::get("IdentifierQuoteHandler")), keep_alive_timeout(keep_alive_timeout_), context(context_) + { + } + + void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override; + +private: + Poco::Logger * log; + size_t keep_alive_timeout; + std::shared_ptr context; +}; +} +#endif diff --git a/dbms/programs/odbc-bridge/MainHandler.cpp b/dbms/programs/odbc-bridge/MainHandler.cpp new file mode 100644 index 00000000000..a187225f4f0 --- /dev/null +++ b/dbms/programs/odbc-bridge/MainHandler.cpp @@ -0,0 +1,127 @@ +#include "MainHandler.h" + +#include "validateODBCConnectionString.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace +{ + std::unique_ptr parseColumns(std::string && column_string) + { + std::unique_ptr sample_block = std::make_unique(); + auto names_and_types = NamesAndTypesList::parse(column_string); + for (const NameAndTypePair & column_data : names_and_types) + sample_block->insert({column_data.type, column_data.name}); + return sample_block; + } +} + + +ODBCHandler::PoolPtr ODBCHandler::getPool(const std::string & connection_str) +{ + std::lock_guard lock(mutex); + if (!pool_map->count(connection_str)) + { + pool_map->emplace(connection_str, createAndCheckResizePocoSessionPool([connection_str] { + return std::make_shared("ODBC", validateODBCConnectionString(connection_str)); + })); + } + return pool_map->at(connection_str); +} + +void ODBCHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) +{ + Poco::Net::HTMLForm params(request, request.stream()); + LOG_TRACE(log, "Request URI: " + request.getURI()); + + auto process_error = [&response, this](const std::string & message) + { + response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); + if (!response.sent()) + response.send() << message << std::endl; + LOG_WARNING(log, message); + }; + + if (!params.has("query")) + { + process_error("No 'query' in request body"); + return; + } + + if (!params.has("columns")) + { + process_error("No 'columns' in request URL"); + return; + } + + if (!params.has("connection_string")) + { + process_error("No 'connection_string' in request URL"); + return; + } + + size_t max_block_size = DEFAULT_BLOCK_SIZE; + if (params.has("max_block_size")) + { + std::string max_block_size_str = params.get("max_block_size", ""); + if (max_block_size_str.empty()) + { + process_error("Empty max_block_size specified"); + return; + } + max_block_size = parse(max_block_size_str); + } + + std::string columns = params.get("columns"); + std::unique_ptr sample_block; + try + { + sample_block = parseColumns(std::move(columns)); + } + catch (const Exception & ex) + { + process_error("Invalid 'columns' parameter in request body '" + ex.message() + "'"); + LOG_WARNING(log, ex.getStackTrace().toString()); + return; + } + + std::string format = params.get("format", "RowBinary"); + std::string query = params.get("query"); + LOG_TRACE(log, "Query: " << query); + + std::string connection_string = params.get("connection_string"); + LOG_TRACE(log, "Connection string: '" << connection_string << "'"); + + WriteBufferFromHTTPServerResponse out(request, response, keep_alive_timeout); + try + { + BlockOutputStreamPtr writer = FormatFactory::instance().getOutput(format, out, *sample_block, *context); + auto pool = getPool(connection_string); + ODBCBlockInputStream inp(pool->get(), query, *sample_block, max_block_size); + copyData(inp, *writer); + } + catch (...) + { + auto message = getCurrentExceptionMessage(true); + response.setStatusAndReason( + Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); // can't call process_error, bacause of too soon response sending + writeStringBinary(message, out); + tryLogCurrentException(log); + } +} +} diff --git a/dbms/programs/odbc-bridge/MainHandler.h b/dbms/programs/odbc-bridge/MainHandler.h new file mode 100644 index 00000000000..ae139f393f8 --- /dev/null +++ b/dbms/programs/odbc-bridge/MainHandler.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-parameter" + #include +#pragma GCC diagnostic pop + +namespace DB +{ +/** Main handler for requests to ODBC driver + * requires connection_string and columns in request params + * and also query in request body + * response in RowBinary format + */ +class ODBCHandler : public Poco::Net::HTTPRequestHandler +{ +public: + using PoolPtr = std::shared_ptr; + using PoolMap = std::unordered_map; + + ODBCHandler(std::shared_ptr pool_map_, + size_t keep_alive_timeout_, + std::shared_ptr context_) + : log(&Poco::Logger::get("ODBCHandler")) + , pool_map(pool_map_) + , keep_alive_timeout(keep_alive_timeout_) + , context(context_) + { + } + + void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override; + +private: + Poco::Logger * log; + + std::shared_ptr pool_map; + size_t keep_alive_timeout; + std::shared_ptr context; + + static inline std::mutex mutex; + + PoolPtr getPool(const std::string & connection_str); +}; + +} diff --git a/dbms/programs/odbc-bridge/ODBCBridge.cpp b/dbms/programs/odbc-bridge/ODBCBridge.cpp new file mode 100644 index 00000000000..6bdd6395652 --- /dev/null +++ b/dbms/programs/odbc-bridge/ODBCBridge.cpp @@ -0,0 +1,205 @@ +#include "ODBCBridge.h" +#include "HandlerFactory.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ARGUMENT_OUT_OF_BOUND; +} + +namespace +{ + Poco::Net::SocketAddress makeSocketAddress(const std::string & host, UInt16 port, Poco::Logger * log) + { + Poco::Net::SocketAddress socket_address; + try + { + socket_address = Poco::Net::SocketAddress(host, port); + } + catch (const Poco::Net::DNSException & e) + { + const auto code = e.code(); + if (code == EAI_FAMILY +#if defined(EAI_ADDRFAMILY) + || code == EAI_ADDRFAMILY +#endif + ) + { + LOG_ERROR(log, + "Cannot resolve listen_host (" << host << "), error " << e.code() << ": " << e.message() + << ". " + "If it is an IPv6 address and your host has disabled IPv6, then consider to " + "specify IPv4 address to listen in element of configuration " + "file. Example: 0.0.0.0"); + } + + throw; + } + return socket_address; + } + + Poco::Net::SocketAddress socketBindListen(Poco::Net::ServerSocket & socket, const std::string & host, UInt16 port, Poco::Logger * log) + { + auto address = makeSocketAddress(host, port, log); +#if POCO_VERSION < 0x01080000 + socket.bind(address, /* reuseAddress = */ true); +#else + socket.bind(address, /* reuseAddress = */ true, /* reusePort = */ false); +#endif + + socket.listen(/* backlog = */ 64); + + return address; + } +} + +void ODBCBridge::handleHelp(const std::string &, const std::string &) +{ + Poco::Util::HelpFormatter helpFormatter(options()); + helpFormatter.setCommand(commandName()); + helpFormatter.setHeader("HTTP-proxy for odbc requests"); + helpFormatter.setUsage("--http-port "); + helpFormatter.format(std::cerr); + + stopOptionsProcessing(); +} + + +void ODBCBridge::defineOptions(Poco::Util::OptionSet & options) +{ + options.addOption(Poco::Util::Option("http-port", "", "port to listen").argument("http-port", true).binding("http-port")); + options.addOption( + Poco::Util::Option("listen-host", "", "hostname to listen, default localhost").argument("listen-host").binding("listen-host")); + options.addOption( + Poco::Util::Option("http-timeout", "", "http timout for socket, default 1800").argument("http-timeout").binding("http-timeout")); + + options.addOption(Poco::Util::Option("max-server-connections", "", "max connections to server, default 1024") + .argument("max-server-connections") + .binding("max-server-connections")); + options.addOption(Poco::Util::Option("keep-alive-timeout", "", "keepalive timeout, default 10") + .argument("keep-alive-timeout") + .binding("keep-alive-timeout")); + + options.addOption(Poco::Util::Option("log-level", "", "sets log level, default info").argument("log-level").binding("logger.level")); + + options.addOption( + Poco::Util::Option("log-path", "", "log path for all logs, default console").argument("log-path").binding("logger.log")); + + options.addOption(Poco::Util::Option("err-log-path", "", "err log path for all logs, default no") + .argument("err-log-path") + .binding("logger.errorlog")); + + using Me = std::decay_t; + options.addOption(Poco::Util::Option("help", "", "produce this help message") + .binding("help") + .callback(Poco::Util::OptionCallback(this, &Me::handleHelp))); + + ServerApplication::defineOptions(options); /// Don't need complex BaseDaemon's .xml config +} + +void ODBCBridge::initialize(Application & self) +{ + BaseDaemon::closeFDs(); + is_help = config().has("help"); + + if (is_help) + return; + + if (!config().has("logger.log")) + config().setBool("logger.console", true); + + config().setString("logger", "ODBCBridge"); + + buildLoggers(config()); + log = &logger(); + hostname = config().getString("listen-host", "localhost"); + port = config().getUInt("http-port"); + if (port > 0xFFFF) + throw Exception("Out of range 'http-port': " + std::to_string(port), ErrorCodes::ARGUMENT_OUT_OF_BOUND); + + http_timeout = config().getUInt("http-timeout", DEFAULT_HTTP_READ_BUFFER_TIMEOUT); + max_server_connections = config().getUInt("max-server-connections", 1024); + keep_alive_timeout = config().getUInt("keep-alive-timeout", 10); + + initializeTerminationAndSignalProcessing(); + + ServerApplication::initialize(self); +} + +void ODBCBridge::uninitialize() +{ + BaseDaemon::uninitialize(); +} + +int ODBCBridge::main(const std::vector & /*args*/) +{ + if (is_help) + return Application::EXIT_OK; + + LOG_INFO(log, "Starting up"); + Poco::Net::ServerSocket socket; + auto address = socketBindListen(socket, hostname, port, log); + socket.setReceiveTimeout(http_timeout); + socket.setSendTimeout(http_timeout); + Poco::ThreadPool server_pool(3, max_server_connections); + Poco::Net::HTTPServerParams::Ptr http_params = new Poco::Net::HTTPServerParams; + http_params->setTimeout(http_timeout); + http_params->setKeepAliveTimeout(keep_alive_timeout); + + context = std::make_shared(Context::createGlobal()); + context->setGlobalContext(*context); + + auto server = Poco::Net::HTTPServer( + new HandlerFactory("ODBCRequestHandlerFactory-factory", keep_alive_timeout, context), server_pool, socket, http_params); + server.start(); + + LOG_INFO(log, "Listening http://" + address.toString()); + + SCOPE_EXIT({ + LOG_DEBUG(log, "Received termination signal."); + LOG_DEBUG(log, "Waiting for current connections to close."); + server.stop(); + for (size_t count : ext::range(1, 6)) + { + if (server.currentConnections() == 0) + break; + LOG_DEBUG(log, "Waiting for " << server.currentConnections() << " connections, try " << count); + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + } + }); + + waitForTerminationRequest(); + return Application::EXIT_OK; +} +} + +int mainEntryClickHouseODBCBridge(int argc, char ** argv) +{ + DB::ODBCBridge app; + try + { + return app.run(argc, argv); + } + catch (...) + { + std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; + auto code = DB::getCurrentExceptionCode(); + return code ? code : 1; + } +} diff --git a/dbms/programs/odbc-bridge/ODBCBridge.h b/dbms/programs/odbc-bridge/ODBCBridge.h new file mode 100644 index 00000000000..4ae11ad7301 --- /dev/null +++ b/dbms/programs/odbc-bridge/ODBCBridge.h @@ -0,0 +1,41 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ +/** Class represents clickhouse-odbc-bridge server, which listen + * incoming HTTP POST and GET requests on specified port and host. + * Has two handlers '/' for all incoming POST requests to ODBC driver + * and /ping for GET request about service status + */ +class ODBCBridge : public BaseDaemon +{ +public: + void defineOptions(Poco::Util::OptionSet & options) override; + +protected: + void initialize(Application & self) override; + + void uninitialize() override; + + int main(const std::vector & args) override; + +private: + void handleHelp(const std::string &, const std::string &); + + bool is_help; + std::string hostname; + size_t port; + size_t http_timeout; + std::string log_level; + size_t max_server_connections; + size_t keep_alive_timeout; + + Poco::Logger * log; + + std::shared_ptr context; /// need for settings only +}; +} diff --git a/dbms/programs/odbc-bridge/PingHandler.cpp b/dbms/programs/odbc-bridge/PingHandler.cpp new file mode 100644 index 00000000000..b0313e46bf3 --- /dev/null +++ b/dbms/programs/odbc-bridge/PingHandler.cpp @@ -0,0 +1,22 @@ +#include "PingHandler.h" +#include +#include +#include +#include + +namespace DB +{ +void PingHandler::handleRequest(Poco::Net::HTTPServerRequest & /*request*/, Poco::Net::HTTPServerResponse & response) +{ + try + { + setResponseDefaultHeaders(response, keep_alive_timeout); + const char * data = "Ok.\n"; + response.sendBuffer(data, strlen(data)); + } + catch (...) + { + tryLogCurrentException("PingHandler"); + } +} +} diff --git a/dbms/programs/odbc-bridge/PingHandler.h b/dbms/programs/odbc-bridge/PingHandler.h new file mode 100644 index 00000000000..d8109a50bb6 --- /dev/null +++ b/dbms/programs/odbc-bridge/PingHandler.h @@ -0,0 +1,17 @@ +#pragma once +#include + +namespace DB +{ +/** Simple ping handler, answers "Ok." to GET request + */ +class PingHandler : public Poco::Net::HTTPRequestHandler +{ +public: + PingHandler(size_t keep_alive_timeout_) : keep_alive_timeout(keep_alive_timeout_) {} + void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override; + +private: + size_t keep_alive_timeout; +}; +} diff --git a/dbms/programs/odbc-bridge/README.md b/dbms/programs/odbc-bridge/README.md new file mode 100644 index 00000000000..70b413c9c1f --- /dev/null +++ b/dbms/programs/odbc-bridge/README.md @@ -0,0 +1,38 @@ +# clickhouse-odbc-bridge + +Simple HTTP-server which works like a proxy for ODBC driver. The main motivation +was possible segfaults or another faults in ODBC implementations, which can +crash whole clickhouse-server process. + +This tool works via HTTP, not via pipes, shared memory, or TCP because: +- It's simpler to implement +- It's simpler to debug +- jdbc-bridge can be implemented in the same way + +## Usage + +`clickhouse-server` use this tool inside odbc table function and StorageODBC. +However it can be used as standalone tool from command line with the following +parameters in POST-request URL: +- `connection_string` -- ODBC connection string. +- `columns` -- columns in ClickHouse NamesAndTypesList format, name in backticks, + type as string. Name and type are space separated, rows separated with + newline. +- `max_block_size` -- optional parameter, sets maximum size of single block. +Query is send in post body. Response is returned in RowBinary format. + +## Example: + +```bash +$ clickhouse-odbc-bridge --http-port 9018 --daemon + +$ curl -d "query=SELECT PageID, ImpID, AdType FROM Keys ORDER BY PageID, ImpID" --data-urlencode "connection_string=DSN=ClickHouse;DATABASE=stat" --data-urlencode "columns=columns format version: 1 +3 columns: +\`PageID\` String +\`ImpID\` String +\`AdType\` String +" "http://localhost:9018/" > result.txt + +$ cat result.txt +12246623837185725195925621517 +``` diff --git a/dbms/programs/odbc-bridge/getIdentifierQuote.cpp b/dbms/programs/odbc-bridge/getIdentifierQuote.cpp new file mode 100644 index 00000000000..055b211e99d --- /dev/null +++ b/dbms/programs/odbc-bridge/getIdentifierQuote.cpp @@ -0,0 +1,44 @@ +#include "getIdentifierQuote.h" +#if USE_POCO_SQLODBC || USE_POCO_DATAODBC + +#if USE_POCO_SQLODBC +#include // Y_IGNORE +#include // Y_IGNORE +#include // Y_IGNORE +#define POCO_SQL_ODBC_CLASS Poco::SQL::ODBC +#endif +#if USE_POCO_DATAODBC +#include +#include +#include +#define POCO_SQL_ODBC_CLASS Poco::Data::ODBC +#endif + + +namespace DB +{ +std::string getIdentifierQuote(SQLHDBC hdbc) +{ + std::string identifier; + + SQLSMALLINT t; + SQLRETURN r = POCO_SQL_ODBC_CLASS::SQLGetInfo(hdbc, SQL_IDENTIFIER_QUOTE_CHAR, nullptr, 0, &t); + + if (POCO_SQL_ODBC_CLASS::Utility::isError(r)) + throw POCO_SQL_ODBC_CLASS::ConnectionException(hdbc); + + if (t > 0) + { + // I have no idea, why to add '2' here, got from: contrib/poco/Data/ODBC/src/ODBCStatementImpl.cpp:60 (SQL_DRIVER_NAME) + identifier.resize(static_cast(t) + 2); + + if (POCO_SQL_ODBC_CLASS::Utility::isError(POCO_SQL_ODBC_CLASS::SQLGetInfo( + hdbc, SQL_IDENTIFIER_QUOTE_CHAR, &identifier[0], SQLSMALLINT((identifier.length() - 1) * sizeof(identifier[0])), &t))) + throw POCO_SQL_ODBC_CLASS::ConnectionException(hdbc); + + identifier.resize(static_cast(t)); + } + return identifier; +} +} +#endif diff --git a/dbms/programs/odbc-bridge/getIdentifierQuote.h b/dbms/programs/odbc-bridge/getIdentifierQuote.h new file mode 100644 index 00000000000..1a361dedea3 --- /dev/null +++ b/dbms/programs/odbc-bridge/getIdentifierQuote.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include +#include +#include + +#if USE_POCO_SQLODBC || USE_POCO_DATAODBC + +#if USE_POCO_SQLODBC +#include // Y_IGNORE +#endif +#if USE_POCO_DATAODBC +#include +#endif + +namespace DB +{ + +std::string getIdentifierQuote(SQLHDBC hdbc); +} +#endif \ No newline at end of file diff --git a/dbms/programs/odbc-bridge/odbc-bridge.cpp b/dbms/programs/odbc-bridge/odbc-bridge.cpp new file mode 100644 index 00000000000..af42eef8647 --- /dev/null +++ b/dbms/programs/odbc-bridge/odbc-bridge.cpp @@ -0,0 +1,2 @@ +int mainEntryClickHouseODBCBridge(int argc, char ** argv); +int main(int argc_, char ** argv_) { return mainEntryClickHouseODBCBridge(argc_, argv_); } diff --git a/dbms/programs/odbc-bridge/tests/CMakeLists.txt b/dbms/programs/odbc-bridge/tests/CMakeLists.txt new file mode 100644 index 00000000000..5211c39d111 --- /dev/null +++ b/dbms/programs/odbc-bridge/tests/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable (validate-odbc-connection-string validate-odbc-connection-string.cpp) +target_link_libraries (validate-odbc-connection-string PRIVATE clickhouse-odbc-bridge-lib clickhouse_common_io) diff --git a/dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.cpp b/dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.cpp new file mode 100644 index 00000000000..c4558811f77 --- /dev/null +++ b/dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.cpp @@ -0,0 +1,24 @@ +#include +#include +#include "../validateODBCConnectionString.h" + + +using namespace DB; + +int main(int argc, char ** argv) +try +{ + if (argc < 2) + { + std::cerr << "Usage: validate-odbc-connection-string 'ConnectionString'\n"; + return 1; + } + + std::cout << validateODBCConnectionString(argv[1]) << '\n'; + return 0; +} +catch (...) +{ + std::cerr << getCurrentExceptionMessage(false) << "\n"; + return 2; +} diff --git a/dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.reference b/dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.reference new file mode 100644 index 00000000000..2f1a0eb5bcd --- /dev/null +++ b/dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.reference @@ -0,0 +1,39 @@ +Code: 404, e.displayText() = DB::Exception: ODBC connection string cannot be empty, e.what() = DB::Exception +Code: 404, e.displayText() = DB::Exception: ODBC connection string parameter doesn't have value, e.what() = DB::Exception +Code: 404, e.displayText() = DB::Exception: DSN parameter is mandatory for ODBC connection string, e.what() = DB::Exception +Code: 404, e.displayText() = DB::Exception: ODBC connection string parameter doesn't have value, e.what() = DB::Exception +Code: 404, e.displayText() = DB::Exception: DSN parameter is mandatory for ODBC connection string, e.what() = DB::Exception +Code: 404, e.displayText() = DB::Exception: ODBC connection string parameter value is unescaped and contains illegal character, e.what() = DB::Exception +Code: 404, e.displayText() = DB::Exception: DSN parameter is mandatory for ODBC connection string, e.what() = DB::Exception +DSN={hello};ABC={de[f}; +DSN={hello};ABC={de}}f}; +Code: 404, e.displayText() = DB::Exception: ODBC connection string parameter value is unescaped and contains illegal character, e.what() = DB::Exception +Code: 404, e.displayText() = DB::Exception: ODBC connection string parameter is escaped but there is no closing curly brace, e.what() = DB::Exception +Code: 404, e.displayText() = DB::Exception: Unexpected character found after parameter value in ODBC connection string, e.what() = DB::Exception +Code: 404, e.displayText() = DB::Exception: Unexpected character found after parameter value in ODBC connection string, e.what() = DB::Exception +Code: 404, e.displayText() = DB::Exception: Unexpected character found after parameter value in ODBC connection string, e.what() = DB::Exception +DSN={hello};ABC={de}}f}; +Code: 404, e.displayText() = DB::Exception: ODBC connection string parameter is escaped but there is no closing curly brace, e.what() = DB::Exception +Code: 404, e.displayText() = DB::Exception: ODBC connection string parameter is escaped but there is no closing curly brace, e.what() = DB::Exception +DSN={hello};ABC={ }; +DSN={hello};ABC={ }; +Code: 404, e.displayText() = DB::Exception: Unexpected character found after parameter value in ODBC connection string, e.what() = DB::Exception +DSN={hello world};ABC={ }; +Code: 404, e.displayText() = DB::Exception: Unexpected character found after parameter value in ODBC connection string, e.what() = DB::Exception +Code: 404, e.displayText() = DB::Exception: ODBC connection string parameter name doesn't begin with valid identifier character, e.what() = DB::Exception +Code: 404, e.displayText() = DB::Exception: ODBC connection string parameter name doesn't begin with valid identifier character, e.what() = DB::Exception +DSN={hello world};ABC={ };_={}; +DSN={hello world};ABC={ };_={}; +DSN={hello world};ABC={ };_={}; +DSN={hello world};ABC={ };_={}}}; +DSN={hello world};ABC={ };_={...................................................................}; +DSN={hello world};ABC={ };_={....................................................................................}; +Code: 404, e.displayText() = DB::Exception: ODBC connection string has too long keyword or value, e.what() = DB::Exception +Code: 404, e.displayText() = DB::Exception: ODBC connection string has forbidden parameter, e.what() = DB::Exception +Code: 404, e.displayText() = DB::Exception: ODBC connection string has forbidden parameter, e.what() = DB::Exception +Code: 404, e.displayText() = DB::Exception: ODBC connection string has forbidden parameter, e.what() = DB::Exception +Code: 404, e.displayText() = DB::Exception: Duplicate parameter found in ODBC connection string, e.what() = DB::Exception +Code: 404, e.displayText() = DB::Exception: ODBC connection string parameter name doesn't begin with valid identifier character, e.what() = DB::Exception +DSN={myconnection}; +DSN={myconnection};DATABASE={my_db};HOST={127.0.0.1};PORT={5432};PWD={password};UID={username}; +DSN={MSSQL};PWD={test};UID={test}; diff --git a/dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.sh b/dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.sh new file mode 100755 index 00000000000..39789a8ab44 --- /dev/null +++ b/dbms/programs/odbc-bridge/tests/validate-odbc-connection-string.sh @@ -0,0 +1,41 @@ +#!/bin/sh + +./validate-odbc-connection-string '' 2>&1 +./validate-odbc-connection-string 'abc' 2>&1 +./validate-odbc-connection-string 'abc=' 2>&1 +./validate-odbc-connection-string 'ab"c=' 2>&1 +./validate-odbc-connection-string 'abc=def' 2>&1 +./validate-odbc-connection-string 'abc=de[f' 2>&1 +./validate-odbc-connection-string 'abc={de[f}' 2>&1 +./validate-odbc-connection-string 'abc={de[f};dsn=hello' 2>&1 +./validate-odbc-connection-string 'abc={de}}f};dsn=hello' 2>&1 +./validate-odbc-connection-string 'abc=de}}f};dsn=hello' 2>&1 +./validate-odbc-connection-string 'abc={de}}f;dsn=hello' 2>&1 +./validate-odbc-connection-string 'abc={de}f;dsn=hello' 2>&1 +./validate-odbc-connection-string 'abc={de}f;dsn=hello' 2>&1 +./validate-odbc-connection-string 'abc={de}f};dsn=hello' 2>&1 +./validate-odbc-connection-string 'abc={de}}f};dsn=hello' 2>&1 +./validate-odbc-connection-string 'abc={de}}f;dsn=hello' 2>&1 +./validate-odbc-connection-string 'abc={de}} ;dsn=hello' 2>&1 +./validate-odbc-connection-string 'abc={ } ;dsn=hello' 2>&1 +./validate-odbc-connection-string 'abc={ } ; dsn=hello ' 2>&1 +./validate-odbc-connection-string 'abc={ } ; dsn=hello world ' 2>&1 +./validate-odbc-connection-string 'abc={ } ; dsn = {hello world} ' 2>&1 +./validate-odbc-connection-string 'abc={ } ; dsn = {hello world} ...' 2>&1 +./validate-odbc-connection-string 'abc={ } ; dsn = {hello world} ;...' 2>&1 +./validate-odbc-connection-string 'abc={ } ; dsn = {hello world} ;=' 2>&1 +./validate-odbc-connection-string 'abc={ } ; dsn = {hello world} ;_=' 2>&1 +./validate-odbc-connection-string 'abc={ } ; dsn = {hello world} ;_= ' 2>&1 +./validate-odbc-connection-string 'abc={ } ; dsn = {hello world} ;_= {}' 2>&1 +./validate-odbc-connection-string 'abc={ } ; dsn = {hello world} ;_= {}}}' 2>&1 +./validate-odbc-connection-string 'abc={ } ; dsn = {hello world} ;_= {...................................................................}' 2>&1 +./validate-odbc-connection-string 'abc={ } ; dsn = {hello world} ;_= {....................................................................................}' 2>&1 +./validate-odbc-connection-string 'abc={ } ; dsn = {hello world} ;_= {.....................................................................................................}' 2>&1 +./validate-odbc-connection-string 'abc={ } ; dsn = {hello world} ;_= {...}; FILEDSN=x' 2>&1 +./validate-odbc-connection-string 'abc={ } ; dsn = {hello world} ;_= {...}; FileDsn = x' 2>&1 +./validate-odbc-connection-string 'abc={ } ; dsn = {hello world} ;_= {...}; Driver=x' 2>&1 +./validate-odbc-connection-string 'abc={}; abc=def' 2>&1 +./validate-odbc-connection-string 'abc={};;' 2>&1 +./validate-odbc-connection-string 'DSN=myconnection' 2>&1 +./validate-odbc-connection-string 'DSN=myconnection;UID=username;PWD=password;HOST=127.0.0.1;PORT=5432;DATABASE=my_db' 2>&1 +./validate-odbc-connection-string 'DSN=MSSQL;UID=test;PWD=test' 2>&1 diff --git a/dbms/programs/odbc-bridge/validateODBCConnectionString.cpp b/dbms/programs/odbc-bridge/validateODBCConnectionString.cpp new file mode 100644 index 00000000000..a64d7bb0b7c --- /dev/null +++ b/dbms/programs/odbc-bridge/validateODBCConnectionString.cpp @@ -0,0 +1,241 @@ +#include +#include +#include +#include +#include +#include +#include +#include "validateODBCConnectionString.h" + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int BAD_ODBC_CONNECTION_STRING; +} + + +std::string validateODBCConnectionString(const std::string & connection_string) +{ + /// Connection string is a list of name, value pairs. + /// name and value are separated by '='. + /// names are case insensitive. + /// name=value pairs are sepated by ';'. + /// ASCII whitespace characters are skipped before and after delimiters. + /// value may be optionally enclosed by {} + /// in enclosed value, } is escaped as }}. + /// + /// Example: PWD={a}}b} means that password is a}b + /// + /// https://docs.microsoft.com/en-us/sql/odbc/reference/syntax/sqldriverconnect-function?view=sql-server-2017#comments + + /// unixODBC has fixed size buffers on stack and has buffer overflow bugs. + /// We will limit string sizes to small values. + + static constexpr size_t MAX_ELEMENT_SIZE = 100; + static constexpr size_t MAX_CONNECTION_STRING_SIZE = 1000; + + if (connection_string.empty()) + throw Exception("ODBC connection string cannot be empty", ErrorCodes::BAD_ODBC_CONNECTION_STRING); + + if (connection_string.size() >= MAX_CONNECTION_STRING_SIZE) + throw Exception("ODBC connection string is too long", ErrorCodes::BAD_ODBC_CONNECTION_STRING); + + const char * pos = connection_string.data(); + const char * end = pos + connection_string.size(); + + auto skip_whitespaces = [&] + { + while (pos < end && isWhitespaceASCII(*pos)) + { + if (*pos != ' ') + throw Exception("ODBC connection string parameter contains unusual whitespace character", ErrorCodes::BAD_ODBC_CONNECTION_STRING); + ++pos; + } + }; + + auto read_name = [&] + { + const char * begin = pos; + + if (pos < end && isValidIdentifierBegin(*pos)) + ++pos; + else + throw Exception("ODBC connection string parameter name doesn't begin with valid identifier character", ErrorCodes::BAD_ODBC_CONNECTION_STRING); + + while (pos < end && isWordCharASCII(*pos)) + ++pos; + + return std::string(begin, pos); + }; + + auto read_plain_value = [&] + { + const char * begin = pos; + + while (pos < end && *pos != ';' && !isWhitespaceASCII(*pos)) + { + signed char c = *pos; + if (c < 32 || strchr("[]{}(),;?*=!@'\"", c) != nullptr) + throw Exception("ODBC connection string parameter value is unescaped and contains illegal character", ErrorCodes::BAD_ODBC_CONNECTION_STRING); + ++pos; + } + + return std::string(begin, pos); + }; + + auto read_escaped_value = [&] + { + std::string res; + + if (pos < end && *pos == '{') + ++pos; + else + throw Exception("ODBC connection string parameter value doesn't begin with opening curly brace", ErrorCodes::BAD_ODBC_CONNECTION_STRING); + + while (pos < end) + { + if (*pos == '}') + { + ++pos; + if (pos >= end || *pos != '}') + return res; + } + + if (*pos == 0) + throw Exception("ODBC connection string parameter value contains ASCII NUL character", ErrorCodes::BAD_ODBC_CONNECTION_STRING); + + res += *pos; + ++pos; + } + + throw Exception("ODBC connection string parameter is escaped but there is no closing curly brace", ErrorCodes::BAD_ODBC_CONNECTION_STRING); + }; + + auto read_value = [&] + { + if (pos >= end) + return std::string{}; + + if (*pos == '{') + return read_escaped_value(); + else + return read_plain_value(); + }; + + std::map parameters; + + while (pos < end) + { + skip_whitespaces(); + std::string name = read_name(); + skip_whitespaces(); + + Poco::toUpperInPlace(name); + if (name == "FILEDSN" || name == "SAVEFILE" || name == "DRIVER") + throw Exception("ODBC connection string has forbidden parameter", ErrorCodes::BAD_ODBC_CONNECTION_STRING); + + if (pos >= end) + throw Exception("ODBC connection string parameter doesn't have value", ErrorCodes::BAD_ODBC_CONNECTION_STRING); + + if (*pos == '=') + ++pos; + else + throw Exception("ODBC connection string parameter doesn't have value", ErrorCodes::BAD_ODBC_CONNECTION_STRING); + + skip_whitespaces(); + std::string value = read_value(); + skip_whitespaces(); + + if (name.size() > MAX_ELEMENT_SIZE || value.size() > MAX_ELEMENT_SIZE) + throw Exception("ODBC connection string has too long keyword or value", ErrorCodes::BAD_ODBC_CONNECTION_STRING); + + if (!parameters.emplace(name, value).second) + throw Exception("Duplicate parameter found in ODBC connection string", ErrorCodes::BAD_ODBC_CONNECTION_STRING); + + if (pos >= end) + break; + + if (*pos == ';') + ++pos; + else + throw Exception("Unexpected character found after parameter value in ODBC connection string", ErrorCodes::BAD_ODBC_CONNECTION_STRING); + } + + /// Reconstruct the connection string. + + auto it = parameters.find("DSN"); + + if (parameters.end() == it) + throw Exception("DSN parameter is mandatory for ODBC connection string", ErrorCodes::BAD_ODBC_CONNECTION_STRING); + + std::string dsn = it->second; + + if (dsn.empty()) + throw Exception("DSN parameter cannot be empty in ODBC connection string", ErrorCodes::BAD_ODBC_CONNECTION_STRING); + + parameters.erase(it); + + std::string reconstructed_connection_string; + + auto write_plain_value = [&](const std::string & value) + { + reconstructed_connection_string += value; + }; + + auto write_escaped_value = [&](const std::string & value) + { + reconstructed_connection_string += '{'; + + const char * value_pos = value.data(); + const char * value_end = value_pos + value.size(); + while (true) + { + const char * next_pos = find_first_symbols<'}'>(value_pos, value_end); + + if (next_pos == value_end) + { + reconstructed_connection_string.append(value_pos, next_pos - value_pos); + break; + } + else + { + reconstructed_connection_string.append(value_pos, next_pos - value_pos); + reconstructed_connection_string.append("}}"); + value_pos = next_pos + 1; + } + } + + reconstructed_connection_string += '}'; + }; + + auto write_value = [&](const std::string & value) + { + if (std::all_of(value.begin(), value.end(), isWordCharASCII)) + write_plain_value(value); + else + write_escaped_value(value); + }; + + auto write_element = [&](const std::string & name, const std::string & value) + { + reconstructed_connection_string.append(name); + reconstructed_connection_string += '='; + write_value(value); + reconstructed_connection_string += ';'; + }; + + /// Place DSN first because that's more safe. + write_element("DSN", dsn); + for (const auto & elem : parameters) + write_element(elem.first, elem.second); + + if (reconstructed_connection_string.size() >= MAX_CONNECTION_STRING_SIZE) + throw Exception("ODBC connection string is too long", ErrorCodes::BAD_ODBC_CONNECTION_STRING); + + return reconstructed_connection_string; +} + +} diff --git a/dbms/programs/odbc-bridge/validateODBCConnectionString.h b/dbms/programs/odbc-bridge/validateODBCConnectionString.h new file mode 100644 index 00000000000..8d2a23ca877 --- /dev/null +++ b/dbms/programs/odbc-bridge/validateODBCConnectionString.h @@ -0,0 +1,21 @@ +#pragma once + +#include + + +namespace DB +{ + +/** Passing arbitrary connection string to ODBC Driver Manager is insecure, for the following reasons: + * 1. Driver Manager like unixODBC has multiple bugs like buffer overflow. + * 2. Driver Manager can interpret some parameters as a path to library for dlopen or a file to read, + * thus allows arbitrary remote code execution. + * + * This function will throw exception if connection string has insecure parameters. + * It may also modify connection string to harden it. + * + * Note that it is intended for ANSI (not multibyte) variant of connection string. + */ +std::string validateODBCConnectionString(const std::string & connection_string); + +} diff --git a/dbms/programs/performance-test/CMakeLists.txt b/dbms/programs/performance-test/CMakeLists.txt new file mode 100644 index 00000000000..f1a08172009 --- /dev/null +++ b/dbms/programs/performance-test/CMakeLists.txt @@ -0,0 +1,8 @@ +add_library (clickhouse-performance-test-lib ${LINK_MODE} PerformanceTest.cpp) +target_link_libraries (clickhouse-performance-test-lib PRIVATE dbms clickhouse_common_io clickhouse_common_config ${Boost_PROGRAM_OPTIONS_LIBRARY}) +target_include_directories (clickhouse-performance-test-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR}) + +if (CLICKHOUSE_SPLIT_BINARY) + add_executable (clickhouse-performance-test clickhouse-performance-test.cpp) + target_link_libraries (clickhouse-performance-test PRIVATE clickhouse-performance-test-lib) +endif () diff --git a/dbms/src/Server/PerformanceTest.cpp b/dbms/programs/performance-test/PerformanceTest.cpp similarity index 91% rename from dbms/src/Server/PerformanceTest.cpp rename to dbms/programs/performance-test/PerformanceTest.cpp index 7c9d4436f3c..6a06283a6b8 100644 --- a/dbms/src/Server/PerformanceTest.cpp +++ b/dbms/programs/performance-test/PerformanceTest.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -29,7 +30,14 @@ #include #include #include -#include "InterruptListener.h" +#include +#include +#include + +#ifndef __clang__ +#pragma GCC optimize("-fno-var-tracking-assignments") +#endif + /** Tests launcher for ClickHouse. * The tool walks through given or default folder in order to find files with @@ -43,10 +51,10 @@ namespace DB { namespace ErrorCodes { - extern const int POCO_EXCEPTION; - extern const int STD_EXCEPTION; - extern const int UNKNOWN_EXCEPTION; extern const int NOT_IMPLEMENTED; + extern const int LOGICAL_ERROR; + extern const int BAD_ARGUMENTS; + extern const int FILE_DOESNT_EXIST; } static String pad(size_t padding) @@ -63,7 +71,7 @@ private: size_t padding; public: - explicit JSONString(size_t padding_ = 1) : padding(padding_){}; + explicit JSONString(size_t padding_ = 1) : padding(padding_) {} void set(const String key, String value, bool wrap = true) { @@ -150,7 +158,7 @@ struct StopConditionsSet else if (key == "average_speed_not_changing_for_ms") average_speed_not_changing_for_ms.value = stop_conditions_view->getUInt64(key); else - throw DB::Exception("Met unkown stop condition: " + key); + throw DB::Exception("Met unkown stop condition: " + key, DB::ErrorCodes::LOGICAL_ERROR); ++initialized_count; } @@ -231,13 +239,13 @@ public: conditions_any_of.report(value, conditions_any_of.CONDITION); \ } - DEFINE_REPORT_FUNC(reportTotalTime, total_time_ms); - DEFINE_REPORT_FUNC(reportRowsRead, rows_read); - DEFINE_REPORT_FUNC(reportBytesReadUncompressed, bytes_read_uncompressed); - DEFINE_REPORT_FUNC(reportIterations, iterations); - DEFINE_REPORT_FUNC(reportMinTimeNotChangingFor, min_time_not_changing_for_ms); - DEFINE_REPORT_FUNC(reportMaxSpeedNotChangingFor, max_speed_not_changing_for_ms); - DEFINE_REPORT_FUNC(reportAverageSpeedNotChangingFor, average_speed_not_changing_for_ms); + DEFINE_REPORT_FUNC(reportTotalTime, total_time_ms) + DEFINE_REPORT_FUNC(reportRowsRead, rows_read) + DEFINE_REPORT_FUNC(reportBytesReadUncompressed, bytes_read_uncompressed) + DEFINE_REPORT_FUNC(reportIterations, iterations) + DEFINE_REPORT_FUNC(reportMinTimeNotChangingFor, min_time_not_changing_for_ms) + DEFINE_REPORT_FUNC(reportMaxSpeedNotChangingFor, max_speed_not_changing_for_ms) + DEFINE_REPORT_FUNC(reportAverageSpeedNotChangingFor, average_speed_not_changing_for_ms) #undef REPORT @@ -481,17 +489,18 @@ struct Stats double Stats::avg_rows_speed_precision = 0.001; double Stats::avg_bytes_speed_precision = 0.001; -class PerformanceTest +class PerformanceTest : public Poco::Util::Application { public: using Strings = std::vector; PerformanceTest(const String & host_, const UInt16 port_, + const bool secure_, const String & default_database_, const String & user_, const String & password_, - const bool & lite_output_, + const bool lite_output_, const String & profiles_file_, Strings && input_files_, Strings && tests_tags_, @@ -501,7 +510,7 @@ public: Strings && tests_names_regexp_, Strings && skip_names_regexp_, const ConnectionTimeouts & timeouts) - : connection(host_, port_, default_database_, user_, password_, timeouts), + : connection(host_, port_, default_database_, user_, password_, timeouts, "performance-test", Protocol::Compression::Enable, secure_ ? Protocol::Secure::Enable : Protocol::Secure::Disable), gotSIGINT(false), lite_output(lite_output_), profiles_file(profiles_file_), @@ -515,20 +524,35 @@ public: { if (input_files.size() < 1) { - throw DB::Exception("No tests were specified", 0); + throw DB::Exception("No tests were specified", DB::ErrorCodes::BAD_ARGUMENTS); } + } + void initialize(Poco::Util::Application & self [[maybe_unused]]) + { + std::string home_path; + const char * home_path_cstr = getenv("HOME"); + if (home_path_cstr) + home_path = home_path_cstr; + configReadClient(Poco::Util::Application::instance().config(), home_path); + } + + int main(const std::vector < std::string > & /* args */) + { std::string name; UInt64 version_major; UInt64 version_minor; + UInt64 version_patch; UInt64 version_revision; - connection.getServerVersion(name, version_major, version_minor, version_revision); + connection.getServerVersion(name, version_major, version_minor, version_patch, version_revision); std::stringstream ss; - ss << version_major << "." << version_minor << "." << version_revision; + ss << version_major << "." << version_minor << "." << version_patch; server_version = ss.str(); processTestsConfigurations(input_files); + + return 0; } private: @@ -687,7 +711,7 @@ private: size_t ram_size_needed = config->getUInt64("preconditions.ram_size"); size_t actual_ram = getMemoryAmount(); if (!actual_ram) - throw DB::Exception("ram_size precondition not available on this platform", ErrorCodes::NOT_IMPLEMENTED); + throw DB::Exception("ram_size precondition not available on this platform", DB::ErrorCodes::NOT_IMPLEMENTED); if (ram_size_needed > actual_ram) { @@ -861,12 +885,12 @@ private: if (!test_config->has("query") && !test_config->has("query_file")) { - throw DB::Exception("Missing query fields in test's config: " + test_name); + throw DB::Exception("Missing query fields in test's config: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS); } if (test_config->has("query") && test_config->has("query_file")) { - throw DB::Exception("Found both query and query_file fields. Choose only one"); + throw DB::Exception("Found both query and query_file fields. Choose only one", DB::ErrorCodes::BAD_ARGUMENTS); } if (test_config->has("query")) @@ -878,7 +902,7 @@ private: { const String filename = test_config->getString("query_file"); if (filename.empty()) - throw DB::Exception("Empty file name"); + throw DB::Exception("Empty file name", DB::ErrorCodes::BAD_ARGUMENTS); bool tsv = fs::path(filename).extension().string() == ".tsv"; @@ -902,7 +926,7 @@ private: if (queries.empty()) { - throw DB::Exception("Did not find any query to execute: " + test_name); + throw DB::Exception("Did not find any query to execute: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS); } if (test_config->has("substitutions")) @@ -922,7 +946,7 @@ private: if (!test_config->has("type")) { - throw DB::Exception("Missing type property in config: " + test_name); + throw DB::Exception("Missing type property in config: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS); } String config_exec_type = test_config->getString("type"); @@ -931,7 +955,7 @@ private: else if (config_exec_type == "once") exec_type = ExecutionType::Once; else - throw DB::Exception("Unknown type " + config_exec_type + " in :" + test_name); + throw DB::Exception("Unknown type " + config_exec_type + " in :" + test_name, DB::ErrorCodes::BAD_ARGUMENTS); times_to_run = test_config->getUInt("times_to_run", 1); @@ -944,7 +968,7 @@ private: } if (stop_conditions_template.empty()) - throw DB::Exception("No termination conditions were found in config"); + throw DB::Exception("No termination conditions were found in config", DB::ErrorCodes::BAD_ARGUMENTS); for (size_t i = 0; i < times_to_run * queries.size(); ++i) stop_conditions_by_run.push_back(stop_conditions_template); @@ -971,7 +995,7 @@ private: else { if (lite_output) - throw DB::Exception("Specify main_metric for lite output"); + throw DB::Exception("Specify main_metric for lite output", DB::ErrorCodes::BAD_ARGUMENTS); } if (metrics.size() > 0) @@ -1016,22 +1040,14 @@ private: if (exec_type == ExecutionType::Loop) { for (const String & metric : metrics) - { if (std::find(non_loop_metrics.begin(), non_loop_metrics.end(), metric) != non_loop_metrics.end()) - { - throw DB::Exception("Wrong type of metric for loop execution type (" + metric + ")"); - } - } + throw DB::Exception("Wrong type of metric for loop execution type (" + metric + ")", DB::ErrorCodes::BAD_ARGUMENTS); } else { for (const String & metric : metrics) - { if (std::find(loop_metrics.begin(), loop_metrics.end(), metric) != loop_metrics.end()) - { - throw DB::Exception("Wrong type of metric for non-loop execution type (" + metric + ")"); - } - } + throw DB::Exception("Wrong type of metric for non-loop execution type (" + metric + ")", DB::ErrorCodes::BAD_ARGUMENTS); } } @@ -1152,11 +1168,8 @@ private: StringToVector::iterator substitutions_first = substitutions.begin(); StringToVector::iterator substitutions_last = substitutions.end(); - --substitutions_last; - std::map substitutions_map; - - runThroughAllOptionsAndPush(substitutions_first, substitutions_last, query, queries, substitutions_map); + runThroughAllOptionsAndPush(substitutions_first, substitutions_last, query, queries); return queries; } @@ -1166,44 +1179,37 @@ private: void runThroughAllOptionsAndPush(StringToVector::iterator substitutions_left, StringToVector::iterator substitutions_right, const String & template_query, - std::vector & queries, - const StringKeyValue & template_substitutions_map = StringKeyValue()) + std::vector & queries) { - String name = substitutions_left->first; - std::vector values = substitutions_left->second; + if (substitutions_left == substitutions_right) + { + queries.push_back(template_query); /// completely substituted query + return; + } - for (const String & value : values) + String substitution_mask = "{" + substitutions_left->first + "}"; + + if (template_query.find(substitution_mask) == String::npos) /// nothing to substitute here + { + runThroughAllOptionsAndPush(std::next(substitutions_left), substitutions_right, template_query, queries); + return; + } + + for (const String & value : substitutions_left->second) { /// Copy query string for each unique permutation Query query = template_query; - StringKeyValue substitutions_map = template_substitutions_map; size_t substr_pos = 0; while (substr_pos != String::npos) { - substr_pos = query.find("{" + name + "}"); + substr_pos = query.find(substitution_mask); if (substr_pos != String::npos) - { - query.replace(substr_pos, 1 + name.length() + 1, value); - } + query.replace(substr_pos, substitution_mask.length(), value); } - substitutions_map[name] = value; - - /// If we've reached the end of substitution chain - if (substitutions_left == substitutions_right) - { - queries.push_back(query); - substitutions_maps.push_back(substitutions_map); - } - else - { - StringToVector::iterator next_it = substitutions_left; - ++next_it; - - runThroughAllOptionsAndPush(next_it, substitutions_right, query, queries, substitutions_map); - } + runThroughAllOptionsAndPush(std::next(substitutions_left), substitutions_right, query, queries); } } @@ -1386,6 +1392,7 @@ static void getFilesFromDir(const fs::path & dir, std::vector & input_fi } } + int mainEntryClickHousePerformanceTest(int argc, char ** argv) try { @@ -1393,21 +1400,28 @@ try using Strings = std::vector; boost::program_options::options_description desc("Allowed options"); - desc.add_options()("help", "produce help message")("lite", "use lite version of output")( - "profiles-file", value()->default_value(""), "Specify a file with global profiles")( - "host,h", value()->default_value("localhost"), "")("port", value()->default_value(9000), "")( - "database", value()->default_value("default"), "")("user", value()->default_value("default"), "")( - "password", value()->default_value(""), "")("tags", value()->multitoken(), "Run only tests with tag")( - "skip-tags", value()->multitoken(), "Do not run tests with tag")("names", - value()->multitoken(), - "Run tests with specific name")("skip-names", value()->multitoken(), "Do not run tests with name")( - "names-regexp", value()->multitoken(), "Run tests with names matching regexp")("skip-names-regexp", - value()->multitoken(), - "Do not run tests with names matching regexp")("recursive,r", "Recurse in directories to find all xml's"); + desc.add_options() + ("help", "produce help message") + ("lite", "use lite version of output") + ("profiles-file", value()->default_value(""), "Specify a file with global profiles") + ("host,h", value()->default_value("localhost"), "") + ("port", value()->default_value(9000), "") + ("secure,s", "Use TLS connection") + ("database", value()->default_value("default"), "") + ("user", value()->default_value("default"), "") + ("password", value()->default_value(""), "") + ("tags", value()->multitoken(), "Run only tests with tag") + ("skip-tags", value()->multitoken(), "Do not run tests with tag") + ("names", value()->multitoken(), "Run tests with specific name") + ("skip-names", value()->multitoken(), "Do not run tests with name") + ("names-regexp", value()->multitoken(), "Run tests with names matching regexp") + ("skip-names-regexp", value()->multitoken(), "Do not run tests with names matching regexp") + ("recursive,r", "Recurse in directories to find all xml's"); /// These options will not be displayed in --help boost::program_options::options_description hidden("Hidden options"); - hidden.add_options()("input-files", value>(), ""); + hidden.add_options() + ("input-files", value>(), ""); /// But they will be legit, though. And they must be given without name boost::program_options::positional_options_description positional; @@ -1441,7 +1455,7 @@ try if (input_files.empty()) { std::cerr << std::endl; - throw DB::Exception("Did not find any xml files"); + throw DB::Exception("Did not find any xml files", DB::ErrorCodes::BAD_ARGUMENTS); } else std::cerr << " found " << input_files.size() << " files." << std::endl; @@ -1456,7 +1470,7 @@ try fs::path file(filename); if (!fs::exists(file)) - throw DB::Exception("File '" + filename + "' does not exist"); + throw DB::Exception("File '" + filename + "' does not exist", DB::ErrorCodes::FILE_DOESNT_EXIST); if (fs::is_directory(file)) { @@ -1465,7 +1479,7 @@ try else { if (file.extension().string() != ".xml") - throw DB::Exception("File '" + filename + "' does not have .xml extension"); + throw DB::Exception("File '" + filename + "' does not have .xml extension", DB::ErrorCodes::BAD_ARGUMENTS); collected_files.push_back(filename); } } @@ -1482,8 +1496,12 @@ try auto timeouts = DB::ConnectionTimeouts::getTCPTimeoutsWithoutFailover(DB::Settings()); - DB::PerformanceTest performanceTest(options["host"].as(), + DB::UseSSL use_ssl; + + DB::PerformanceTest performance_test( + options["host"].as(), options["port"].as(), + options.count("secure"), options["database"].as(), options["user"].as(), options["password"].as(), @@ -1497,8 +1515,7 @@ try std::move(tests_names_regexp), std::move(skip_names_regexp), timeouts); - - return 0; + return performance_test.run(); } catch (...) { diff --git a/dbms/src/Server/clickhouse-performance-test.cpp b/dbms/programs/performance-test/clickhouse-performance-test.cpp similarity index 100% rename from dbms/src/Server/clickhouse-performance-test.cpp rename to dbms/programs/performance-test/clickhouse-performance-test.cpp diff --git a/dbms/programs/server/CMakeLists.txt b/dbms/programs/server/CMakeLists.txt new file mode 100644 index 00000000000..d8caa07b743 --- /dev/null +++ b/dbms/programs/server/CMakeLists.txt @@ -0,0 +1,34 @@ +add_library (clickhouse-server-lib ${LINK_MODE} + HTTPHandler.cpp + InterserverIOHTTPHandler.cpp + MetricsTransmitter.cpp + NotFoundHandler.cpp + PingRequestHandler.cpp + ReplicasStatusHandler.cpp + RootRequestHandler.cpp + Server.cpp + TCPHandler.cpp + ) + +target_link_libraries (clickhouse-server-lib PRIVATE clickhouse_common_io daemon clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions ${Poco_Net_LIBRARY}) +if (USE_POCO_NETSSL) + target_link_libraries (clickhouse-server-lib PRIVATE ${Poco_NetSSL_LIBRARY} ${Poco_Crypto_LIBRARY}) +endif () + +target_include_directories (clickhouse-server-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include) + +if (CLICKHOUSE_SPLIT_BINARY) + add_executable (clickhouse-server clickhouse-server.cpp) + target_link_libraries (clickhouse-server PRIVATE clickhouse-server-lib) + install (TARGETS clickhouse-server ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) +endif () + +if (OS_LINUX AND MAKE_STATIC_LIBRARIES) + set (GLIBC_MAX_REQUIRED 2.4 CACHE INTERNAL "") + add_test(NAME GLIBC_required_version COMMAND bash -c "readelf -s ${CMAKE_CURRENT_BINARY_DIR}/../clickhouse-server | grep '@GLIBC' | grep -oP 'GLIBC_[\\d\\.]+' | sort | uniq | sort -r | perl -lnE 'exit 1 if $_ gt q{GLIBC_${GLIBC_MAX_REQUIRED}}'") +endif () + +install ( + FILES config.xml users.xml + DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-server + COMPONENT clickhouse) diff --git a/dbms/src/Server/HTTPHandler.cpp b/dbms/programs/server/HTTPHandler.cpp similarity index 94% rename from dbms/src/Server/HTTPHandler.cpp rename to dbms/programs/server/HTTPHandler.cpp index fba60c5c12b..34037a7c7c0 100644 --- a/dbms/src/Server/HTTPHandler.cpp +++ b/dbms/programs/server/HTTPHandler.cpp @@ -9,10 +9,12 @@ #include -#include +#include #include #include #include +#include +#include #include #include #include @@ -61,6 +63,8 @@ namespace ErrorCodes extern const int TOO_BIG_AST; extern const int UNEXPECTED_AST_STRUCTURE; + extern const int SYNTAX_ERROR; + extern const int UNKNOWN_TABLE; extern const int UNKNOWN_FUNCTION; extern const int UNKNOWN_IDENTIFIER; @@ -107,6 +111,8 @@ static Poco::Net::HTTPResponse::HTTPStatus exceptionCodeToHTTPStatus(int excepti exception_code == ErrorCodes::TOO_BIG_AST || exception_code == ErrorCodes::UNEXPECTED_AST_STRUCTURE) return HTTPResponse::HTTP_BAD_REQUEST; + else if (exception_code == ErrorCodes::SYNTAX_ERROR) + return HTTPResponse::HTTP_BAD_REQUEST; else if (exception_code == ErrorCodes::UNKNOWN_TABLE || exception_code == ErrorCodes::UNKNOWN_FUNCTION || exception_code == ErrorCodes::UNKNOWN_IDENTIFIER || @@ -208,6 +214,11 @@ void HTTPHandler::processQuery( Poco::Net::HTTPServerResponse & response, Output & used_output) { + Context context = server.context(); + context.setGlobalContext(server.context()); + + CurrentThread::QueryScope query_scope(context); + LOG_TRACE(log, "Request URI: " << request.getURI()); std::istream & istr = request.stream(); @@ -257,12 +268,6 @@ void HTTPHandler::processQuery( } std::string query_id = params.get("query_id", ""); - - const auto & config = server.config(); - - Context context = server.context(); - context.setGlobalContext(server.context()); - context.setUser(user, password, request.clientAddress(), quota_key); context.setCurrentQueryId(query_id); @@ -273,6 +278,7 @@ void HTTPHandler::processQuery( String session_id; std::chrono::steady_clock::duration session_timeout; bool session_is_set = params.has("session_id"); + const auto & config = server.config(); if (session_is_set) { @@ -421,22 +427,45 @@ void HTTPHandler::processQuery( std::unique_ptr in; + static const NameSet reserved_param_names{"query", "compress", "decompress", "user", "password", "quota_key", "query_id", "stacktrace", + "buffer_size", "wait_end_of_query", "session_id", "session_timeout", "session_check"}; + + Names reserved_param_suffixes; + + auto param_could_be_skipped = [&] (const String & name) + { + if (reserved_param_names.count(name)) + return true; + + for (const String & suffix : reserved_param_suffixes) + { + if (endsWith(name, suffix)) + return true; + } + + return false; + }; + + /// Used in case of POST request with form-data, but it isn't expected to be deleted after that scope. + std::string full_query; + /// Support for "external data for query processing". if (startsWith(request.getContentType().data(), "multipart/form-data")) { - in = std::move(in_param); ExternalTablesHandler handler(context, params); - params.load(request, istr, handler); - /// Erase unneeded parameters to avoid confusing them later with context settings or query - /// parameters. - for (const auto & it : handler.names) - { - params.erase(it + "_format"); - params.erase(it + "_types"); - params.erase(it + "_structure"); - } + /// Skip unneeded parameters to avoid confusing them later with context settings or query parameters. + reserved_param_suffixes.emplace_back("_format"); + reserved_param_suffixes.emplace_back("_types"); + reserved_param_suffixes.emplace_back("_structure"); + + /// Params are of both form params POST and uri (GET params) + for (const auto & it : params) + if (it.first == "query") + full_query += it.second; + + in = std::make_unique(full_query); } else in = std::make_unique(*in_param, *in_post_maybe_compressed); @@ -463,11 +492,6 @@ void HTTPHandler::processQuery( auto readonly_before_query = settings.readonly; - NameSet reserved_param_names{"query", "compress", "decompress", "user", "password", "quota_key", "query_id", "stacktrace", - "buffer_size", "wait_end_of_query", - "session_id", "session_timeout", "session_check" - }; - for (auto it = params.begin(); it != params.end(); ++it) { if (it->first == "database") @@ -478,7 +502,7 @@ void HTTPHandler::processQuery( { context.setDefaultFormat(it->second); } - else if (reserved_param_names.find(it->first) != reserved_param_names.end()) + else if (param_could_be_skipped(it->first)) { } else @@ -622,6 +646,8 @@ void HTTPHandler::trySendExceptionToClient(const std::string & s, int exception_ void HTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) { + setThreadName("HTTPHandler"); + Output used_output; /// In case of exception, send stack trace to client. diff --git a/dbms/src/Server/HTTPHandler.h b/dbms/programs/server/HTTPHandler.h similarity index 97% rename from dbms/src/Server/HTTPHandler.h rename to dbms/programs/server/HTTPHandler.h index 72606afa827..58cb6bb67f1 100644 --- a/dbms/src/Server/HTTPHandler.h +++ b/dbms/programs/server/HTTPHandler.h @@ -55,7 +55,7 @@ private: IServer & server; Poco::Logger * log; - /// It is the name of the server that will be sent in an http-header X-ClickHouse-Server-Display-Name. + /// It is the name of the server that will be sent in an http-header X-ClickHouse-Server-Display-Name. String server_display_name; CurrentMetrics::Increment metric_increment{CurrentMetrics::HTTPConnection}; diff --git a/dbms/src/Server/HTTPHandlerFactory.h b/dbms/programs/server/HTTPHandlerFactory.h similarity index 86% rename from dbms/src/Server/HTTPHandlerFactory.h rename to dbms/programs/server/HTTPHandlerFactory.h index 34c49ce1671..6012af9fbb5 100644 --- a/dbms/src/Server/HTTPHandlerFactory.h +++ b/dbms/programs/server/HTTPHandlerFactory.h @@ -38,7 +38,13 @@ public: << ", Address: " << request.clientAddress().toString() << ", User-Agent: " - << (request.has("User-Agent") ? request.get("User-Agent") : "none")); + << (request.has("User-Agent") ? request.get("User-Agent") : "none") + << (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : ("")) +#if !NDEBUG + << ", Content Type: " << request.getContentType() + << ", Transfer Encoding: " << request.getTransferEncoding() +#endif + ); const auto & uri = request.getURI(); diff --git a/dbms/src/Server/IServer.h b/dbms/programs/server/IServer.h similarity index 100% rename from dbms/src/Server/IServer.h rename to dbms/programs/server/IServer.h diff --git a/dbms/src/Server/InterserverIOHTTPHandler.cpp b/dbms/programs/server/InterserverIOHTTPHandler.cpp similarity index 59% rename from dbms/src/Server/InterserverIOHTTPHandler.cpp rename to dbms/programs/server/InterserverIOHTTPHandler.cpp index 3cdbaa69b64..3c93ee1989a 100644 --- a/dbms/src/Server/InterserverIOHTTPHandler.cpp +++ b/dbms/programs/server/InterserverIOHTTPHandler.cpp @@ -1,9 +1,11 @@ +#include #include #include #include #include +#include #include #include #include @@ -17,20 +19,43 @@ namespace DB namespace ErrorCodes { extern const int ABORTED; - extern const int POCO_EXCEPTION; - extern const int STD_EXCEPTION; - extern const int UNKNOWN_EXCEPTION; extern const int TOO_MANY_SIMULTANEOUS_QUERIES; } +std::pair InterserverIOHTTPHandler::checkAuthentication(Poco::Net::HTTPServerRequest & request) const +{ + const auto & config = server.config(); + + if (config.has("interserver_http_credentials.user")) + { + if (!request.hasCredentials()) + return {"Server requires HTTP Basic authentification, but client doesn't provide it", false}; + String scheme, info; + request.getCredentials(scheme, info); + + if (scheme != "Basic") + return {"Server requires HTTP Basic authentification but client provides another method", false}; + + String user = config.getString("interserver_http_credentials.user"); + String password = config.getString("interserver_http_credentials.password", ""); + + Poco::Net::HTTPBasicCredentials credentials(info); + if (std::make_pair(user, password) != std::make_pair(credentials.getUsername(), credentials.getPassword())) + return {"Incorrect user or password in HTTP Basic authentification", false}; + } + else if (request.hasCredentials()) + { + return {"Client requires HTTP Basic authentification, but server doesn't provide it", false}; + } + return {"", true}; +} + void InterserverIOHTTPHandler::processQuery(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) { HTMLForm params(request); LOG_TRACE(log, "Request URI: " << request.getURI()); - /// NOTE: You can do authentication here if you need to. - String endpoint_name = params.get("endpoint"); bool compress = params.get("compress") == "true"; @@ -59,14 +84,26 @@ void InterserverIOHTTPHandler::processQuery(Poco::Net::HTTPServerRequest & reque void InterserverIOHTTPHandler::handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) { + setThreadName("IntersrvHandler"); + /// In order to work keep-alive. if (request.getVersion() == Poco::Net::HTTPServerRequest::HTTP_1_1) response.setChunkedTransferEncoding(true); try { - processQuery(request, response); - LOG_INFO(log, "Done processing query"); + if (auto [msg, success] = checkAuthentication(request); success) + { + processQuery(request, response); + LOG_INFO(log, "Done processing query"); + } + else + { + response.setStatusAndReason(Poco::Net::HTTPServerResponse::HTTP_UNAUTHORIZED); + if (!response.sent()) + response.send() << msg << std::endl; + LOG_WARNING(log, "Query processing failed request: '" << request.getURI() << "' authentification failed"); + } } catch (Exception & e) { diff --git a/dbms/src/Server/InterserverIOHTTPHandler.h b/dbms/programs/server/InterserverIOHTTPHandler.h similarity index 89% rename from dbms/src/Server/InterserverIOHTTPHandler.h rename to dbms/programs/server/InterserverIOHTTPHandler.h index bf9fef59982..fbaf432d4f9 100644 --- a/dbms/src/Server/InterserverIOHTTPHandler.h +++ b/dbms/programs/server/InterserverIOHTTPHandler.h @@ -34,6 +34,8 @@ private: CurrentMetrics::Increment metric_increment{CurrentMetrics::InterserverConnection}; void processQuery(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response); + + std::pair checkAuthentication(Poco::Net::HTTPServerRequest & request) const; }; } diff --git a/dbms/src/Server/MetricsTransmitter.cpp b/dbms/programs/server/MetricsTransmitter.cpp similarity index 91% rename from dbms/src/Server/MetricsTransmitter.cpp rename to dbms/programs/server/MetricsTransmitter.cpp index 278347a6774..3268641942e 100644 --- a/dbms/src/Server/MetricsTransmitter.cpp +++ b/dbms/programs/server/MetricsTransmitter.cpp @@ -81,11 +81,11 @@ void MetricsTransmitter::transmit(std::vector & prev_count { for (size_t i = 0, end = ProfileEvents::end(); i < end; ++i) { - const auto counter = ProfileEvents::counters[i].load(std::memory_order_relaxed); + const auto counter = ProfileEvents::global_counters[i].load(std::memory_order_relaxed); const auto counter_increment = counter - prev_counters[i]; prev_counters[i] = counter; - std::string key{ProfileEvents::getDescription(static_cast(i))}; + std::string key{ProfileEvents::getName(static_cast(i))}; key_vals.emplace_back(profile_events_path_prefix + key, counter_increment); } } @@ -96,7 +96,7 @@ void MetricsTransmitter::transmit(std::vector & prev_count { const auto value = CurrentMetrics::values[i].load(std::memory_order_relaxed); - std::string key{CurrentMetrics::getDescription(static_cast(i))}; + std::string key{CurrentMetrics::getName(static_cast(i))}; key_vals.emplace_back(current_metrics_path_prefix + key, value); } } diff --git a/dbms/src/Server/MetricsTransmitter.h b/dbms/programs/server/MetricsTransmitter.h similarity index 100% rename from dbms/src/Server/MetricsTransmitter.h rename to dbms/programs/server/MetricsTransmitter.h diff --git a/dbms/src/Server/NotFoundHandler.cpp b/dbms/programs/server/NotFoundHandler.cpp similarity index 100% rename from dbms/src/Server/NotFoundHandler.cpp rename to dbms/programs/server/NotFoundHandler.cpp diff --git a/dbms/src/Server/NotFoundHandler.h b/dbms/programs/server/NotFoundHandler.h similarity index 100% rename from dbms/src/Server/NotFoundHandler.h rename to dbms/programs/server/NotFoundHandler.h diff --git a/dbms/src/Server/PingRequestHandler.cpp b/dbms/programs/server/PingRequestHandler.cpp similarity index 100% rename from dbms/src/Server/PingRequestHandler.cpp rename to dbms/programs/server/PingRequestHandler.cpp diff --git a/dbms/src/Server/PingRequestHandler.h b/dbms/programs/server/PingRequestHandler.h similarity index 100% rename from dbms/src/Server/PingRequestHandler.h rename to dbms/programs/server/PingRequestHandler.h diff --git a/dbms/src/Server/ReplicasStatusHandler.cpp b/dbms/programs/server/ReplicasStatusHandler.cpp similarity index 100% rename from dbms/src/Server/ReplicasStatusHandler.cpp rename to dbms/programs/server/ReplicasStatusHandler.cpp diff --git a/dbms/src/Server/ReplicasStatusHandler.h b/dbms/programs/server/ReplicasStatusHandler.h similarity index 100% rename from dbms/src/Server/ReplicasStatusHandler.h rename to dbms/programs/server/ReplicasStatusHandler.h diff --git a/dbms/src/Server/RootRequestHandler.cpp b/dbms/programs/server/RootRequestHandler.cpp similarity index 100% rename from dbms/src/Server/RootRequestHandler.cpp rename to dbms/programs/server/RootRequestHandler.cpp diff --git a/dbms/src/Server/RootRequestHandler.h b/dbms/programs/server/RootRequestHandler.h similarity index 100% rename from dbms/src/Server/RootRequestHandler.h rename to dbms/programs/server/RootRequestHandler.h diff --git a/dbms/src/Server/Server.cpp b/dbms/programs/server/Server.cpp similarity index 64% rename from dbms/src/Server/Server.cpp rename to dbms/programs/server/Server.cpp index 820a593cd09..32f88079ce3 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -2,6 +2,8 @@ #include #include +#include +#include #include #include #include @@ -10,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -19,24 +22,33 @@ #include #include #include +#include #include +#include #include #include #include #include +#include #include #include #include #include #include #include +#include #include #include "HTTPHandlerFactory.h" #include "MetricsTransmitter.h" -#include "StatusFile.h" +#include #include "TCPHandlerFactory.h" -#if Poco_NetSSL_FOUND +#if defined(__linux__) +#include +#include +#endif + +#if USE_POCO_NETSSL #include #include #endif @@ -44,6 +56,7 @@ namespace CurrentMetrics { extern const Metric Revision; + extern const Metric VersionInteger; } namespace DB @@ -54,6 +67,9 @@ namespace ErrorCodes extern const int NO_ELEMENTS_IN_CONFIG; extern const int SUPPORT_IS_DISABLED; extern const int ARGUMENT_OUT_OF_BOUND; + extern const int EXCESSIVE_ELEMENT_IN_CONFIG; + extern const int INVALID_CONFIG_PARAMETER; + extern const int SYSTEM_ERROR; } @@ -61,10 +77,10 @@ static std::string getCanonicalPath(std::string && path) { Poco::trimInPlace(path); if (path.empty()) - throw Exception("path configuration parameter is empty"); + throw Exception("path configuration parameter is empty", ErrorCodes::INVALID_CONFIG_PARAMETER); if (path.back() != '/') path += '/'; - return path; + return std::move(path); } void Server::uninitialize() @@ -81,19 +97,23 @@ void Server::initialize(Poco::Util::Application & self) std::string Server::getDefaultCorePath() const { - return getCanonicalPath(config().getString("path")) + "cores"; + return getCanonicalPath(config().getString("path", DBMS_DEFAULT_PATH)) + "cores"; } int Server::main(const std::vector & /*args*/) { Logger * log = &logger(); + UseSSL use_ssl; + registerFunctions(); registerAggregateFunctions(); registerTableFunctions(); registerStorages(); + registerDictionaries(); CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::get()); + CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger()); /** Context contains all that query execution is dependent: * settings, available functions, data types, aggregate functions, databases... @@ -102,12 +122,7 @@ int Server::main(const std::vector & /*args*/) global_context->setGlobalContext(*global_context); global_context->setApplicationType(Context::ApplicationType::SERVER); - bool has_zookeeper = false; - if (config().has("zookeeper")) - { - global_context->setZooKeeper(std::make_shared(config(), "zookeeper")); - has_zookeeper = true; - } + bool has_zookeeper = config().has("zookeeper"); zkutil::ZooKeeperNodeCache main_config_zk_node_cache([&] { return global_context->getZooKeeper(); }); if (loaded_config.has_zk_includes) @@ -116,12 +131,38 @@ int Server::main(const std::vector & /*args*/) ConfigProcessor config_processor(config_path); loaded_config = config_processor.loadConfigWithZooKeeperIncludes( main_config_zk_node_cache, /* fallback_to_preprocessed = */ true); - config_processor.savePreprocessedConfig(loaded_config); + config_processor.savePreprocessedConfig(loaded_config, config().getString("path", DBMS_DEFAULT_PATH)); config().removeConfiguration(old_configuration.get()); config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false); } - std::string path = getCanonicalPath(config().getString("path")); + const auto memory_amount = getMemoryAmount(); + +#if defined(__linux__) + /// After full config loaded + { + if (config().getBool("mlock_executable", false)) + { + if (hasLinuxCapability(CAP_IPC_LOCK)) + { + LOG_TRACE(log, "Will mlockall to prevent executable memory from being paged out. It may take a few seconds."); + if (0 != mlockall(MCL_CURRENT)) + LOG_WARNING(log, "Failed mlockall: " + errnoToString(ErrorCodes::SYSTEM_ERROR)); + else + LOG_TRACE(log, "The memory map of clickhouse executable has been mlock'ed"); + } + else + { + LOG_INFO(log, "It looks like the process has no CAP_IPC_LOCK capability, binary mlock will be disabled." + " It could happen due to incorrect ClickHouse package installation." + " You could resolve the problem manually with 'sudo setcap cap_ipc_lock=+ep /usr/bin/clickhouse'." + " Note that it will not work on 'nosuid' mounted filesystems."); + } + } + } +#endif + + std::string path = getCanonicalPath(config().getString("path", DBMS_DEFAULT_PATH)); std::string default_database = config().getString("default_database", "default"); global_context->setPath(path); @@ -174,7 +215,7 @@ int Server::main(const std::vector & /*args*/) DateLUT::instance(); LOG_TRACE(log, "Initialized DateLUT with time zone `" << DateLUT::instance().getTimeZone() << "'."); - /// Directory with temporary data for processing of hard queries. + /// Directory with temporary data for processing of heavy queries. { std::string tmp_path = config().getString("tmp_path", path + "tmp/"); global_context->setTemporaryPath(tmp_path); @@ -196,28 +237,63 @@ int Server::main(const std::vector & /*args*/) * Flags may be cleared automatically after being applied by the server. * Examples: do repair of local data; clone all replicated tables from replica. */ - Poco::File(path + "flags/").createDirectories(); - global_context->setFlagsPath(path + "flags/"); - - if (config().has("interserver_http_port")) { - String this_host = config().getString("interserver_http_host", ""); + Poco::File(path + "flags/").createDirectories(); + global_context->setFlagsPath(path + "flags/"); + } - if (this_host.empty()) + /** Directory with user provided files that are usable by 'file' table function. + */ + { + + std::string user_files_path = config().getString("user_files_path", path + "user_files/"); + global_context->setUserFilesPath(user_files_path); + Poco::File(user_files_path).createDirectories(); + } + + if (config().has("interserver_http_port") && config().has("interserver_https_port")) + throw Exception("Both http and https interserver ports are specified", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG); + + static const auto interserver_tags = + { + std::make_tuple("interserver_http_host", "interserver_http_port", "http"), + std::make_tuple("interserver_https_host", "interserver_https_port", "https") + }; + + for (auto [host_tag, port_tag, scheme] : interserver_tags) + { + if (config().has(port_tag)) { - this_host = getFQDNOrHostName(); - LOG_DEBUG(log, - "Configuration parameter 'interserver_http_host' doesn't exist or exists and empty. Will use '" + this_host - + "' as replica host."); + String this_host = config().getString(host_tag, ""); + + if (this_host.empty()) + { + this_host = getFQDNOrHostName(); + LOG_DEBUG(log, + "Configuration parameter '" + String(host_tag) + "' doesn't exist or exists and empty. Will use '" + this_host + + "' as replica host."); + } + + String port_str = config().getString(port_tag); + int port = parse(port_str); + + if (port < 0 || port > 0xFFFF) + throw Exception("Out of range '" + String(port_tag) + "': " + toString(port), ErrorCodes::ARGUMENT_OUT_OF_BOUND); + + global_context->setInterserverIOAddress(this_host, port); + global_context->setInterserverScheme(scheme); } + } - String port_str = config().getString("interserver_http_port"); - int port = parse(port_str); + if (config().has("interserver_http_credentials")) + { + String user = config().getString("interserver_http_credentials.user", ""); + String password = config().getString("interserver_http_credentials.password", ""); - if (port < 0 || port > 0xFFFF) - throw Exception("Out of range 'interserver_http_port': " + toString(port), ErrorCodes::ARGUMENT_OUT_OF_BOUND); + if (user.empty()) + throw Exception("Configuration parameter interserver_http_credentials user can't be empty", ErrorCodes::NO_ELEMENTS_IN_CONFIG); - global_context->setInterserverIOAddress(this_host, port); + global_context->setInterserverCredentials(user, password); } if (config().has("macros")) @@ -227,9 +303,11 @@ int Server::main(const std::vector & /*args*/) std::string include_from_path = config().getString("include_from", "/etc/metrika.xml"); auto main_config_reloader = std::make_unique(config_path, include_from_path, + config().getString("path", ""), std::move(main_config_zk_node_cache), [&](ConfigurationPtr config) { + buildLoggers(*config); global_context->setClustersConfig(config); global_context->setMacros(std::make_unique(*config, "macros")); }, @@ -247,12 +325,14 @@ int Server::main(const std::vector & /*args*/) } auto users_config_reloader = std::make_unique(users_config_path, include_from_path, + config().getString("path", ""), zkutil::ZooKeeperNodeCache([&] { return global_context->getZooKeeper(); }), [&](ConfigurationPtr config) { global_context->setUsersConfig(config); }, /* already_loaded = */ false); /// Reload config in SYSTEM RELOAD CONFIG query. - global_context->setConfigReloadCallback([&]() { + global_context->setConfigReloadCallback([&]() + { main_config_reloader->reload(); users_config_reloader->reload(); }); @@ -264,6 +344,9 @@ int Server::main(const std::vector & /*args*/) if (config().has("max_table_size_to_drop")) global_context->setMaxTableSizeToDrop(config().getUInt64("max_table_size_to_drop")); + if (config().has("max_partition_size_to_drop")) + global_context->setMaxPartitionSizeToDrop(config().getUInt64("max_partition_size_to_drop")); + /// Size of cache for uncompressed blocks. Zero means disabled. size_t uncompressed_cache_size = config().getUInt64("uncompressed_cache_size", 0); if (uncompressed_cache_size) @@ -278,6 +361,12 @@ int Server::main(const std::vector & /*args*/) if (mark_cache_size) global_context->setMarkCache(mark_cache_size); +#if USE_EMBEDDED_COMPILER + size_t compiled_expression_cache_size = config().getUInt64("compiled_expression_cache_size", std::numeric_limits::max()); + if (compiled_expression_cache_size) + global_context->setCompiledExpressionCache(compiled_expression_cache_size); +#endif + /// Set path for format schema files auto format_schema_path = Poco::File(config().getString("format_schema_path", path + "format_schemas/")); global_context->setFormatSchemaPath(format_schema_path.path() + "/"); @@ -313,19 +402,44 @@ int Server::main(const std::vector & /*args*/) global_context->setDDLWorker(std::make_shared(ddl_zookeeper_path, *global_context, &config(), "distributed_ddl")); } + std::unique_ptr dns_cache_updater; + if (config().has("disable_internal_dns_cache") && config().getInt("disable_internal_dns_cache")) + { + /// Disable DNS caching at all + DNSResolver::instance().setDisableCacheFlag(); + } + else + { + /// Initialize a watcher updating DNS cache in case of network errors + dns_cache_updater = std::make_unique(*global_context); + } + +#if defined(__linux__) + if (!TaskStatsInfoGetter::checkPermissions()) + { + LOG_INFO(log, "It looks like the process has no CAP_NET_ADMIN capability, 'taskstats' performance statistics will be disabled." + " It could happen due to incorrect ClickHouse package installation." + " You could resolve the problem manually with 'sudo setcap cap_net_admin=+ep /usr/bin/clickhouse'." + " Note that it will not work on 'nosuid' mounted filesystems." + " It also doesn't work if you run clickhouse-server inside network namespace as it happens in some containers."); + } +#else + LOG_INFO(log, "TaskStats is not implemented for this OS. IO accounting will be disabled."); +#endif + { Poco::Timespan keep_alive_timeout(config().getUInt("keep_alive_timeout", 10), 0); Poco::ThreadPool server_pool(3, config().getUInt("max_connections", 1024)); Poco::Net::HTTPServerParams::Ptr http_params = new Poco::Net::HTTPServerParams; - http_params->setTimeout(settings.receive_timeout); + http_params->setTimeout(settings.http_receive_timeout); http_params->setKeepAliveTimeout(keep_alive_timeout); std::vector> servers; std::vector listen_hosts = DB::getMultipleValuesFromConfig(config(), "", "listen_host"); - bool listen_try = config().getUInt("listen_try", false); + bool listen_try = config().getBool("listen_try", false); if (listen_hosts.empty()) { listen_hosts.emplace_back("::1"); @@ -361,6 +475,27 @@ int Server::main(const std::vector & /*args*/) return socket_address; }; + auto socket_bind_listen = [&](auto & socket, const std::string & host, UInt16 port, bool secure = 0) + { + auto address = make_socket_address(host, port); +#if !POCO_CLICKHOUSE_PATCH || POCO_VERSION <= 0x02000000 // TODO: fill correct version + if (secure) + /// Bug in old poco, listen() after bind() with reusePort param will fail because have no implementation in SecureServerSocketImpl + /// https://github.com/pocoproject/poco/pull/2257 + socket.bind(address, /* reuseAddress = */ true); + else +#endif +#if POCO_VERSION < 0x01080000 + socket.bind(address, /* reuseAddress = */ true); +#else + socket.bind(address, /* reuseAddress = */ true, /* reusePort = */ config().getBool("listen_reuse_port", false)); +#endif + + socket.listen(/* backlog = */ config().getUInt("listen_backlog", 64)); + + return address; + }; + for (const auto & listen_host : listen_hosts) { /// For testing purposes, user may omit tcp_port or http_port or https_port in configuration file. @@ -369,37 +504,34 @@ int Server::main(const std::vector & /*args*/) /// HTTP if (config().has("http_port")) { - Poco::Net::SocketAddress http_socket_address = make_socket_address(listen_host, config().getInt("http_port")); - Poco::Net::ServerSocket http_socket(http_socket_address); - http_socket.setReceiveTimeout(settings.http_receive_timeout); - http_socket.setSendTimeout(settings.http_send_timeout); - + Poco::Net::ServerSocket socket; + auto address = socket_bind_listen(socket, listen_host, config().getInt("http_port")); + socket.setReceiveTimeout(settings.http_receive_timeout); + socket.setSendTimeout(settings.http_send_timeout); servers.emplace_back(new Poco::Net::HTTPServer( new HTTPHandlerFactory(*this, "HTTPHandler-factory"), server_pool, - http_socket, + socket, http_params)); - LOG_INFO(log, "Listening http://" + http_socket_address.toString()); + LOG_INFO(log, "Listening http://" + address.toString()); } /// HTTPS if (config().has("https_port")) { -#if Poco_NetSSL_FOUND - std::call_once(ssl_init_once, SSLInit); - Poco::Net::SocketAddress http_socket_address = make_socket_address(listen_host, config().getInt("https_port")); - Poco::Net::SecureServerSocket http_socket(http_socket_address); - http_socket.setReceiveTimeout(settings.http_receive_timeout); - http_socket.setSendTimeout(settings.http_send_timeout); - +#if USE_POCO_NETSSL + Poco::Net::SecureServerSocket socket; + auto address = socket_bind_listen(socket, listen_host, config().getInt("https_port"), /* secure = */ true); + socket.setReceiveTimeout(settings.http_receive_timeout); + socket.setSendTimeout(settings.http_send_timeout); servers.emplace_back(new Poco::Net::HTTPServer( new HTTPHandlerFactory(*this, "HTTPSHandler-factory"), server_pool, - http_socket, + socket, http_params)); - LOG_INFO(log, "Listening https://" + http_socket_address.toString()); + LOG_INFO(log, "Listening https://" + address.toString()); #else throw Exception{"HTTPS protocol is disabled because Poco library was built without NetSSL support.", ErrorCodes::SUPPORT_IS_DISABLED}; @@ -409,34 +541,33 @@ int Server::main(const std::vector & /*args*/) /// TCP if (config().has("tcp_port")) { - std::call_once(ssl_init_once, SSLInit); - Poco::Net::SocketAddress tcp_address = make_socket_address(listen_host, config().getInt("tcp_port")); - Poco::Net::ServerSocket tcp_socket(tcp_address); - tcp_socket.setReceiveTimeout(settings.receive_timeout); - tcp_socket.setSendTimeout(settings.send_timeout); + Poco::Net::ServerSocket socket; + auto address = socket_bind_listen(socket, listen_host, config().getInt("tcp_port")); + socket.setReceiveTimeout(settings.receive_timeout); + socket.setSendTimeout(settings.send_timeout); servers.emplace_back(new Poco::Net::TCPServer( new TCPHandlerFactory(*this), server_pool, - tcp_socket, + socket, new Poco::Net::TCPServerParams)); - LOG_INFO(log, "Listening tcp: " + tcp_address.toString()); + LOG_INFO(log, "Listening tcp: " + address.toString()); } /// TCP with SSL - if (config().has("tcp_ssl_port")) + if (config().has("tcp_port_secure")) { -#if Poco_NetSSL_FOUND - Poco::Net::SocketAddress tcp_address = make_socket_address(listen_host, config().getInt("tcp_ssl_port")); - Poco::Net::SecureServerSocket tcp_socket(tcp_address); - tcp_socket.setReceiveTimeout(settings.receive_timeout); - tcp_socket.setSendTimeout(settings.send_timeout); +#if USE_POCO_NETSSL + Poco::Net::SecureServerSocket socket; + auto address = socket_bind_listen(socket, listen_host, config().getInt("tcp_port_secure"), /* secure = */ true); + socket.setReceiveTimeout(settings.receive_timeout); + socket.setSendTimeout(settings.send_timeout); servers.emplace_back(new Poco::Net::TCPServer( - new TCPHandlerFactory(*this, /* secure= */ true ), - server_pool, - tcp_socket, - new Poco::Net::TCPServerParams)); - LOG_INFO(log, "Listening tcp_ssl: " + tcp_address.toString()); + new TCPHandlerFactory(*this, /* secure= */ true), + server_pool, + socket, + new Poco::Net::TCPServerParams)); + LOG_INFO(log, "Listening tcp_secure: " + address.toString()); #else throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.", ErrorCodes::SUPPORT_IS_DISABLED}; @@ -450,17 +581,37 @@ int Server::main(const std::vector & /*args*/) /// Interserver IO HTTP if (config().has("interserver_http_port")) { - Poco::Net::SocketAddress interserver_address = make_socket_address(listen_host, config().getInt("interserver_http_port")); - Poco::Net::ServerSocket interserver_io_http_socket(interserver_address); - interserver_io_http_socket.setReceiveTimeout(settings.http_receive_timeout); - interserver_io_http_socket.setSendTimeout(settings.http_send_timeout); + Poco::Net::ServerSocket socket; + auto address = socket_bind_listen(socket, listen_host, config().getInt("interserver_http_port")); + socket.setReceiveTimeout(settings.http_receive_timeout); + socket.setSendTimeout(settings.http_send_timeout); servers.emplace_back(new Poco::Net::HTTPServer( new InterserverIOHTTPHandlerFactory(*this, "InterserverIOHTTPHandler-factory"), server_pool, - interserver_io_http_socket, + socket, http_params)); - LOG_INFO(log, "Listening interserver: " + interserver_address.toString()); + LOG_INFO(log, "Listening interserver http: " + address.toString()); + } + + if (config().has("interserver_https_port")) + { +#if USE_POCO_NETSSL + Poco::Net::SecureServerSocket socket; + auto address = socket_bind_listen(socket, listen_host, config().getInt("interserver_https_port"), /* secure = */ true); + socket.setReceiveTimeout(settings.http_receive_timeout); + socket.setSendTimeout(settings.http_send_timeout); + servers.emplace_back(new Poco::Net::HTTPServer( + new InterserverIOHTTPHandlerFactory(*this, "InterserverIOHTTPHandler-factory"), + server_pool, + socket, + http_params)); + + LOG_INFO(log, "Listening interserver https: " + address.toString()); +#else + throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.", + ErrorCodes::SUPPORT_IS_DISABLED}; +#endif } } catch (const Poco::Net::NetException & e) @@ -487,10 +638,10 @@ int Server::main(const std::vector & /*args*/) { std::stringstream message; - message << "Available RAM = " << formatReadableSizeWithBinarySuffix(getMemoryAmount()) << ";" + message << "Available RAM = " << formatReadableSizeWithBinarySuffix(memory_amount) << ";" << " physical cores = " << getNumberOfPhysicalCPUCores() << ";" // on ARM processors it can show only enabled at current moment cores - << " threads = " << std::thread::hardware_concurrency() << "."; + << " threads = " << std::thread::hardware_concurrency() << "."; LOG_INFO(log, message.str()); } diff --git a/dbms/src/Server/Server.h b/dbms/programs/server/Server.h similarity index 100% rename from dbms/src/Server/Server.h rename to dbms/programs/server/Server.h diff --git a/dbms/src/Server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp similarity index 75% rename from dbms/src/Server/TCPHandler.cpp rename to dbms/programs/server/TCPHandler.cpp index faf39dded67..834369582af 100644 --- a/dbms/src/Server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -1,37 +1,38 @@ #include - +#include #include +#include #include - +#include #include - +#include +#include +#include +#include +#include #include - #include #include #include #include +#include +#include #include - #include - #include #include #include #include #include #include - +#include #include #include - -#include +#include #include "TCPHandler.h" -#include - namespace DB { @@ -51,6 +52,8 @@ namespace ErrorCodes void TCPHandler::runImpl() { + setThreadName("TCPHandler"); + connection_context = server.context(); connection_context.setSessionContext(connection_context); @@ -90,7 +93,7 @@ void TCPHandler::runImpl() try { /// We try to send error information to the client. - sendException(e); + sendException(e, connection_context.getSettingsRef().calculate_text_stack_trace); } catch (...) {} @@ -105,7 +108,7 @@ void TCPHandler::runImpl() Exception e("Database " + default_database + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); LOG_ERROR(log, "Code: " << e.code() << ", e.displayText() = " << e.displayText() << ", Stack trace:\n\n" << e.getStackTrace().toString()); - sendException(e); + sendException(e, connection_context.getSettingsRef().calculate_text_stack_trace); return; } @@ -118,6 +121,9 @@ void TCPHandler::runImpl() while (1) { + /// Restore context of request. + query_context = connection_context; + /// We are waiting for a packet from the client. Thus, every `POLL_INTERVAL` seconds check whether we need to shut down. while (!static_cast(*in).poll(global_settings.poll_interval * 1000000) && !server.isCancelled()) ; @@ -129,15 +135,21 @@ void TCPHandler::runImpl() Stopwatch watch; state.reset(); + /// Initialized later. + std::optional query_scope; + /** An exception during the execution of request (it must be sent over the network to the client). * The client will be able to accept it, if it did not happen while sending another packet and the client has not disconnected yet. */ std::unique_ptr exception; + bool network_error = false; + + bool send_exception_with_stack_trace = connection_context.getSettingsRef().calculate_text_stack_trace; try { - /// Restore context of request. - query_context = connection_context; + /// If a user passed query-local timeouts, reset socket to initial state at the end of the query + SCOPE_EXIT({state.timeout_setter.reset();}); /** If Query - process it. If Ping or Cancel - go back to the beginning. * There may come settings for a separate query that modify `query_context`. @@ -145,13 +157,32 @@ void TCPHandler::runImpl() if (!receivePacket()) continue; - /// Get blocks of temporary tables - readData(global_settings); + query_scope.emplace(query_context); - /// Reset the input stream, as we received an empty block while receiving external table data. - /// So, the stream has been marked as cancelled and we can't read from it anymore. - state.block_in.reset(); - state.maybe_compressed_in.reset(); /// For more accurate accounting by MemoryTracker. + send_exception_with_stack_trace = query_context.getSettingsRef().calculate_text_stack_trace; + + /// Should we send internal logs to client? + if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_LOGS + && query_context.getSettingsRef().send_logs_level.value != "none") + { + state.logs_queue = std::make_shared(); + state.logs_queue->max_priority = Poco::Logger::parseLevel(query_context.getSettingsRef().send_logs_level.value); + CurrentThread::attachInternalTextLogsQueue(state.logs_queue); + } + + query_context.setExternalTablesInitializer([&global_settings, this] (Context & context) + { + if (&context != &query_context) + throw Exception("Unexpected context in external tables initializer", ErrorCodes::LOGICAL_ERROR); + + /// Get blocks of temporary tables + readData(global_settings); + + /// Reset the input stream, as we received an empty block while receiving external table data. + /// So, the stream has been marked as cancelled and we can't read from it anymore. + state.block_in.reset(); + state.maybe_compressed_in.reset(); /// For more accurate accounting by MemoryTracker. + }); /// Processing Query state.io = executeQuery(state.query, query_context, false, state.stage); @@ -168,8 +199,13 @@ void TCPHandler::runImpl() else processOrdinaryQuery(); + /// Do it before sending end of stream, to have a chance to show log message in client. + query_scope->logPeakMemoryUsage(); + + sendLogs(); sendEndOfStream(); + query_scope.reset(); state.reset(); } catch (const Exception & e) @@ -179,6 +215,10 @@ void TCPHandler::runImpl() if (e.code() == ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT) throw; + + /// If a timeout occurred, try to inform client about it and close the session + if (e.code() == ErrorCodes::SOCKET_TIMEOUT) + network_error = true; } catch (const Poco::Net::NetException & e) { @@ -207,12 +247,23 @@ void TCPHandler::runImpl() exception = std::make_unique("Unknown exception", ErrorCodes::UNKNOWN_EXCEPTION); } - bool network_error = false; - try { if (exception) - sendException(*exception); + { + try + { + /// Try to send logs to client, but it could be risky too + /// Assume that we can't break output here + sendLogs(); + } + catch (...) + { + tryLogCurrentException(log, "Can't send logs to client"); + } + + sendException(*exception, send_exception_with_stack_trace); + } } catch (...) { @@ -223,6 +274,7 @@ void TCPHandler::runImpl() try { + query_scope.reset(); state.reset(); } catch (...) @@ -247,14 +299,24 @@ void TCPHandler::runImpl() void TCPHandler::readData(const Settings & global_settings) { - while (1) + auto receive_timeout = query_context.getSettingsRef().receive_timeout.value; + + /// Poll interval should not be greater than receive_timeout + size_t default_poll_interval = global_settings.poll_interval.value * 1000000; + size_t current_poll_interval = static_cast(receive_timeout.totalMicroseconds()); + constexpr size_t min_poll_interval = 5000; // 5 ms + size_t poll_interval = std::max(min_poll_interval, std::min(default_poll_interval, current_poll_interval)); + + sendLogs(); + + while (true) { Stopwatch watch(CLOCK_MONOTONIC_COARSE); /// We are waiting for a packet from the client. Thus, every `POLL_INTERVAL` seconds check whether we need to shut down. - while (1) + while (true) { - if (static_cast(*in).poll(global_settings.poll_interval * 1000000)) + if (static_cast(*in).poll(poll_interval)) break; /// Do we need to shut down? @@ -265,8 +327,16 @@ void TCPHandler::readData(const Settings & global_settings) * If we periodically poll, the receive_timeout of the socket itself does not work. * Therefore, an additional check is added. */ - if (watch.elapsedSeconds() > global_settings.receive_timeout.totalSeconds()) - throw Exception("Timeout exceeded while receiving data from client", ErrorCodes::SOCKET_TIMEOUT); + double elapsed = watch.elapsedSeconds(); + if (elapsed > receive_timeout.totalSeconds()) + { + std::stringstream ss; + ss << "Timeout exceeded while receiving data from client."; + ss << " Waited for " << static_cast(elapsed) << " seconds,"; + ss << " timeout is " << receive_timeout.totalSeconds() << " seconds."; + + throw Exception(ss.str(), ErrorCodes::SOCKET_TIMEOUT); + } } /// If client disconnected. @@ -276,6 +346,8 @@ void TCPHandler::readData(const Settings & global_settings) /// We accept and process data. And if they are over, then we leave. if (!receivePacket()) break; + + sendLogs(); } } @@ -333,6 +405,8 @@ void TCPHandler::processOrdinaryQuery() sendProgress(); } + sendLogs(); + if (async_in.poll(query_context.getSettingsRef().interactive_delay / 1000)) { /// There is the following result block. @@ -355,6 +429,7 @@ void TCPHandler::processOrdinaryQuery() sendExtremes(); sendProfileInfo(); sendProgress(); + sendLogs(); } sendData(block); @@ -480,6 +555,7 @@ void TCPHandler::receiveHello() readStringBinary(client_name, *in); readVarUInt(client_version_major, *in); readVarUInt(client_version_minor, *in); + // NOTE For backward compatibility of the protocol, client cannot send its version_patch. readVarUInt(client_revision, *in); readStringBinary(default_database, *in); readStringBinary(user, *in); @@ -488,7 +564,8 @@ void TCPHandler::receiveHello() LOG_DEBUG(log, "Connected " << client_name << " version " << client_version_major << "." << client_version_minor - << "." << client_revision + << "." << client_version_patch + << ", revision: " << client_revision << (!default_database.empty() ? ", database: " + default_database : "") << (!user.empty() ? ", user: " + user : "") << "."); @@ -505,13 +582,11 @@ void TCPHandler::sendHello() writeVarUInt(DBMS_VERSION_MINOR, *out); writeVarUInt(ClickHouseRevision::get(), *out); if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE) - { writeStringBinary(DateLUT::instance().getTimeZone(), *out); - } if (client_revision >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME) - { writeStringBinary(server_display_name, *out); - } + if (client_revision >= DBMS_MIN_REVISION_WITH_VERSION_PATCH) + writeVarUInt(DBMS_VERSION_PATCH, *out); out->next(); } @@ -556,7 +631,7 @@ bool TCPHandler::receivePacket() return false; default: - throw Exception("Unknown packet from client", ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT); + throw Exception("Unknown packet " + toString(packet_type) + " from client", ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT); } } @@ -584,6 +659,7 @@ void TCPHandler::receiveQuery() client_info.client_name = client_name; client_info.client_version_major = client_version_major; client_info.client_version_minor = client_version_minor; + client_info.client_version_patch = client_version_patch; client_info.client_revision = client_revision; } @@ -600,7 +676,14 @@ void TCPHandler::receiveQuery() } /// Per query settings. - query_context.getSettingsRef().deserialize(*in); + Settings & settings = query_context.getSettingsRef(); + settings.deserialize(*in); + + /// Sync timeouts on client and server during current query to avoid dangling queries on server + /// NOTE: We use settings.send_timeout for the receive timeout and vice versa (change arguments ordering in TimeoutSetter), + /// because settings.send_timeout is client-side setting which has opposite meaning on the server side. + /// NOTE: these settings are applied only for current connection (not for distributed tables' connections) + state.timeout_setter = std::make_unique(socket(), settings.receive_timeout, settings.send_timeout); readVarUInt(stage, *in); state.stage = QueryProcessingStage::Enum(stage); @@ -635,7 +718,7 @@ bool TCPHandler::receiveData() { NamesAndTypesList columns = block.getNamesAndTypesList(); storage = StorageMemory::create(external_table_name, - ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}}); + ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, ColumnComments{}}); storage->startup(); query_context.addExternalTable(external_table_name, storage); } @@ -671,11 +754,14 @@ void TCPHandler::initBlockOutput(const Block & block) { if (!state.block_out) { - if (state.compression == Protocol::Compression::Enable) - state.maybe_compressed_out = std::make_shared( - *out, CompressionSettings(query_context.getSettingsRef())); - else - state.maybe_compressed_out = out; + if (!state.maybe_compressed_out) + { + if (state.compression == Protocol::Compression::Enable) + state.maybe_compressed_out = std::make_shared( + *out, CompressionSettings(query_context.getSettingsRef())); + else + state.maybe_compressed_out = out; + } state.block_out = std::make_shared( *state.maybe_compressed_out, @@ -684,6 +770,18 @@ void TCPHandler::initBlockOutput(const Block & block) } } +void TCPHandler::initLogsBlockOutput(const Block & block) +{ + if (!state.logs_block_out) + { + /// Use uncompressed stream since log blocks usually contain only one row + state.logs_block_out = std::make_shared( + *out, + client_revision, + block.cloneEmpty()); + } +} + bool TCPHandler::isQueryCancelled() { @@ -724,6 +822,7 @@ void TCPHandler::sendData(const Block & block) initBlockOutput(block); writeVarUInt(Protocol::Server::Data, *out); + /// Send external table name (empty name is the main table) writeStringBinary("", *out); state.block_out->write(block); @@ -732,10 +831,23 @@ void TCPHandler::sendData(const Block & block) } -void TCPHandler::sendException(const Exception & e) +void TCPHandler::sendLogData(const Block & block) +{ + initLogsBlockOutput(block); + + writeVarUInt(Protocol::Server::Log, *out); + /// Send log tag (empty tag is the default tag) + writeStringBinary("", *out); + + state.logs_block_out->write(block); + out->next(); +} + + +void TCPHandler::sendException(const Exception & e, bool with_stack_trace) { writeVarUInt(Protocol::Server::Exception, *out); - writeException(e, *out); + writeException(e, *out, with_stack_trace); out->next(); } @@ -763,6 +875,37 @@ void TCPHandler::sendProgress() } +void TCPHandler::sendLogs() +{ + if (!state.logs_queue) + return; + + MutableColumns logs_columns; + MutableColumns curr_logs_columns; + size_t rows = 0; + + for (; state.logs_queue->tryPop(curr_logs_columns); ++rows) + { + if (rows == 0) + { + logs_columns = std::move(curr_logs_columns); + } + else + { + for (size_t j = 0; j < logs_columns.size(); ++j) + logs_columns[j]->insertRangeFrom(*curr_logs_columns[j], 0, curr_logs_columns[j]->size()); + } + } + + if (rows > 0) + { + Block block = InternalTextLogsQueue::getSampleBlock(); + block.setColumns(std::move(logs_columns)); + sendLogData(block); + } +} + + void TCPHandler::run() { try diff --git a/dbms/src/Server/TCPHandler.h b/dbms/programs/server/TCPHandler.h similarity index 85% rename from dbms/src/Server/TCPHandler.h rename to dbms/programs/server/TCPHandler.h index f53850ee487..098019e86da 100644 --- a/dbms/src/Server/TCPHandler.h +++ b/dbms/programs/server/TCPHandler.h @@ -5,15 +5,16 @@ #include #include #include -#include #include #include +#include #include -#include -#include +#include +#include #include "IServer.h" + namespace CurrentMetrics { extern const Metric TCPConnection; @@ -59,6 +60,12 @@ struct QueryState /// To output progress, the difference after the previous sending of progress. Progress progress; + /// Timeouts setter for current query + std::unique_ptr timeout_setter; + + /// A queue with internal logs that will be passed to client + InternalTextLogsQueuePtr logs_queue; + BlockOutputStreamPtr logs_block_out; void reset() { @@ -82,7 +89,7 @@ public: , connection_context(server.context()) , query_context(server.context()) { - server_display_name = server.config().getString("display_name", getFQDNOrHostName()); + server_display_name = server.config().getString("display_name", getFQDNOrHostName()); } void run(); @@ -94,6 +101,7 @@ private: String client_name; UInt64 client_version_major = 0; UInt64 client_version_minor = 0; + UInt64 client_version_patch = 0; UInt64 client_revision = 0; Context connection_context; @@ -113,8 +121,8 @@ private: QueryState state; CurrentMetrics::Increment metric_increment{CurrentMetrics::TCPConnection}; - - /// It is the name of the server that will be sent to the client. + + /// It is the name of the server that will be sent to the client. String server_display_name; void runImpl(); @@ -135,8 +143,10 @@ private: void sendHello(); void sendData(const Block & block); /// Write a block to the network. - void sendException(const Exception & e); + void sendLogData(const Block & block); + void sendException(const Exception & e, bool with_stack_trace); void sendProgress(); + void sendLogs(); void sendEndOfStream(); void sendProfileInfo(); void sendTotals(); @@ -145,6 +155,7 @@ private: /// Creates state.block_in/block_out for blocks read/write, depending on whether compression is enabled. void initBlockInput(); void initBlockOutput(const Block & block); + void initLogsBlockOutput(const Block & block); bool isQueryCancelled(); diff --git a/dbms/src/Server/TCPHandlerFactory.h b/dbms/programs/server/TCPHandlerFactory.h similarity index 100% rename from dbms/src/Server/TCPHandlerFactory.h rename to dbms/programs/server/TCPHandlerFactory.h diff --git a/dbms/src/Server/clickhouse-server.cpp b/dbms/programs/server/clickhouse-server.cpp similarity index 100% rename from dbms/src/Server/clickhouse-server.cpp rename to dbms/programs/server/clickhouse-server.cpp diff --git a/dbms/programs/server/config.d/listen.xml b/dbms/programs/server/config.d/listen.xml new file mode 100644 index 00000000000..24c64bbb60a --- /dev/null +++ b/dbms/programs/server/config.d/listen.xml @@ -0,0 +1 @@ +0.0.0.0 \ No newline at end of file diff --git a/dbms/src/Server/config.d/log_to_console.xml b/dbms/programs/server/config.d/log_to_console.xml similarity index 100% rename from dbms/src/Server/config.d/log_to_console.xml rename to dbms/programs/server/config.d/log_to_console.xml diff --git a/dbms/src/Server/config.d/more_clusters.xml b/dbms/programs/server/config.d/more_clusters.xml similarity index 100% rename from dbms/src/Server/config.d/more_clusters.xml rename to dbms/programs/server/config.d/more_clusters.xml diff --git a/dbms/src/Server/config.d/path.xml b/dbms/programs/server/config.d/path.xml similarity index 72% rename from dbms/src/Server/config.d/path.xml rename to dbms/programs/server/config.d/path.xml index 14690435d90..14b7deb9de0 100644 --- a/dbms/src/Server/config.d/path.xml +++ b/dbms/programs/server/config.d/path.xml @@ -1,5 +1,6 @@ ./ ./tmp/ + ./user_files/ ./format_schemas/ diff --git a/dbms/programs/server/config.d/zookeeper.xml b/dbms/programs/server/config.d/zookeeper.xml new file mode 100644 index 00000000000..095f4be78c1 --- /dev/null +++ b/dbms/programs/server/config.d/zookeeper.xml @@ -0,0 +1,16 @@ + + + + localhost + 2181 + + + yandex.ru + 2181 + + + 111.0.1.2 + 2181 + + + diff --git a/dbms/programs/server/config.xml b/dbms/programs/server/config.xml new file mode 100644 index 00000000000..514a081eaca --- /dev/null +++ b/dbms/programs/server/config.xml @@ -0,0 +1,390 @@ + + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + + 8123 + 9000 + + + + + + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + + /etc/clickhouse-server/dhparam.pem + none + true + true + sslv2,sslv3 + true + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + + + + + 9009 + + + + + + + + + + + + + + + + + + + + 4096 + 3 + + + 100 + + + + + + 8589934592 + + + 5368709120 + + + + /var/lib/clickhouse/ + + + /var/lib/clickhouse/tmp/ + + + /var/lib/clickhouse/user_files/ + + + users.xml + + + default + + + + + + default + + + + + + + + + false + + + + + + + + localhost + 9000 + + + + + + + localhost + 9000 + + + + + localhost + 9000 + + + + + + + localhost + 9440 + 1 + + + + + + + + + + + + + + + + + 3600 + + + + 3600 + + + 60 + + + + + + + + + + system +
query_log
+ + toYYYYMM(event_date) + + 7500 + + + + + + + + + + + + + + + + + *_dictionary.xml + + + + + + + + + + /clickhouse/task_queue/ddl + + + + + + + + + + + + + + + + click_cost + any + + 0 + 3600 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + + + + /var/lib/clickhouse/format_schemas/ + + + + diff --git a/dbms/src/Server/data/.gitignore b/dbms/programs/server/data/.gitignore similarity index 100% rename from dbms/src/Server/data/.gitignore rename to dbms/programs/server/data/.gitignore diff --git a/dbms/src/Server/data/default/.gitignore b/dbms/programs/server/data/default/.gitignore similarity index 100% rename from dbms/src/Server/data/default/.gitignore rename to dbms/programs/server/data/default/.gitignore diff --git a/dbms/src/Server/metadata/default/.gitignore b/dbms/programs/server/metadata/default/.gitignore similarity index 100% rename from dbms/src/Server/metadata/default/.gitignore rename to dbms/programs/server/metadata/default/.gitignore diff --git a/dbms/src/Server/users.d/allow_only_from_localhost.xml b/dbms/programs/server/users.d/allow_only_from_localhost.xml similarity index 100% rename from dbms/src/Server/users.d/allow_only_from_localhost.xml rename to dbms/programs/server/users.d/allow_only_from_localhost.xml diff --git a/dbms/src/Server/users.d/log_queries.xml b/dbms/programs/server/users.d/log_queries.xml similarity index 100% rename from dbms/src/Server/users.d/log_queries.xml rename to dbms/programs/server/users.d/log_queries.xml diff --git a/dbms/src/Server/users.xml b/dbms/programs/server/users.xml similarity index 97% rename from dbms/src/Server/users.xml rename to dbms/programs/server/users.xml index 41ac6057e9a..979b2d3fc17 100644 --- a/dbms/src/Server/users.xml +++ b/dbms/programs/server/users.xml @@ -55,7 +55,8 @@ 127.0.0.1 Each element of list has one of the following forms: - IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 2a02:6b8::3 or 2a02:6b8::3/64. + IP-address or network mask. Examples: 213.180.204.3 or 10.0.0.1/8 or 10.0.0.1/255.255.255.0 + 2a02:6b8::3 or 2a02:6b8::3/64 or 2a02:6b8::3/ffff:ffff:ffff:ffff::. Hostname. Example: server01.yandex.ru. To check access, DNS query is performed, and all received addresses compared to peer address. Regular expression for host names. Example, ^server\d\d-\d\d-\d\.yandex\.ru$ diff --git a/dbms/scripts/gen-bias-data.py b/dbms/scripts/gen-bias-data.py index 7edc9948e76..034cfcca7dd 100755 --- a/dbms/scripts/gen-bias-data.py +++ b/dbms/scripts/gen-bias-data.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3.4 +#!/usr/bin/python3 # -*- coding: utf-8 -*- import sys diff --git a/dbms/scripts/gen_benchmark_data/README.md b/dbms/scripts/gen_benchmark_data/README.md new file mode 100644 index 00000000000..c7f8bd30391 --- /dev/null +++ b/dbms/scripts/gen_benchmark_data/README.md @@ -0,0 +1 @@ +Hits table generator based on LSTM neural network trained on real hits. You need to have weights for model or train model on real hits to generate data. diff --git a/dbms/scripts/gen_benchmark_data/generate.py b/dbms/scripts/gen_benchmark_data/generate.py new file mode 100644 index 00000000000..b54651fe1b1 --- /dev/null +++ b/dbms/scripts/gen_benchmark_data/generate.py @@ -0,0 +1,22 @@ +import argparse + +from model import Model +parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('-n', type=int, default=100000, + help='number of objects to generate') +parser.add_argument('--output_file', type=str, default='out.tsv', + help='output file name') +parser.add_argument('--weights_path', type=str, + help='path to weights') + + +args = parser.parse_args() + +if __name__ == '__main__': + if not args.weights_path: + raise Exception('please specify path to model weights with --weights_path') + + gen = Model() + gen.generate(args.n, args.output_file, args.weights_path) + diff --git a/dbms/scripts/gen_benchmark_data/model.py b/dbms/scripts/gen_benchmark_data/model.py new file mode 100644 index 00000000000..3e2ec9c4942 --- /dev/null +++ b/dbms/scripts/gen_benchmark_data/model.py @@ -0,0 +1,147 @@ +import numpy as np +import os +import pickle +import tensorflow as tf + +from random import sample +from keras.layers import Dense, Embedding +from tqdm import tqdm + +RNN_NUM_UNITS = 256 +EMB_SIZE = 32 +MAX_LENGTH = 1049 + + +with open('tokens', 'rb') as f: + tokens = pickle.load(f) +n_tokens = len(tokens) + +token_to_id = {c: i for i, c in enumerate(tokens)} + + +def to_matrix(objects, max_len=None, pad=0, dtype='int32'): + max_len = max_len or max(map(len, objects)) + matrix = np.zeros([len(objects), max_len], dtype) + pad + + for i in range(len(objects)): + name_ix = list(map(token_to_id.get, objects[i])) + matrix[i, :len(name_ix)] = name_ix + return matrix.T + + +class Model: + def __init__(self, learning_rate=0.0001): + # an embedding layer that converts character ids into embeddings + self.embed_x = Embedding(n_tokens, EMB_SIZE) + get_h_next = Dense(1024, activation='relu') + # a dense layer that maps current hidden state + # to probabilities of characters [h_t+1]->P(x_t+1|h_t+1) + self.get_probas = Dense(n_tokens, activation='softmax') + + self.input_sequence = tf.placeholder('int32', (MAX_LENGTH, None)) + batch_size = tf.shape(self.input_sequence)[1] + + self.gru_cell_first = tf.nn.rnn_cell.GRUCell(RNN_NUM_UNITS) + self.lstm_cell_second = tf.nn.rnn_cell.LSTMCell(RNN_NUM_UNITS) + + h_prev_first = self.gru_cell_first.zero_state(batch_size, dtype=tf.float32) + h_prev_second = tf.nn.rnn_cell.LSTMStateTuple( + tf.zeros([batch_size, RNN_NUM_UNITS]), # initial cell state, + tf.zeros([batch_size, RNN_NUM_UNITS]) # initial hidden state + ) + + predicted_probas = [] + for t in range(MAX_LENGTH): + x_t = self.input_sequence[t] + # convert character id into embedding + x_t_emb = self.embed_x(tf.reshape(x_t, [-1, 1]))[:, 0] + + out_next_first, h_next_first = self.gru_cell_first(x_t_emb, h_prev_first) + h_prev_first = h_next_first + + out_next_second, h_next_second = self.lstm_cell_second(out_next_first, h_prev_second) + h_prev_second = h_next_second + + probas_next = self.get_probas(out_next_second) + predicted_probas.append(probas_next) + + predicted_probas = tf.stack(predicted_probas) + + predictions_matrix = tf.reshape(predicted_probas[:-1], [-1, len(tokens)]) + answers_matrix = tf.one_hot(tf.reshape(self.input_sequence[1:], [-1]), n_tokens) + + self.loss = tf.reduce_mean(tf.reduce_sum( + -answers_matrix * tf.log(tf.clip_by_value(predictions_matrix, 1e-7, 1.0)), + reduction_indices=[1] + )) + optimizer = tf.train.AdamOptimizer(learning_rate) + gvs = optimizer.compute_gradients(self.loss) + capped_gvs = [(gr if gr is None else tf.clip_by_value(gr, -1., 1.), var) for gr, var in gvs] + self.optimize = optimizer.apply_gradients(capped_gvs) + + self.sess = tf.Session() + self.sess.run(tf.global_variables_initializer()) + self.saver = tf.train.Saver() + + def train(self, train_data_path, save_dir, num_iters, batch_size=64, restore_from=False): + history = [] + if restore_from: + with open(restore_from + '_history') as f: + history = pickle.load(f) + self.saver.restore(self.sess, restore_from) + with open(train_data_path, 'r') as f: + train_data = f.readlines() + + train_data = filter(lambda a: len(a) < MAX_LENGTH, train_data) + + for i in tqdm(range(num_iters)): + batch = to_matrix( + map(lambda a: '\n' + a.rstrip('\n'), sample(train_data, batch_size)), + max_len=MAX_LENGTH + ) + loss_i, _ = self.sess.run([self.loss, self.optimize], {self.input_sequence: batch}) + history.append(loss_i) + if len(history) % 2000 == 0: + self.saver.save(self.sess, os.path.join(save_dir, '{}_iters'.format(len(history)))) + self.saver.save(self.sess, os.path.join(save_dir, '{}_iters'.format(len(history)))) + with open(os.path.join(save_dir, '{}_iters_history'.format(len(history)))) as f: + pickle.dump(history, f) + + def generate(self, num_objects, output_file, weights_path): + self.saver.restore(self.sess, weights_path) + batch_size = num_objects + x_t = tf.placeholder('int32', (None, batch_size)) + h_t_first = tf.Variable(tf.zeros([batch_size, RNN_NUM_UNITS])) + h_t_second = tf.nn.rnn_cell.LSTMStateTuple( + tf.Variable(tf.zeros([batch_size, RNN_NUM_UNITS])), + tf.Variable(tf.zeros([batch_size, RNN_NUM_UNITS])) + ) + + x_t_emb = self.embed_x(tf.reshape(x_t, [-1, 1]))[:, 0] + first_out_next, next_h_first = self.gru_cell_first(x_t_emb, h_t_first) + second_out_next, next_h_second = self.lstm_cell_second(first_out_next, h_t_second) + next_probs = self.get_probas(second_out_next) + + x_sequence = np.zeros(shape=(1, batch_size), dtype=int) + token_to_id['\n'] + self.sess.run( + [tf.assign(h_t_first, h_t_first.initial_value), + tf.assign(h_t_second[0], h_t_second[0].initial_value), + tf.assign(h_t_second[1], h_t_second[1].initial_value)] + ) + + for i in tqdm(range(MAX_LENGTH - 1)): + x_probs, _, _, _ = self.sess.run( + [next_probs, + tf.assign(h_t_second[0], next_h_second[0]), + tf.assign(h_t_second[1], next_h_second[1]), + tf.assign(h_t_first, next_h_first)], + {x_t: [x_sequence[-1, :]]} + ) + + next_char = [np.random.choice(n_tokens, p=x_probs[i]) for i in range(batch_size)] + if sum(next_char) == 0: + break + x_sequence = np.append(x_sequence, [next_char], axis=0) + + with open(output_file, 'w') as f: + f.writelines([''.join([tokens[ix] for ix in x_sequence.T[k]]) + '\n' for k in range(batch_size)]) diff --git a/dbms/scripts/gen_benchmark_data/requirements.txt b/dbms/scripts/gen_benchmark_data/requirements.txt new file mode 100644 index 00000000000..b02bc51fee1 --- /dev/null +++ b/dbms/scripts/gen_benchmark_data/requirements.txt @@ -0,0 +1,3 @@ +Keras==2.0.6 +numpy +tensorflow-gpu==1.4.0 \ No newline at end of file diff --git a/dbms/scripts/gen_benchmark_data/tokens b/dbms/scripts/gen_benchmark_data/tokens new file mode 100644 index 00000000000..f80b0dd4208 --- /dev/null +++ b/dbms/scripts/gen_benchmark_data/tokens @@ -0,0 +1,506 @@ +(lp0 +S'\x83' +p1 +aS'\x04' +p2 +aS'\x87' +p3 +aS'\x8b' +p4 +aS'\x8f' +p5 +aS'\x10' +p6 +aS'\x93' +p7 +aS'\x14' +p8 +aS'\x97' +p9 +aS'\x18' +p10 +aS'\x9b' +p11 +aS'\x1c' +p12 +aS'\x9f' +p13 +aS' ' +p14 +aS'\xa3' +p15 +aS'$' +p16 +aS'\xa7' +p17 +aS'(' +p18 +aS'\xab' +p19 +aS',' +p20 +aS'\xaf' +p21 +aS'0' +p22 +aS'\xb3' +p23 +aS'4' +p24 +aS'\xb7' +p25 +aS'8' +p26 +aS'\xbb' +p27 +aS'<' +p28 +aS'\xbf' +p29 +aS'@' +p30 +aS'\xc3' +p31 +aS'D' +p32 +aS'\xc7' +p33 +aS'H' +p34 +aS'\xcb' +p35 +aS'L' +p36 +aS'\xcf' +p37 +aS'P' +p38 +aS'\xd3' +p39 +aS'T' +p40 +aS'\xd7' +p41 +aS'X' +p42 +aS'\xdb' +p43 +aS'\\' +p44 +aS'\xdf' +p45 +aS'`' +p46 +aS'\xe3' +p47 +aS'd' +p48 +aS'\xe7' +p49 +aS'h' +p50 +aS'\xeb' +p51 +aS'l' +p52 +aS'\xef' +p53 +aS'p' +p54 +aS'\xf3' +p55 +aS't' +p56 +aS'\xf7' +p57 +aS'x' +p58 +aS'\xfb' +p59 +aS'|' +p60 +aS'\xff' +p61 +aS'\x80' +p62 +aS'\x03' +p63 +aS'\x84' +p64 +aS'\x07' +p65 +aS'\x88' +p66 +aS'\x0b' +p67 +aS'\x8c' +p68 +aS'\x0f' +p69 +aS'\x90' +p70 +aS'\x13' +p71 +aS'\x94' +p72 +aS'\x17' +p73 +aS'\x98' +p74 +aS'\x1b' +p75 +aS'\x9c' +p76 +aS'\x1f' +p77 +aS'\xa0' +p78 +aS'#' +p79 +aS'\xa4' +p80 +aS"'" +p81 +aS'\xa8' +p82 +aS'+' +p83 +aS'\xac' +p84 +aS'/' +p85 +aS'\xb0' +p86 +aS'3' +p87 +aS'\xb4' +p88 +aS'7' +p89 +aS'\xb8' +p90 +aS';' +p91 +aS'\xbc' +p92 +aS'?' +p93 +aS'\xc0' +p94 +aS'C' +p95 +aS'\xc4' +p96 +aS'G' +p97 +aS'\xc8' +p98 +aS'K' +p99 +aS'\xcc' +p100 +aS'O' +p101 +aS'\xd0' +p102 +aS'S' +p103 +aS'\xd4' +p104 +aS'W' +p105 +aS'\xd8' +p106 +aS'[' +p107 +aS'\xdc' +p108 +aS'_' +p109 +aS'\xe0' +p110 +aS'c' +p111 +aS'\xe4' +p112 +aS'g' +p113 +aS'\xe8' +p114 +aS'k' +p115 +aS'\xec' +p116 +aS'o' +p117 +aS'\xf0' +p118 +aS's' +p119 +aS'\xf4' +p120 +aS'w' +p121 +aS'\xf8' +p122 +aS'{' +p123 +aS'\xfc' +p124 +aS'\x7f' +p125 +aS'\x81' +p126 +aS'\x02' +p127 +aS'\x85' +p128 +aS'\x06' +p129 +aS'\x89' +p130 +aS'\n' +p131 +aS'\x8d' +p132 +aS'\x0e' +p133 +aS'\x91' +p134 +aS'\x12' +p135 +aS'\x95' +p136 +aS'\x16' +p137 +aS'\x99' +p138 +aS'\x1a' +p139 +aS'\x9d' +p140 +aS'\x1e' +p141 +aS'\xa1' +p142 +aS'"' +p143 +aS'\xa5' +p144 +aS'&' +p145 +aS'\xa9' +p146 +aS'*' +p147 +aS'\xad' +p148 +aS'.' +p149 +aS'\xb1' +p150 +aS'2' +p151 +aS'\xb5' +p152 +aS'6' +p153 +aS'\xb9' +p154 +aS':' +p155 +aS'\xbd' +p156 +aS'>' +p157 +aS'\xc1' +p158 +aS'B' +p159 +aS'\xc5' +p160 +aS'F' +p161 +aS'\xc9' +p162 +aS'J' +p163 +aS'\xcd' +p164 +aS'N' +p165 +aS'\xd1' +p166 +aS'R' +p167 +aS'\xd5' +p168 +aS'V' +p169 +aS'\xd9' +p170 +aS'Z' +p171 +aS'\xdd' +p172 +aS'^' +p173 +aS'\xe1' +p174 +aS'b' +p175 +aS'\xe5' +p176 +aS'f' +p177 +aS'\xe9' +p178 +aS'j' +p179 +aS'\xed' +p180 +aS'n' +p181 +aS'\xf1' +p182 +aS'r' +p183 +aS'\xf5' +p184 +aS'v' +p185 +aS'\xf9' +p186 +aS'z' +p187 +aS'\xfd' +p188 +aS'~' +p189 +aS'\x01' +p190 +aS'\x82' +p191 +aS'\x05' +p192 +aS'\x86' +p193 +aS'\t' +p194 +aS'\x8a' +p195 +aS'\x8e' +p196 +aS'\x11' +p197 +aS'\x92' +p198 +aS'\x15' +p199 +aS'\x96' +p200 +aS'\x19' +p201 +aS'\x9a' +p202 +aS'\x1d' +p203 +aS'\x9e' +p204 +aS'!' +p205 +aS'\xa2' +p206 +aS'%' +p207 +aS'\xa6' +p208 +aS')' +p209 +aS'\xaa' +p210 +aS'-' +p211 +aS'\xae' +p212 +aS'1' +p213 +aS'\xb2' +p214 +aS'5' +p215 +aS'\xb6' +p216 +aS'9' +p217 +aS'\xba' +p218 +aS'=' +p219 +aS'\xbe' +p220 +aS'A' +p221 +aS'\xc2' +p222 +aS'E' +p223 +aS'\xc6' +p224 +aS'I' +p225 +aS'\xca' +p226 +aS'M' +p227 +aS'\xce' +p228 +aS'Q' +p229 +aS'\xd2' +p230 +aS'U' +p231 +aS'\xd6' +p232 +aS'Y' +p233 +aS'\xda' +p234 +aS']' +p235 +aS'\xde' +p236 +aS'a' +p237 +aS'\xe2' +p238 +aS'e' +p239 +aS'\xe6' +p240 +aS'i' +p241 +aS'\xea' +p242 +aS'm' +p243 +aS'\xee' +p244 +aS'q' +p245 +aS'\xf2' +p246 +aS'u' +p247 +aS'\xf6' +p248 +aS'y' +p249 +aS'\xfa' +p250 +aS'}' +p251 +aS'\xfe' +p252 +a. \ No newline at end of file diff --git a/dbms/scripts/gen_benchmark_data/train.py b/dbms/scripts/gen_benchmark_data/train.py new file mode 100644 index 00000000000..fd93805f50e --- /dev/null +++ b/dbms/scripts/gen_benchmark_data/train.py @@ -0,0 +1,26 @@ +import argparse + +from model import Model +parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('--n_iter', type=int, default=10000, + help='number of iterations') +parser.add_argument('--save_dir', type=str, default='save', + help='dir for saving weights') +parser.add_argument('--data_path', type=str, + help='path to train data') +parser.add_argument('--learning_rate', type=int, default=0.0001, + help='learning rate') +parser.add_argument('--batch_size', type=int, default=64, + help='batch size') +parser.add_argument('--restore_from', type=str, + help='path to train saved weights') + +args = parser.parse_args() + +if __name__ == '__main__': + if not args.data_path: + raise Exception('please specify path to train data with --data_path') + + gen = Model(args.learning_rate) + gen.train(args.data_path, args.save_dir, args.n_iter, args.batch_size, args.restore_from) diff --git a/dbms/scripts/merge_algorithm/add_parts.sh b/dbms/scripts/merge_algorithm/add_parts.sh index 09f7e66d639..cf1a5ee8fc2 100644 --- a/dbms/scripts/merge_algorithm/add_parts.sh +++ b/dbms/scripts/merge_algorithm/add_parts.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash for (( i = 0; i < 1000; i++ )); do if (( RANDOM % 10 )); then diff --git a/dbms/scripts/test_uniq_functions.sh b/dbms/scripts/test_uniq_functions.sh index a71258cd93d..9a4b6f20433 100755 --- a/dbms/scripts/test_uniq_functions.sh +++ b/dbms/scripts/test_uniq_functions.sh @@ -1,4 +1,5 @@ -#!/bin/bash +#!/usr/bin/env bash + for ((p = 2; p <= 10; p++)) do for ((i = 1; i <= 9; i++)) diff --git a/dbms/src/AggregateFunctions/AggregateFunctionArray.cpp b/dbms/src/AggregateFunctions/AggregateFunctionArray.cpp index c1238af01de..995759d0082 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionArray.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionArray.cpp @@ -1,6 +1,6 @@ #include #include - +#include namespace DB { @@ -14,10 +14,13 @@ namespace ErrorCodes class AggregateFunctionCombinatorArray final : public IAggregateFunctionCombinator { public: - String getName() const override { return "Array"; }; + String getName() const override { return "Array"; } DataTypes transformArguments(const DataTypes & arguments) const override { + if (0 == arguments.size()) + throw Exception("-Array aggregate functions require at least one argument", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + DataTypes nested_arguments; for (const auto & type : arguments) { diff --git a/dbms/src/AggregateFunctions/AggregateFunctionArray.h b/dbms/src/AggregateFunctions/AggregateFunctionArray.h index 004e860a394..ccf5f787309 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionArray.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionArray.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include #include @@ -32,7 +31,7 @@ public: : nested_func(nested_), num_arguments(arguments.size()) { for (const auto & type : arguments) - if (!typeid_cast(type.get())) + if (!isArray(type)) throw Exception("All arguments for aggregate function " + getName() + " must be arrays", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } @@ -71,6 +70,11 @@ public: return nested_func->alignOfData(); } + bool isState() const override + { + return nested_func->isState(); + } + void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena * arena) const override { const IColumn * nested[num_arguments]; diff --git a/dbms/src/AggregateFunctions/AggregateFunctionAvg.cpp b/dbms/src/AggregateFunctions/AggregateFunctionAvg.cpp index 31b65fe7404..565f1f0c335 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionAvg.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionAvg.cpp @@ -9,16 +9,31 @@ namespace DB namespace { +template +struct Avg +{ + using FieldType = std::conditional_t, Decimal128, NearestFieldType>; + using Function = AggregateFunctionAvg>; +}; + +template +using AggregateFuncAvg = typename Avg::Function; + AggregateFunctionPtr createAggregateFunctionAvg(const std::string & name, const DataTypes & argument_types, const Array & parameters) { assertNoParameters(name, parameters); assertUnary(name, argument_types); - AggregateFunctionPtr res(createWithNumericType(*argument_types[0])); + AggregateFunctionPtr res; + DataTypePtr data_type = argument_types[0]; + if (isDecimal(data_type)) + res.reset(createWithDecimalType(*data_type, *data_type)); + else + res.reset(createWithNumericType(*data_type)); if (!res) - throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - + throw Exception("Illegal type " + argument_types[0]->getName() + " of argument for aggregate function " + name, + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return res; } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionAvg.h b/dbms/src/AggregateFunctions/AggregateFunctionAvg.h index 84288cecb86..9cc2d8e26ed 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionAvg.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionAvg.h @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -12,30 +13,65 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} template struct AggregateFunctionAvgData { T sum = 0; UInt64 count = 0; + + template + ResultT result() const + { + if constexpr (std::is_floating_point_v) + if constexpr (std::numeric_limits::is_iec559) + return static_cast(sum) / count; /// allow division by zero + + if (!count) + throw Exception("AggregateFunctionAvg with zero values", ErrorCodes::LOGICAL_ERROR); + return static_cast(sum) / count; + } }; /// Calculates arithmetic mean of numbers. -template -class AggregateFunctionAvg final : public IAggregateFunctionDataHelper::Type>, AggregateFunctionAvg> +template +class AggregateFunctionAvg final : public IAggregateFunctionDataHelper> { public: + using ResultType = std::conditional_t, Decimal128, Float64>; + using ResultDataType = std::conditional_t, DataTypeDecimal, DataTypeNumber>; + using ColVecType = std::conditional_t, ColumnDecimal, ColumnVector>; + using ColVecResult = std::conditional_t, ColumnDecimal, ColumnVector>; + + /// ctor for native types + AggregateFunctionAvg() + : scale(0) + {} + + /// ctor for Decimals + AggregateFunctionAvg(const IDataType & data_type) + : scale(getDecimalScale(data_type)) + {} + String getName() const override { return "avg"; } DataTypePtr getReturnType() const override { - return std::make_shared(); + if constexpr (IsDecimalNumber) + return std::make_shared(ResultDataType::maxPrecision(), scale); + else + return std::make_shared(); } void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override { - this->data(place).sum += static_cast &>(*columns[0]).getData()[row_num]; + const auto & column = static_cast(*columns[0]); + this->data(place).sum += column.getData()[row_num]; ++this->data(place).count; } @@ -59,11 +95,14 @@ public: void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override { - static_cast(to).getData().push_back( - static_cast(this->data(place).sum) / this->data(place).count); + auto & column = static_cast(to); + column.getData().push_back(this->data(place).template result()); } const char * getHeaderFilePath() const override { return __FILE__; } + +private: + UInt32 scale; }; diff --git a/dbms/src/AggregateFunctions/AggregateFunctionBitwise.cpp b/dbms/src/AggregateFunctions/AggregateFunctionBitwise.cpp index 762baf2451b..8c188bcbb8e 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionBitwise.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionBitwise.cpp @@ -38,9 +38,9 @@ void registerAggregateFunctionsBitwise(AggregateFunctionFactory & factory) factory.registerFunction("groupBitXor", createAggregateFunctionBitwise); /// Aliases for compatibility with MySQL. - factory.registerFunction("BIT_OR", createAggregateFunctionBitwise, AggregateFunctionFactory::CaseInsensitive); - factory.registerFunction("BIT_AND", createAggregateFunctionBitwise, AggregateFunctionFactory::CaseInsensitive); - factory.registerFunction("BIT_XOR", createAggregateFunctionBitwise, AggregateFunctionFactory::CaseInsensitive); + factory.registerAlias("BIT_OR", "groupBitOr", AggregateFunctionFactory::CaseInsensitive); + factory.registerAlias("BIT_AND", "groupBitAnd", AggregateFunctionFactory::CaseInsensitive); + factory.registerAlias("BIT_XOR", "groupBitXor", AggregateFunctionFactory::CaseInsensitive); } } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionCombinatorFactory.h b/dbms/src/AggregateFunctions/AggregateFunctionCombinatorFactory.h index 4c70cc6c068..579951cecb1 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionCombinatorFactory.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionCombinatorFactory.h @@ -15,6 +15,10 @@ namespace DB */ class AggregateFunctionCombinatorFactory final: public ext::singleton { +private: + using Dict = std::unordered_map; + Dict dict; + public: /// Not thread safe. You must register before using tryGet. void registerCombinator(const AggregateFunctionCombinatorPtr & value); @@ -22,8 +26,10 @@ public: /// Example: if the name is 'avgIf', it will return combinator -If. AggregateFunctionCombinatorPtr tryFindSuffix(const std::string & name) const; -private: - std::unordered_map dict; + const Dict & getAllAggregateFunctionCombinators() const + { + return dict; + } }; } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp b/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp index eca854a031b..7c1cb537c7c 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp @@ -13,6 +13,7 @@ #include #include +#include namespace DB @@ -41,6 +42,20 @@ void AggregateFunctionFactory::registerFunction(const String & name, Creator cre ErrorCodes::LOGICAL_ERROR); } +static DataTypes convertLowCardinalityTypesToNested(const DataTypes & types) +{ + DataTypes res_types; + res_types.reserve(types.size()); + for (const auto & type : types) + { + if (auto * low_cardinality_type = typeid_cast(type.get())) + res_types.push_back(low_cardinality_type->getDictionaryType()); + else + res_types.push_back(type); + } + + return res_types; +} AggregateFunctionPtr AggregateFunctionFactory::get( const String & name, @@ -48,6 +63,8 @@ AggregateFunctionPtr AggregateFunctionFactory::get( const Array & parameters, int recursion_level) const { + auto type_without_low_cardinality = convertLowCardinalityTypesToNested(argument_types); + /// If one of types is Nullable, we apply aggregate function combinator "Null". if (std::any_of(argument_types.begin(), argument_types.end(), @@ -57,7 +74,7 @@ AggregateFunctionPtr AggregateFunctionFactory::get( if (!combinator) throw Exception("Logical error: cannot find aggregate function combinator to apply a function to Nullable arguments.", ErrorCodes::LOGICAL_ERROR); - DataTypes nested_types = combinator->transformArguments(argument_types); + DataTypes nested_types = combinator->transformArguments(type_without_low_cardinality); AggregateFunctionPtr nested_function; @@ -70,7 +87,7 @@ AggregateFunctionPtr AggregateFunctionFactory::get( return combinator->transformAggregateFunction(nested_function, argument_types, parameters); } - auto res = getImpl(name, argument_types, parameters, recursion_level); + auto res = getImpl(name, type_without_low_cardinality, parameters, recursion_level); if (!res) throw Exception("Logical error: AggregateFunctionFactory returned nullptr", ErrorCodes::LOGICAL_ERROR); return res; @@ -78,11 +95,12 @@ AggregateFunctionPtr AggregateFunctionFactory::get( AggregateFunctionPtr AggregateFunctionFactory::getImpl( - const String & name, + const String & name_param, const DataTypes & argument_types, const Array & parameters, int recursion_level) const { + String name = getAliasToOrName(name_param); /// Find by exact match. auto it = aggregate_functions.find(name); if (it != aggregate_functions.end()) @@ -103,12 +121,12 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl( if (AggregateFunctionCombinatorPtr combinator = AggregateFunctionCombinatorFactory::instance().tryFindSuffix(name)) { - if (combinator->getName() == "Null") - throw Exception("Aggregate function combinator 'Null' is only for internal usage", ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION); + if (combinator->isForInternalUsageOnly()) + throw Exception("Aggregate function combinator '" + combinator->getName() + "' is only for internal usage", ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION); String nested_name = name.substr(0, name.size() - combinator->getName().size()); DataTypes nested_types = combinator->transformArguments(argument_types); - AggregateFunctionPtr nested_function = getImpl(nested_name, nested_types, parameters, recursion_level + 1); + AggregateFunctionPtr nested_function = get(nested_name, nested_types, parameters, recursion_level + 1); return combinator->transformAggregateFunction(nested_function, argument_types, parameters); } @@ -126,10 +144,11 @@ AggregateFunctionPtr AggregateFunctionFactory::tryGet(const String & name, const bool AggregateFunctionFactory::isAggregateFunctionName(const String & name, int recursion_level) const { - if (aggregate_functions.count(name)) + if (aggregate_functions.count(name) || isAlias(name)) return true; - if (recursion_level == 0 && case_insensitive_aggregate_functions.count(Poco::toLower(name))) + String name_lowercase = Poco::toLower(name); + if (recursion_level == 0 && (case_insensitive_aggregate_functions.count(name_lowercase) || isAlias(name_lowercase))) return true; if (AggregateFunctionCombinatorPtr combinator = AggregateFunctionCombinatorFactory::instance().tryFindSuffix(name)) diff --git a/dbms/src/AggregateFunctions/AggregateFunctionFactory.h b/dbms/src/AggregateFunctions/AggregateFunctionFactory.h index bc36e76c11f..92598e52509 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionFactory.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionFactory.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include @@ -20,27 +21,18 @@ class IDataType; using DataTypePtr = std::shared_ptr; using DataTypes = std::vector; +/** Creator have arguments: name of aggregate function, types of arguments, values of parameters. + * Parameters are for "parametric" aggregate functions. + * For example, in quantileWeighted(0.9)(x, weight), 0.9 is "parameter" and x, weight are "arguments". + */ +using AggregateFunctionCreator = std::function; + /** Creates an aggregate function by name. */ -class AggregateFunctionFactory final : public ext::singleton +class AggregateFunctionFactory final : public ext::singleton, public IFactoryWithAliases { - friend class StorageSystemFunctions; - public: - /** Creator have arguments: name of aggregate function, types of arguments, values of parameters. - * Parameters are for "parametric" aggregate functions. - * For example, in quantileWeighted(0.9)(x, weight), 0.9 is "parameter" and x, weight are "arguments". - */ - using Creator = std::function; - - /// For compatibility with SQL, it's possible to specify that certain aggregate function name is case insensitive. - enum CaseSensitiveness - { - CaseSensitive, - CaseInsensitive - }; - /// Register a function by its name. /// No locking, you must register all functions before usage of get. void registerFunction( @@ -77,6 +69,13 @@ private: /// Case insensitive aggregate functions will be additionally added here with lowercased name. AggregateFunctions case_insensitive_aggregate_functions; + + const AggregateFunctions & getCreatorMap() const override { return aggregate_functions; } + + const AggregateFunctions & getCaseInsensitiveCreatorMap() const override { return case_insensitive_aggregate_functions; } + + String getFactoryName() const override { return "AggregateFunctionFactory"; } + }; } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionForEach.cpp b/dbms/src/AggregateFunctions/AggregateFunctionForEach.cpp index 7eade008614..aa5d78b0a95 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionForEach.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionForEach.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB @@ -13,7 +14,7 @@ namespace ErrorCodes class AggregateFunctionCombinatorForEach final : public IAggregateFunctionCombinator { public: - String getName() const override { return "ForEach"; }; + String getName() const override { return "ForEach"; } DataTypes transformArguments(const DataTypes & arguments) const override { diff --git a/dbms/src/AggregateFunctions/AggregateFunctionForEach.h b/dbms/src/AggregateFunctions/AggregateFunctionForEach.h index 0c39f4c58f1..ddd94f8c918 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionForEach.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionForEach.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include #include @@ -59,10 +58,11 @@ private: size_t old_size = state.dynamic_array_size; if (old_size < new_size) { - state.array_of_aggregate_datas = arena.realloc( + state.array_of_aggregate_datas = arena.alignedRealloc( state.array_of_aggregate_datas, old_size * nested_size_of_data, - new_size * nested_size_of_data); + new_size * nested_size_of_data, + nested_func->alignOfData()); size_t i = old_size; char * nested_state = state.array_of_aggregate_datas + i * nested_size_of_data; @@ -105,7 +105,7 @@ public: throw Exception("Aggregate function " + getName() + " require at least one argument", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); for (const auto & type : arguments) - if (!typeid_cast(type.get())) + if (!isArray(type)) throw Exception("All arguments for aggregate function " + getName() + " must be arrays", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } diff --git a/dbms/src/AggregateFunctions/AggregateFunctionGroupArray.cpp b/dbms/src/AggregateFunctions/AggregateFunctionGroupArray.cpp index 59ba79b3449..51bc04ea39c 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionGroupArray.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionGroupArray.cpp @@ -20,8 +20,9 @@ namespace template