Merge branch 'master' into query-poor-mans-profiler

2024-09-20 08:40:50 +00:00 · 2019-02-03 10:06:05 +00:00 · 2019-02-03 10:06:05 +00:00 · 89582cfef2
commit 89582cfef2
parent df3db1bff2 91d0d403a2
1414 changed files with 56172 additions and 17375 deletions
--- a/.gitignore
+++ b/.gitignore
@ -180,18 +180,10 @@ utils/zookeeper-create-entry-to-download-part/zookeeper-create-entry-to-download
 utils/zookeeper-dump-tree/zookeeper-dump-tree
 utils/zookeeper-remove-by-list/zookeeper-remove-by-list
 dbms/src/Storages/tests/remove_symlink_directory
-debian/control
-debian/copyright
-debian/tmp/
 libs/libcommon/src/tests/json_test
 utils/compressor/zstd_test
 utils/wikistat-loader/wikistat-loader
 dbms/src/Common/tests/pod_array
-debian/clickhouse-benchmark/
-debian/clickhouse-client/
-debian/clickhouse-server-base/
-debian/clickhouse-server-common/
-debian/files

 dbms/src/Server/data/*
 dbms/src/Server/metadata/*
@ -210,9 +202,6 @@ vgcore*
 *.changes
 build-stamp
 configure-stamp
-debian/*.debhelper.log
-debian/*.debhelper
-debian/*.substvars

 *.bin
 *.mrk
@ -251,3 +240,6 @@ website/package-lock.json

 # cquery cache
 /.cquery-cache
+
+# ccls cache
+/.ccls-cache
--- a/.gitmodules
+++ b/.gitmodules
@ -36,7 +36,7 @@
 	url = https://github.com/ClickHouse-Extras/llvm
 [submodule "contrib/mariadb-connector-c"]
 	path = contrib/mariadb-connector-c
-	url = https://github.com/MariaDB/mariadb-connector-c.git
+	url = https://github.com/ClickHouse-Extras/mariadb-connector-c.git
 [submodule "contrib/jemalloc"]
 	path = contrib/jemalloc
 	url = https://github.com/jemalloc/jemalloc.git
@ -61,3 +61,9 @@
 [submodule "contrib/libgsasl"]
 	path = contrib/libgsasl
 	url = https://github.com/ClickHouse-Extras/libgsasl.git
+[submodule "contrib/cppkafka"]
+	path = contrib/cppkafka
+	url = https://github.com/mfontanini/cppkafka.git
+[submodule "contrib/pdqsort"]
+	path = contrib/pdqsort
+	url = https://github.com/orlp/pdqsort
--- a/.travis.yml.bak
+++ b/.travis.yml.bak
@ -1,50 +0,0 @@
-language: generic
-
-matrix:
-  fast_finish: true
-  include:
-    # We need to have gcc7 headers to compile c++17 code on clang
-#    - os: linux
-#
-#      cache:
-#        ccache: true
-#        timeout: 1000
-#        directories:
-#        - /home/travis/.ccache
-#
-#      addons:
-#        apt:
-#          update: true
-#          sources:
-#            - ubuntu-toolchain-r-test
-#            - llvm-toolchain-trusty-5.0
-#          packages: [ ninja-build, g++-7, clang-5.0, lld-5.0, libicu-dev, libreadline-dev, libmysqlclient-dev, unixodbc-dev, libltdl-dev, libssl-dev, libboost-dev, zlib1g-dev, libdouble-conversion-dev, libsparsehash-dev, librdkafka-dev, libcapnp-dev, libsparsehash-dev, libgoogle-perftools-dev, bash, expect, python, python-lxml, python-termcolor, curl, perl, sudo, openssl]
-#
-#      env:
-#        - MATRIX_EVAL="export CC=clang-5.0 CXX=clang++-5.0"
-#
-#      script:
-#        - utils/travis/normal.sh
-
-    - os: linux
-
-      sudo: required
-
-      cache:
-        timeout: 1000
-        directories:
-        - /var/cache/pbuilder/ccache
-
-      addons:
-        apt:
-          update: true
-          packages: [ pbuilder, fakeroot, debhelper ]
-
-      script:
-        - utils/travis/pbuilder.sh
-
-  allow_failures:
-    - os: osx
-
-before_script:
-    - eval "${MATRIX_EVAL}"
--- a/CHANGELOG.draft.md
+++ b/CHANGELOG.draft.md
@ -1 +0,0 @@
-
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -1,3 +1,114 @@
+## ClickHouse release 19.1.6, 2019-01-24
+
+### New Features
+
+* Custom per column compression codecs for tables. [#3899](https://github.com/yandex/ClickHouse/pull/3899) [#4111](https://github.com/yandex/ClickHouse/pull/4111) ([alesapin](https://github.com/alesapin), [Winter Zhang](https://github.com/zhang2014), [Anatoly](https://github.com/Sindbag))
+* Added compression codec `Delta`. [#4052](https://github.com/yandex/ClickHouse/pull/4052) ([alesapin](https://github.com/alesapin))
+* Allow to `ALTER` compression codecs. [#4054](https://github.com/yandex/ClickHouse/pull/4054) ([alesapin](https://github.com/alesapin))
+* Added functions `left`, `right`, `trim`, `ltrim`, `rtrim`, `timestampadd`, `timestampsub` for SQL standard compatibility. [#3826](https://github.com/yandex/ClickHouse/pull/3826) ([Ivan Blinkov](https://github.com/blinkov))
+* Support for write in `HDFS` tables and `hdfs` table function. [#4084](https://github.com/yandex/ClickHouse/pull/4084) ([alesapin](https://github.com/alesapin))
+* Added functions to search for multiple constant strings from big haystack: `multiPosition`, `multiSearch` ,`firstMatch` also with `-UTF8`, `-CaseInsensitive`, and `-CaseInsensitiveUTF8` variants. [#4053](https://github.com/yandex/ClickHouse/pull/4053) ([Danila Kutenin](https://github.com/danlark1))
+* Pruning of unused shards if `SELECT` query filters by sharding key (setting `distributed_optimize_skip_select_on_unused_shards`). [#3851](https://github.com/yandex/ClickHouse/pull/3851) ([Ivan](https://github.com/abyss7))
+* Allow `Kafka` engine to ignore some number of parsing errors per block. [#4094](https://github.com/yandex/ClickHouse/pull/4094) ([Ivan](https://github.com/abyss7))
+* Added support for `CatBoost` multiclass models evaluation. Function `modelEvaluate` returns tuple with per-class raw predictions for multiclass models. `libcatboostmodel.so` should be built with [#607](https://github.com/catboost/catboost/pull/607). [#3959](https://github.com/yandex/ClickHouse/pull/3959) ([KochetovNicolai](https://github.com/KochetovNicolai))
+* Added functions `filesystemAvailable`, `filesystemFree`, `filesystemCapacity`. [#4097](https://github.com/yandex/ClickHouse/pull/4097) ([Boris Granveaud](https://github.com/bgranvea))
+* Added hashing functions `xxHash64` and `xxHash32`. [#3905](https://github.com/yandex/ClickHouse/pull/3905) ([filimonov](https://github.com/filimonov))
+* Added `gccMurmurHash` hashing function (GCC flavoured Murmur hash) which uses the same hash seed as [gcc](https://github.com/gcc-mirror/gcc/blob/41d6b10e96a1de98e90a7c0378437c3255814b16/libstdc%2B%2B-v3/include/bits/functional_hash.h#L191) [#4000](https://github.com/yandex/ClickHouse/pull/4000) ([sundyli](https://github.com/sundy-li))
+* Added hashing functions `javaHash`, `hiveHash`. [#3811](https://github.com/yandex/ClickHouse/pull/3811) ([shangshujie365](https://github.com/shangshujie365))
+* Added table function `remoteSecure`. Function works as `remote`, but uses secure connection. [#4088](https://github.com/yandex/ClickHouse/pull/4088) ([proller](https://github.com/proller))
+
+
+### Experimental features
+
+* Added multiple JOINs emulation (`allow_experimental_multiple_joins_emulation` setting). [#3946](https://github.com/yandex/ClickHouse/pull/3946) ([Artem Zuikov](https://github.com/4ertus2))
+
+
+### Bug Fixes
+
+* Make `compiled_expression_cache_size` setting limited by default to lower memory consumption. [#4041](https://github.com/yandex/ClickHouse/pull/4041) ([alesapin](https://github.com/alesapin))
+* Fix a bug that led to hangups in threads that perform ALTERs of Replicated tables and in the thread that updates configuration from ZooKeeper. [#2947](https://github.com/yandex/ClickHouse/issues/2947) [#3891](https://github.com/yandex/ClickHouse/issues/3891) [#3934](https://github.com/yandex/ClickHouse/pull/3934) ([Alex Zatelepin](https://github.com/ztlpn))
+* Fixed a race condition when executing a distributed ALTER task. The race condition led to more than one replica trying to execute the task and all replicas except one failing with a ZooKeeper error. [#3904](https://github.com/yandex/ClickHouse/pull/3904) ([Alex Zatelepin](https://github.com/ztlpn))
+* Fix a bug when `from_zk` config elements weren't refreshed after a request to ZooKeeper timed out. [#2947](https://github.com/yandex/ClickHouse/issues/2947) [#3947](https://github.com/yandex/ClickHouse/pull/3947) ([Alex Zatelepin](https://github.com/ztlpn))
+* Fix bug with wrong prefix for IPv4 subnet masks. [#3945](https://github.com/yandex/ClickHouse/pull/3945) ([alesapin](https://github.com/alesapin))
+* Fixed crash (`std::terminate`) in rare cases when a new thread cannot be created due to exhausted resources. [#3956](https://github.com/yandex/ClickHouse/pull/3956) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Fix bug when in `remote` table function execution when wrong restrictions were used for in `getStructureOfRemoteTable`. [#4009](https://github.com/yandex/ClickHouse/pull/4009) ([alesapin](https://github.com/alesapin))
+* Fix a leak of netlink sockets. They were placed in a pool where they were never deleted and new sockets were created at the start of a new thread when all current sockets were in use. [#4017](https://github.com/yandex/ClickHouse/pull/4017) ([Alex Zatelepin](https://github.com/ztlpn))
+* Fix bug with closing `/proc/self/fd` directory earlier than all fds were read from `/proc` after forking `odbc-bridge` subprocess. [#4120](https://github.com/yandex/ClickHouse/pull/4120) ([alesapin](https://github.com/alesapin))
+* Fixed String to UInt monotonic conversion in case of usage String in primary key. [#3870](https://github.com/yandex/ClickHouse/pull/3870) ([Winter Zhang](https://github.com/zhang2014))
+* Fixed error in calculation of integer conversion function monotonicity. [#3921](https://github.com/yandex/ClickHouse/pull/3921) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Fixed segfault in `arrayEnumerateUniq`, `arrayEnumerateDense` functions in case of some invalid arguments. [#3909](https://github.com/yandex/ClickHouse/pull/3909) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Fix UB in StorageMerge. [#3910](https://github.com/yandex/ClickHouse/pull/3910) ([Amos Bird](https://github.com/amosbird))
+* Fixed segfault in functions `addDays`, `subtractDays`. [#3913](https://github.com/yandex/ClickHouse/pull/3913) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Fixed error: functions `round`, `floor`, `trunc`, `ceil` may return bogus result when executed on integer argument and large negative scale. [#3914](https://github.com/yandex/ClickHouse/pull/3914) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Fixed a bug induced by 'kill query sync' which leads to a core dump. [#3916](https://github.com/yandex/ClickHouse/pull/3916) ([muVulDeePecker](https://github.com/fancyqlx))
+* Fix bug with long delay after empty replication queue. [#3928](https://github.com/yandex/ClickHouse/pull/3928) [#3932](https://github.com/yandex/ClickHouse/pull/3932) ([alesapin](https://github.com/alesapin))
+* Fixed excessive memory usage in case of inserting into table with `LowCardinality` primary key. [#3955](https://github.com/yandex/ClickHouse/pull/3955) ([KochetovNicolai](https://github.com/KochetovNicolai))
+* Fixed `LowCardinality` serialization for `Native` format in case of empty arrays. [#3907](https://github.com/yandex/ClickHouse/issues/3907) [#4011](https://github.com/yandex/ClickHouse/pull/4011) ([KochetovNicolai](https://github.com/KochetovNicolai))
+* Fixed incorrect result while using distinct by single LowCardinality numeric column. [#3895](https://github.com/yandex/ClickHouse/issues/3895) [#4012](https://github.com/yandex/ClickHouse/pull/4012) ([KochetovNicolai](https://github.com/KochetovNicolai))
+* Fixed specialized aggregation with LowCardinality key (in case when `compile` setting is enabled). [#3886](https://github.com/yandex/ClickHouse/pull/3886) ([KochetovNicolai](https://github.com/KochetovNicolai))
+* Fix user and password forwarding for replicated tables queries. [#3957](https://github.com/yandex/ClickHouse/pull/3957) ([alesapin](https://github.com/alesapin)) ([小路](https://github.com/nicelulu))
+* Fixed very rare race condition that can happen when listing tables in Dictionary database while reloading dictionaries. [#3970](https://github.com/yandex/ClickHouse/pull/3970) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Fixed incorrect result when HAVING was used with ROLLUP or CUBE. [#3756](https://github.com/yandex/ClickHouse/issues/3756) [#3837](https://github.com/yandex/ClickHouse/pull/3837) ([Sam Chou](https://github.com/reflection))
+* Fixed column aliases for query with `JOIN ON` syntax and distributed tables. [#3980](https://github.com/yandex/ClickHouse/pull/3980) ([Winter Zhang](https://github.com/zhang2014))
+* Fixed error in internal implementation of `quantileTDigest` (found by Artem Vakhrushev). This error never happens in ClickHouse and was relevant only for those who use ClickHouse codebase as a library directly. [#3935](https://github.com/yandex/ClickHouse/pull/3935) ([alexey-milovidov](https://github.com/alexey-milovidov))
+
+### Improvements
+
+* Support for `IF NOT EXISTS` in `ALTER TABLE ADD COLUMN` statements along with `IF EXISTS` in `DROP/MODIFY/CLEAR/COMMENT COLUMN`. [#3900](https://github.com/yandex/ClickHouse/pull/3900) ([Boris Granveaud](https://github.com/bgranvea))
+* Function `parseDateTimeBestEffort`: support for formats `DD.MM.YYYY`, `DD.MM.YY`, `DD-MM-YYYY`, `DD-Mon-YYYY`, `DD/Month/YYYY` and similar. [#3922](https://github.com/yandex/ClickHouse/pull/3922) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* `CapnProtoInputStream` now support jagged structures. [#4063](https://github.com/yandex/ClickHouse/pull/4063) ([Odin Hultgren Van Der Horst](https://github.com/Miniwoffer))
+* Usability improvement: added a check that server process is started from the data directory's owner. Do not allow to start server from root if the data belongs to non-root user. [#3785](https://github.com/yandex/ClickHouse/pull/3785) ([sergey-v-galtsev](https://github.com/sergey-v-galtsev))
+* Better logic of checking required columns during analysis of queries with JOINs. [#3930](https://github.com/yandex/ClickHouse/pull/3930) ([Artem Zuikov](https://github.com/4ertus2))
+* Decreased the number of connections in case of large number of Distributed tables in a single server. [#3726](https://github.com/yandex/ClickHouse/pull/3726) ([Winter Zhang](https://github.com/zhang2014))
+* Supported totals row for `WITH TOTALS` query for ODBC driver. [#3836](https://github.com/yandex/ClickHouse/pull/3836) ([Maksim Koritckiy](https://github.com/nightweb))
+* Allowed to use `Enum`s as integers inside if function. [#3875](https://github.com/yandex/ClickHouse/pull/3875) ([Ivan](https://github.com/abyss7))
+* Added `low_cardinality_allow_in_native_format` setting. If disabled, do not use `LowCadrinality` type in `Native` format. [#3879](https://github.com/yandex/ClickHouse/pull/3879) ([KochetovNicolai](https://github.com/KochetovNicolai))
+* Removed some redundant objects from compiled expressions cache to lower memory usage. [#4042](https://github.com/yandex/ClickHouse/pull/4042) ([alesapin](https://github.com/alesapin))
+* Add check that `SET send_logs_level = 'value'` query accept appropriate value. [#3873](https://github.com/yandex/ClickHouse/pull/3873) ([Sabyanin Maxim](https://github.com/s-mx))
+* Fixed data type check in type conversion functions. [#3896](https://github.com/yandex/ClickHouse/pull/3896) ([Winter Zhang](https://github.com/zhang2014))
+
+### Performance Improvements
+
+* Add a MergeTree setting `use_minimalistic_part_header_in_zookeeper`. If enabled, Replicated tables will store compact part metadata in a single part znode. This can dramatically reduce ZooKeeper snapshot size (especially if the tables have a lot of columns). Note that after enabling this setting you will not be able to downgrade to a version that doesn't support it. [#3960](https://github.com/yandex/ClickHouse/pull/3960) ([Alex Zatelepin](https://github.com/ztlpn))
+* Add an DFA-based implementation for functions `sequenceMatch` and `sequenceCount` in case pattern doesn't contain time. [#4004](https://github.com/yandex/ClickHouse/pull/4004) ([Léo Ercolanelli](https://github.com/ercolanelli-leo))
+* Performance improvement for integer numbers serialization. [#3968](https://github.com/yandex/ClickHouse/pull/3968) ([Amos Bird](https://github.com/amosbird))
+* Zero left padding PODArray so that -1 element is always valid and zeroed. It's used for branchless calculation of offsets. [#3920](https://github.com/yandex/ClickHouse/pull/3920) ([Amos Bird](https://github.com/amosbird))
+* Reverted `jemalloc` version which lead to performance degradation. [#4018](https://github.com/yandex/ClickHouse/pull/4018) ([alexey-milovidov](https://github.com/alexey-milovidov))
+
+### Backward Incompatible Changes
+
+* Removed undocumented feature `ALTER MODIFY PRIMARY KEY` because it was superseded by the `ALTER MODIFY ORDER BY` command. [#3887](https://github.com/yandex/ClickHouse/pull/3887) ([Alex Zatelepin](https://github.com/ztlpn))
+* Removed function `shardByHash`. [#3833](https://github.com/yandex/ClickHouse/pull/3833) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Forbid using scalar subqueries with result of type `AggregateFunction`. [#3865](https://github.com/yandex/ClickHouse/pull/3865) ([Ivan](https://github.com/abyss7))
+
+### Build/Testing/Packaging Improvements
+
+* Added support for PowerPC (`ppc64le`) build. [#4132](https://github.com/yandex/ClickHouse/pull/4132) ([Danila Kutenin](https://github.com/danlark1))
+* Stateful functional tests are run on public available dataset. [#3969](https://github.com/yandex/ClickHouse/pull/3969) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Fixed error when the server cannot start with the `bash: /usr/bin/clickhouse-extract-from-config: Operation not permitted` message within Docker or systemd-nspawn. [#4136](https://github.com/yandex/ClickHouse/pull/4136) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Updated `rdkafka` library to v1.0.0-RC5. Used cppkafka instead of raw C interface. [#4025](https://github.com/yandex/ClickHouse/pull/4025) ([Ivan](https://github.com/abyss7))
+* Updated `mariadb-client` library. Fixed one of issues found by UBSan. [#3924](https://github.com/yandex/ClickHouse/pull/3924) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Some fixes for UBSan builds. [#3926](https://github.com/yandex/ClickHouse/pull/3926) [#3021](https://github.com/yandex/ClickHouse/pull/3021) [#3948](https://github.com/yandex/ClickHouse/pull/3948) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Added per-commit runs of tests with UBSan build.
+* Added per-commit runs of PVS-Studio static analyzer.
+* Fixed bugs found by PVS-Studio. [#4013](https://github.com/yandex/ClickHouse/pull/4013) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Fixed glibc compatibility issues. [#4100](https://github.com/yandex/ClickHouse/pull/4100) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Move Docker images to 18.10 and add compatibility file for glibc >= 2.28 [#3965](https://github.com/yandex/ClickHouse/pull/3965) ([alesapin](https://github.com/alesapin))
+* Add env variable if user don't want to chown directories in server Docker image. [#3967](https://github.com/yandex/ClickHouse/pull/3967) ([alesapin](https://github.com/alesapin))
+* Enabled most of the warnings from `-Weverything` in clang. Enabled `-Wpedantic`. [#3986](https://github.com/yandex/ClickHouse/pull/3986) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Added a few more warnings that are available only in clang 8. [#3993](https://github.com/yandex/ClickHouse/pull/3993) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Link to `libLLVM` rather than to individual LLVM libs when using shared linking. [#3989](https://github.com/yandex/ClickHouse/pull/3989) ([Orivej Desh](https://github.com/orivej))
+* Added sanitizer variables for test images. [#4072](https://github.com/yandex/ClickHouse/pull/4072) ([alesapin](https://github.com/alesapin))
+* `clickhouse-server` debian package will recommend `libcap2-bin` package to use `setcap` tool for setting capabilities. This is optional. [#4093](https://github.com/yandex/ClickHouse/pull/4093) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Improved compilation time, fixed includes. [#3898](https://github.com/yandex/ClickHouse/pull/3898) ([proller](https://github.com/proller))
+* Added performance tests for hash functions. [#3918](https://github.com/yandex/ClickHouse/pull/3918) ([filimonov](https://github.com/filimonov))
+* Fixed cyclic library dependences. [#3958](https://github.com/yandex/ClickHouse/pull/3958) ([proller](https://github.com/proller))
+* Improved compilation with low available memory. [#4030](https://github.com/yandex/ClickHouse/pull/4030) ([proller](https://github.com/proller))
+* Added test script to reproduce performance degradation in `jemalloc`. [#4036](https://github.com/yandex/ClickHouse/pull/4036) ([alexey-milovidov](https://github.com/alexey-milovidov))
+* Fixed misspells in comments and string literals under `dbms`. [#4122](https://github.com/yandex/ClickHouse/pull/4122) ([maiha](https://github.com/maiha))
+* Fixed typos in comments. [#4089](https://github.com/yandex/ClickHouse/pull/4089) ([Evgenii Pravda](https://github.com/kvinty))
+
+
 ## ClickHouse release 18.16.1, 2018-12-21

 ### Bug fixes:
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -3,6 +3,21 @@ cmake_minimum_required (VERSION 3.3)

 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake/Modules/")

+option(ENABLE_IPO "Enable inter-procedural optimization (aka LTO)" OFF) # need cmake 3.9+
+if(ENABLE_IPO)
+    cmake_policy(SET CMP0069 NEW)
+    include(CheckIPOSupported)
+    check_ipo_supported(RESULT IPO_SUPPORTED OUTPUT IPO_NOT_SUPPORTED)
+    if(IPO_SUPPORTED)
+        message(STATUS "IPO/LTO is supported, enabling")
+        set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
+    else()
+        message(STATUS "IPO/LTO is not supported: <${IPO_NOT_SUPPORTED}>")
+    endif()
+else()
+    message(STATUS "IPO/LTO not enabled.")
+endif()
+
 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
    # Require at least gcc 7
    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7 AND NOT CMAKE_VERSION VERSION_LESS 2.8.9)
@ -25,11 +40,6 @@ endif ()
 # Write compile_commands.json
 set(CMAKE_EXPORT_COMPILE_COMMANDS 1)

-
-set (MAX_COMPILER_MEMORY 2000 CACHE INTERNAL "")
-set (MAX_LINKER_MEMORY 3500 CACHE INTERNAL "")
-include (cmake/limit_jobs.cmake)
-
 include (cmake/find_ccache.cmake)

 if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None")
@ -86,17 +96,19 @@ option (ENABLE_TESTS "Enables tests" ON)
 if (CMAKE_SYSTEM_PROCESSOR MATCHES "amd64|x86_64")
    option (USE_INTERNAL_MEMCPY "Use internal implementation of 'memcpy' function instead of provided by libc. Only for x86_64." ON)

-    if (OS_LINUX)
+    if (OS_LINUX AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND CMAKE_VERSION VERSION_GREATER "3.9.0")
        option (GLIBC_COMPATIBILITY "Set to TRUE to enable compatibility with older glibc libraries. Only for x86_64, Linux. Implies USE_INTERNAL_MEMCPY." ON)
-    endif()
+        if (GLIBC_COMPATIBILITY)
+            message (STATUS "Some symbols from glibc will be replaced for compatibility")
+            link_libraries(glibc-compatibility)
+        endif ()
+    endif ()
 endif ()

 if (GLIBC_COMPATIBILITY)
    set (USE_INTERNAL_MEMCPY ON)
 endif ()

-set (COMPILER_FLAGS "${COMPILER_FLAGS}")
-
 string(REGEX MATCH "-?[0-9]+(.[0-9]+)?$" COMPILER_POSTFIX ${CMAKE_CXX_COMPILER})

 find_program (LLD_PATH NAMES "lld${COMPILER_POSTFIX}" "lld")
@ -113,12 +125,19 @@ if (LINKER_NAME)
    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")
 endif ()

-option (PIPE "-pipe compiler option [less /tmp usage, more ram usage]" ON)
-if (PIPE)
-    set (COMPILER_FLAGS                  "${COMPILER_FLAGS} -pipe")
-endif ()
+cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd
+if(NOT AVAILABLE_PHYSICAL_MEMORY OR AVAILABLE_PHYSICAL_MEMORY GREATER 8000)
+    option(COMPILER_PIPE "-pipe compiler option [less /tmp usage, more ram usage]" ON)
+endif()
+if(COMPILER_PIPE)
+    set(COMPILER_FLAGS "${COMPILER_FLAGS} -pipe")
+else()
+    message(STATUS "Disabling compiler -pipe option (have only ${AVAILABLE_PHYSICAL_MEMORY} mb of memory)")
+endif()

-include (cmake/test_cpu.cmake)
+if(NOT DISABLE_CPU_OPTIMIZE)
+    include(cmake/test_cpu.cmake)
+endif()

 if(NOT COMPILER_CLANG) # clang: error: the clang compiler does not support '-march=native'
    option(ARCH_NATIVE "Enable -march=native compiler flag" ${ARCH_ARM})
@ -181,8 +200,8 @@ endif (TEST_COVERAGE)

 if (ENABLE_TESTS)
    message (STATUS "Tests are enabled")
-    enable_testing()
 endif ()
+enable_testing() # Enable for tests without binary

 # when installing to /usr - place configs to /etc but for /usr/local place to /usr/local/etc
 if (CMAKE_INSTALL_PREFIX STREQUAL "/usr")
@ -227,9 +246,14 @@ include (cmake/find_re2.cmake)
 include (cmake/find_rdkafka.cmake)
 include (cmake/find_capnp.cmake)
 include (cmake/find_llvm.cmake)
-include (cmake/find_cpuid.cmake)
+include (cmake/find_cpuid.cmake) # Freebsd, bundled
+if (NOT USE_CPUID)
+    include (cmake/find_cpuinfo.cmake) # Debian
+endif()
 include (cmake/find_libgsasl.cmake)
 include (cmake/find_libxml2.cmake)
+include (cmake/find_protobuf.cmake)
+include (cmake/find_pdqsort.cmake)
 include (cmake/find_hdfs3.cmake)
 include (cmake/find_consistent-hashing.cmake)
 include (cmake/find_base64.cmake)
@ -253,7 +277,7 @@ include (libs/libdaemon/cmake/find_unwind.cmake)

 include (cmake/print_flags.cmake)

-add_subdirectory (contrib)
+add_subdirectory (contrib EXCLUDE_FROM_ALL)
 add_subdirectory (libs)
 add_subdirectory (utils)
 add_subdirectory (dbms)
--- a/2
+++ b/2
@ -186,7 +186,7 @@
      same "printed page" as the copyright notice for easier
      identification within third-party archives.

-   Copyright 2016-2018 Yandex LLC
+   Copyright 2016-2019 Yandex LLC

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
--- a/PreLoad.cmake
+++ b/PreLoad.cmake
@ -1,7 +1,7 @@
 # Use Ninja instead of Unix Makefiles by default.
 # https://stackoverflow.com/questions/11269833/cmake-selecting-a-generator-within-cmakelists-txt
 #
-# Reason: it have better startup time than make and it parallelize jobs more uniformly.
+# Reason: it has better startup time than make and it parallelizes jobs more uniformly.
 # (when comparing to make with Makefiles that was generated by CMake)
 #
 # How to install Ninja on Ubuntu:
--- a/README.md
+++ b/README.md
@ -10,3 +10,7 @@ ClickHouse is an open-source column-oriented database management system that all
 * [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events.
 * [Contacts](https://clickhouse.yandex/#contacts) can help to get your questions answered if there are any.
 * You can also [fill this form](https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/) to meet Yandex ClickHouse team in person.
+
+## Upcoming Events
+
+* [C++ ClickHouse and CatBoost Sprints](https://events.yandex.ru/events/ClickHouse/2-feb-2019/) in Moscow on February 2.
--- a/ci/README.md
+++ b/ci/README.md
@ -25,16 +25,17 @@ Various possible options. We are not going to automate testing all of them.

 #### CPU architectures:
 - x86_64;
- AArch64.
+- AArch64;
+- PowerPC64LE.

-x86_64 is the main CPU architecture. We also have minimal support for AArch64.
+x86_64 is the main CPU architecture. We also have minimal support for AArch64 and PowerPC64LE.

 #### Operating systems:
 - Linux;
 - FreeBSD.

-We also target Mac OS X, but it's more difficult to test.  
-Linux is the main. FreeBSD is also supported as production OS.  
+We also target Mac OS X, but it's more difficult to test.
+Linux is the main. FreeBSD is also supported as production OS.
 Mac OS is intended only for development and have minimal support: client should work, server should just start.

 #### Linux distributions:
@ -98,14 +99,14 @@ We also have intent to build RPM and simple tgz packages.
 - from contrib directory (submodules);
 - from OS packages.

-The only production option is to use libraries from contrib directory.  
+The only production option is to use libraries from contrib directory.
 Using libraries from OS packages is discouraged, but we also support this option.

 #### Linkage types:
 - static;
 - shared;

-Static linking is the only option for production usage.  
+Static linking is the only option for production usage.
 We also have support for shared linking, but it is indended only for developers.

 #### Make tools:
--- a/cmake/Modules/Findmetrohash.cmake
+++ b/cmake/Modules/Findmetrohash.cmake
@ -28,7 +28,7 @@ find_library(METROHASH_LIBRARIES

 find_path(METROHASH_INCLUDE_DIR
    NAMES metrohash.h
-    PATHS ${METROHASH_ROOT_DIR}/include ${METROHASH_INCLUDE_PATHS}
+    PATHS ${METROHASH_ROOT_DIR}/include PATH_SUFFIXES metrohash ${METROHASH_INCLUDE_PATHS}
 )

 include(FindPackageHandleStandardArgs)
--- a/cmake/arch.cmake
+++ b/cmake/arch.cmake
@ -24,3 +24,10 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
 elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
    set (COMPILER_CLANG 1)
 endif ()
+
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc64le.*|PPC64LE.*)")
+    set (ARCH_PPC64LE 1)
+    if (COMPILER_CLANG OR (COMPILER_GCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8))
+        message(FATAL_ERROR "Only gcc-8 is supported for powerpc architecture")
+    endif ()
+endif ()
--- a/cmake/find_cpuid.cmake
+++ b/cmake/find_cpuid.cmake
@ -2,11 +2,11 @@ if (NOT ARCH_ARM)
    option (USE_INTERNAL_CPUID_LIBRARY "Set to FALSE to use system cpuid library instead of bundled" ${NOT_UNBUNDLED})
 endif ()

-#if (USE_INTERNAL_CPUID_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcpuid/include/cpuid/libcpuid.h")
-#   message (WARNING "submodule contrib/libcpuid is missing. to fix try run: \n git submodule update --init --recursive")
-#   set (USE_INTERNAL_CPUID_LIBRARY 0)
-#   set (MISSING_INTERNAL_CPUID_LIBRARY 1)
-#endif ()
+if (USE_INTERNAL_CPUID_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcpuid/CMakeLists.txt")
+   message (WARNING "submodule contrib/libcpuid is missing. to fix try run: \n git submodule update --init --recursive")
+   set (USE_INTERNAL_CPUID_LIBRARY 0)
+   set (MISSING_INTERNAL_CPUID_LIBRARY 1)
+endif ()

 if (NOT USE_INTERNAL_CPUID_LIBRARY)
    find_library (CPUID_LIBRARY cpuid)
@ -20,10 +20,12 @@ if (CPUID_LIBRARY AND CPUID_INCLUDE_DIR)
        add_definitions(-DHAVE_STDINT_H)
        # TODO: make virtual target cpuid:cpuid with COMPILE_DEFINITIONS property
    endif ()
+    set (USE_CPUID 1)
 elseif (NOT MISSING_INTERNAL_CPUID_LIBRARY)
    set (CPUID_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcpuid/include)
    set (USE_INTERNAL_CPUID_LIBRARY 1)
    set (CPUID_LIBRARY cpuid)
+    set (USE_CPUID 1)
 endif ()

-message (STATUS "Using cpuid: ${CPUID_INCLUDE_DIR} : ${CPUID_LIBRARY}")
+message (STATUS "Using cpuid=${USE_CPUID}: ${CPUID_INCLUDE_DIR} : ${CPUID_LIBRARY}")
--- a/cmake/find_cpuinfo.cmake
+++ b/cmake/find_cpuinfo.cmake
@ -0,0 +1,17 @@
+option(USE_INTERNAL_CPUINFO_LIBRARY "Set to FALSE to use system cpuinfo library instead of bundled" ${NOT_UNBUNDLED})
+
+if(NOT USE_INTERNAL_CPUINFO_LIBRARY)
+    find_library(CPUINFO_LIBRARY cpuinfo)
+    find_path(CPUINFO_INCLUDE_DIR NAMES cpuinfo.h PATHS ${CPUINFO_INCLUDE_PATHS})
+endif()
+
+if(CPUID_LIBRARY AND CPUID_INCLUDE_DIR)
+    set(USE_CPUINFO 1)
+elseif(NOT MISSING_INTERNAL_CPUINFO_LIBRARY)
+    set(CPUINFO_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcpuinfo/include)
+    set(USE_INTERNAL_CPUINFO_LIBRARY 1)
+    set(CPUINFO_LIBRARY cpuinfo)
+    set(USE_CPUINFO 1)
+endif()
+
+message(STATUS "Using cpuinfo=${USE_CPUINFO}: ${CPUINFO_INCLUDE_DIR} : ${CPUINFO_LIBRARY}")
--- a/cmake/find_gtest.cmake
+++ b/cmake/find_gtest.cmake
@ -8,13 +8,22 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest/CMakeList
   set (MISSING_INTERNAL_GTEST_LIBRARY 1)
 endif ()

-if (NOT USE_INTERNAL_GTEST_LIBRARY)
-    find_package (GTest)
-endif ()

-if (NOT GTEST_INCLUDE_DIRS AND NOT MISSING_INTERNAL_GTEST_LIBRARY)
+if(NOT USE_INTERNAL_GTEST_LIBRARY)
+    # TODO: autodetect of GTEST_SRC_DIR by EXISTS /usr/src/googletest/CMakeLists.txt
+    if(NOT GTEST_SRC_DIR)
+        find_package(GTest)
+    endif()
+endif()
+
+if (NOT GTEST_SRC_DIR AND NOT GTEST_INCLUDE_DIRS AND NOT MISSING_INTERNAL_GTEST_LIBRARY)
    set (USE_INTERNAL_GTEST_LIBRARY 1)
    set (GTEST_MAIN_LIBRARIES gtest_main)
+    set (GTEST_INCLUDE_DIRS ${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest)
 endif ()

-message (STATUS "Using gtest: ${GTEST_INCLUDE_DIRS} : ${GTEST_MAIN_LIBRARIES}")
+if((GTEST_INCLUDE_DIRS AND GTEST_MAIN_LIBRARIES) OR GTEST_SRC_DIR)
+    set(USE_GTEST 1)
+endif()
+
+message (STATUS "Using gtest=${USE_GTEST}: ${GTEST_INCLUDE_DIRS} : ${GTEST_MAIN_LIBRARIES} : ${GTEST_SRC_DIR}")
--- a/cmake/find_llvm.cmake
+++ b/cmake/find_llvm.cmake
@ -79,7 +79,11 @@ endif()

 function(llvm_libs_all REQUIRED_LLVM_LIBRARIES)
    llvm_map_components_to_libnames (result all)
-    list (REMOVE_ITEM result "LTO" "LLVM")
+    if (USE_STATIC_LIBRARIES OR NOT "LLVM" IN_LIST result)
+        list (REMOVE_ITEM result "LTO" "LLVM")
+    else()
+        set (result "LLVM")
+    endif ()
    if (TERMCAP_LIBRARY)
        list (APPEND result ${TERMCAP_LIBRARY})
    endif ()
--- a/cmake/find_pdqsort.cmake
+++ b/cmake/find_pdqsort.cmake
@ -0,0 +1,2 @@
+set(PDQSORT_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/pdqsort)
+message(STATUS "Using pdqsort: ${PDQSORT_INCLUDE_DIR}")
--- a/cmake/find_protobuf.cmake
+++ b/cmake/find_protobuf.cmake
@ -1,18 +1,35 @@
-option (USE_INTERNAL_PROTOBUF_LIBRARY "Set to FALSE to use system protobuf instead of bundled" ON)
+option(USE_INTERNAL_PROTOBUF_LIBRARY "Set to FALSE to use system protobuf instead of bundled" ${NOT_UNBUNDLED})

-if (NOT USE_INTERNAL_PROTOBUF_LIBRARY)
+if(OS_FREEBSD AND SANITIZE STREQUAL "address")
+    # ../contrib/protobuf/src/google/protobuf/arena_impl.h:45:10: fatal error: 'sanitizer/asan_interface.h' file not found
+    set(MISSING_INTERNAL_PROTOBUF_LIBRARY 1)
+    set(USE_INTERNAL_PROTOBUF_LIBRARY 0)
+endif()
+
+if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/protobuf/cmake/CMakeLists.txt")
+   if(USE_INTERNAL_PROTOBUF_LIBRARY)
+       message(WARNING "submodule contrib/protobuf is missing. to fix try run: \n git submodule update --init --recursive")
+       set(USE_INTERNAL_PROTOBUF_LIBRARY 0)
+   endif()
+   set(MISSING_INTERNAL_PROTOBUF_LIBRARY 1)
+endif()
+
+if(NOT USE_INTERNAL_PROTOBUF_LIBRARY)
    find_package(Protobuf)
-endif ()
+endif()

 if (Protobuf_LIBRARY AND Protobuf_INCLUDE_DIR)
-else ()
-    set(Protobuf_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/contrib/protobuf/src)
+    set(USE_PROTOBUF 1)
+elseif(NOT MISSING_INTERNAL_PROTOBUF_LIBRARY)
+    set(Protobuf_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/protobuf/src)

+    set(USE_PROTOBUF 1)
+    set(USE_INTERNAL_PROTOBUF_LIBRARY 1)
    set(Protobuf_LIBRARY libprotobuf)
    set(Protobuf_PROTOC_LIBRARY libprotoc)
    set(Protobuf_LITE_LIBRARY libprotobuf-lite)

-    set(Protobuf_PROTOC_EXECUTABLE ${CMAKE_BINARY_DIR}/contrib/protobuf/cmake/protoc)
+    set(Protobuf_PROTOC_EXECUTABLE ${ClickHouse_BINARY_DIR}/contrib/protobuf/cmake/protoc)

    if(NOT DEFINED PROTOBUF_GENERATE_CPP_APPEND_PATH)
        set(PROTOBUF_GENERATE_CPP_APPEND_PATH TRUE)
@ -77,4 +94,4 @@ else ()
    endfunction()
 endif()

-message (STATUS "Using protobuf: ${Protobuf_INCLUDE_DIR} : ${Protobuf_LIBRARY}")
+message(STATUS "Using protobuf=${USE_PROTOBUF}: ${Protobuf_INCLUDE_DIR} : ${Protobuf_LIBRARY}")
--- a/cmake/find_rdkafka.cmake
+++ b/cmake/find_rdkafka.cmake
@ -1,7 +1,12 @@
-if (NOT ARCH_ARM AND NOT ARCH_32)
+if (NOT ARCH_ARM AND NOT ARCH_32 AND NOT APPLE)
    option (ENABLE_RDKAFKA "Enable kafka" ON)
 endif ()

+if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cppkafka/CMakeLists.txt")
+   message (WARNING "submodule contrib/cppkafka is missing. to fix try run: \n git submodule update --init --recursive")
+   set (ENABLE_RDKAFKA 0)
+endif ()
+
 if (ENABLE_RDKAFKA)

 if (OS_LINUX AND NOT ARCH_ARM)
@ -20,11 +25,13 @@ if (NOT USE_INTERNAL_RDKAFKA_LIBRARY)
    if (USE_STATIC_LIBRARIES AND NOT OS_FREEBSD)
       find_library (SASL2_LIBRARY sasl2)
    endif ()
+    set (CPPKAFKA_LIBRARY cppkafka) # TODO: try to use unbundled version.
 endif ()

 if (RDKAFKA_LIB AND RDKAFKA_INCLUDE_DIR)
    set (USE_RDKAFKA 1)
    set (RDKAFKA_LIBRARY ${RDKAFKA_LIB} ${OPENSSL_LIBRARIES})
+    set (CPPKAFKA_LIBRARY cppkafka)
    if (SASL2_LIBRARY)
       list (APPEND RDKAFKA_LIBRARY ${SASL2_LIBRARY})
    endif ()
@ -35,9 +42,10 @@ elseif (NOT MISSING_INTERNAL_RDKAFKA_LIBRARY AND NOT ARCH_ARM)
    set (USE_INTERNAL_RDKAFKA_LIBRARY 1)
    set (RDKAFKA_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/src")
    set (RDKAFKA_LIBRARY rdkafka)
+    set (CPPKAFKA_LIBRARY cppkafka)
    set (USE_RDKAFKA 1)
 endif ()

 endif ()

-message (STATUS "Using librdkafka=${USE_RDKAFKA}: ${RDKAFKA_INCLUDE_DIR} : ${RDKAFKA_LIBRARY}")
+message (STATUS "Using librdkafka=${USE_RDKAFKA}: ${RDKAFKA_INCLUDE_DIR} : ${RDKAFKA_LIBRARY} ${CPPKAFKA_LIBRARY}")
--- a/cmake/find_re2.cmake
+++ b/cmake/find_re2.cmake
@ -5,13 +5,24 @@ if (NOT USE_INTERNAL_RE2_LIBRARY)
    find_path (RE2_INCLUDE_DIR NAMES re2/re2.h PATHS ${RE2_INCLUDE_PATHS})
 endif ()

+string(FIND ${CMAKE_CURRENT_BINARY_DIR} " " _have_space)
+if(_have_space GREATER 0)
+    message(WARNING "Using spaces in build path [${CMAKE_CURRENT_BINARY_DIR}] highly not recommended. Library re2st will be disabled.")
+    set (MISSING_INTERNAL_RE2_ST_LIBRARY 1)
+endif()
+
 if (RE2_LIBRARY AND RE2_INCLUDE_DIR)
    set (RE2_ST_LIBRARY ${RE2_LIBRARY})
-else ()
+elseif (NOT MISSING_INTERNAL_RE2_LIBRARY)
    set (USE_INTERNAL_RE2_LIBRARY 1)
    set (RE2_LIBRARY re2)
-    set (RE2_ST_LIBRARY re2_st)
-    set (USE_RE2_ST 1)
+    set (RE2_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/re2)
+    if (NOT MISSING_INTERNAL_RE2_ST_LIBRARY)
+        set (RE2_ST_LIBRARY re2_st)
+        set (USE_RE2_ST 1)
+    else ()
+        set (RE2_ST_LIBRARY ${RE2_LIBRARY})
+    endif ()
 endif ()

 message (STATUS "Using re2: ${RE2_INCLUDE_DIR} : ${RE2_LIBRARY}; ${RE2_ST_INCLUDE_DIR} : ${RE2_ST_LIBRARY}")
--- a/cmake/find_ssl.cmake
+++ b/cmake/find_ssl.cmake
@ -6,7 +6,7 @@ set (OPENSSL_USE_STATIC_LIBS ${USE_STATIC_LIBRARIES})

 if (NOT USE_INTERNAL_SSL_LIBRARY)
    if (APPLE)
-        set (OPENSSL_ROOT_DIR "/usr/local/opt/openssl")
+        set (OPENSSL_ROOT_DIR "/usr/local/opt/openssl" CACHE INTERNAL "")
        # https://rt.openssl.org/Ticket/Display.html?user=guest&pass=guest&id=2232
        if (USE_STATIC_LIBRARIES)
            message(WARNING "Disable USE_STATIC_LIBRARIES if you have linking problems with OpenSSL on MacOS")
--- a/cmake/find_zlib.cmake
+++ b/cmake/find_zlib.cmake
@ -8,23 +8,24 @@ endif ()

 if (NOT ZLIB_FOUND)
    if (NOT MSVC)
-        set (INTERNAL_ZLIB_NAME "zlib-ng")
+        set (INTERNAL_ZLIB_NAME "zlib-ng" CACHE INTERNAL "")
    else ()
-        set (INTERNAL_ZLIB_NAME "zlib")
+        set (INTERNAL_ZLIB_NAME "zlib" CACHE INTERNAL "")
        if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}")
            message (WARNING "Will use standard zlib, please clone manually:\n git clone https://github.com/madler/zlib.git ${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}")
        endif ()
    endif ()

    set (USE_INTERNAL_ZLIB_LIBRARY 1)
-    set (ZLIB_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}" "${ClickHouse_BINARY_DIR}/contrib/${INTERNAL_ZLIB_NAME}") # generated zconf.h
+    set (ZLIB_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}" "${ClickHouse_BINARY_DIR}/contrib/${INTERNAL_ZLIB_NAME}" CACHE INTERNAL "") # generated zconf.h
    set (ZLIB_INCLUDE_DIRS ${ZLIB_INCLUDE_DIR}) # for poco
+    set (ZLIB_INCLUDE_DIRECTORIES ${ZLIB_INCLUDE_DIR}) # for protobuf
    set (ZLIB_FOUND 1) # for poco
    if (USE_STATIC_LIBRARIES)
-        set (ZLIB_LIBRARIES zlibstatic)
+        set (ZLIB_LIBRARIES zlibstatic CACHE INTERNAL "")
    else ()
-        set (ZLIB_LIBRARIES zlib)
+        set (ZLIB_LIBRARIES zlib CACHE INTERNAL "")
    endif ()
 endif ()

-message (STATUS "Using zlib: ${ZLIB_INCLUDE_DIR} : ${ZLIB_LIBRARIES}")
+message (STATUS "Using ${INTERNAL_ZLIB_NAME}: ${ZLIB_INCLUDE_DIR} : ${ZLIB_LIBRARIES}")
--- a/cmake/find_zstd.cmake
+++ b/cmake/find_zstd.cmake
@ -14,6 +14,7 @@ if (ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR)
 else ()
    set (USE_INTERNAL_ZSTD_LIBRARY 1)
    set (ZSTD_LIBRARY zstd)
+    set (ZSTD_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/zstd/lib)
 endif ()

 message (STATUS "Using zstd: ${ZSTD_INCLUDE_DIR} : ${ZSTD_LIBRARY}")
--- a/cmake/lib_name.cmake
+++ b/cmake/lib_name.cmake
@ -2,4 +2,5 @@ set(DIVIDE_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libdivide)
 set(COMMON_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/libs/libcommon/include ${ClickHouse_BINARY_DIR}/libs/libcommon/include)
 set(DBMS_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/dbms/src ${ClickHouse_BINARY_DIR}/dbms/src)
 set(DOUBLE_CONVERSION_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/double-conversion)
+set(METROHASH_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src)
 set(PCG_RANDOM_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libpcg-random/include)
--- a/cmake/limit_jobs.cmake
+++ b/cmake/limit_jobs.cmake
@ -4,32 +4,34 @@
 # include (cmake/limit_jobs.cmake)

 cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd
+cmake_host_system_information(RESULT NUMBER_OF_LOGICAL_CORES QUERY NUMBER_OF_LOGICAL_CORES)

 option(PARALLEL_COMPILE_JOBS "Define the maximum number of concurrent compilation jobs" "")
-if (NOT PARALLEL_COMPILE_JOBS AND AVAILABLE_PHYSICAL_MEMORY)
-    math(EXPR PARALLEL_COMPILE_JOBS ${AVAILABLE_PHYSICAL_MEMORY}/2500) # ~2.5gb max per one compiler
+if (NOT PARALLEL_COMPILE_JOBS AND AVAILABLE_PHYSICAL_MEMORY AND MAX_COMPILER_MEMORY)
+    math(EXPR PARALLEL_COMPILE_JOBS ${AVAILABLE_PHYSICAL_MEMORY}/${MAX_COMPILER_MEMORY})
    if (NOT PARALLEL_COMPILE_JOBS)
        set (PARALLEL_COMPILE_JOBS 1)
    endif ()
 endif ()
-if (PARALLEL_COMPILE_JOBS)
-    set_property(GLOBAL APPEND PROPERTY JOB_POOLS compile_job_pool=${PARALLEL_COMPILE_JOBS})
-    set(CMAKE_JOB_POOL_COMPILE compile_job_pool)
+if (PARALLEL_COMPILE_JOBS AND (NOT NUMBER_OF_LOGICAL_CORES OR PARALLEL_COMPILE_JOBS LESS NUMBER_OF_LOGICAL_CORES))
+    set(CMAKE_JOB_POOL_COMPILE compile_job_pool${CMAKE_CURRENT_SOURCE_DIR})
+    string (REGEX REPLACE "[^a-zA-Z0-9]+" "_" CMAKE_JOB_POOL_COMPILE ${CMAKE_JOB_POOL_COMPILE})
+    set_property(GLOBAL APPEND PROPERTY JOB_POOLS ${CMAKE_JOB_POOL_COMPILE}=${PARALLEL_COMPILE_JOBS})
 endif ()

 option(PARALLEL_LINK_JOBS "Define the maximum number of concurrent link jobs" "")
-if (NOT PARALLEL_LINK_JOBS AND AVAILABLE_PHYSICAL_MEMORY)
-    math(EXPR PARALLEL_LINK_JOBS ${AVAILABLE_PHYSICAL_MEMORY}/4000) # ~4gb max per one linker
+if (NOT PARALLEL_LINK_JOBS AND AVAILABLE_PHYSICAL_MEMORY AND MAX_LINKER_MEMORY)
+    math(EXPR PARALLEL_LINK_JOBS ${AVAILABLE_PHYSICAL_MEMORY}/${MAX_LINKER_MEMORY})
    if (NOT PARALLEL_LINK_JOBS)
        set (PARALLEL_LINK_JOBS 1)
    endif ()
 endif ()
+if (PARALLEL_LINK_JOBS AND (NOT NUMBER_OF_LOGICAL_CORES OR PARALLEL_COMPILE_JOBS LESS NUMBER_OF_LOGICAL_CORES))
+    set(CMAKE_JOB_POOL_LINK link_job_pool${CMAKE_CURRENT_SOURCE_DIR})
+    string (REGEX REPLACE "[^a-zA-Z0-9]+" "_" CMAKE_JOB_POOL_LINK ${CMAKE_JOB_POOL_LINK})
+    set_property(GLOBAL APPEND PROPERTY JOB_POOLS ${CMAKE_JOB_POOL_LINK}=${PARALLEL_LINK_JOBS})
+endif ()
+
 if (PARALLEL_COMPILE_JOBS OR PARALLEL_LINK_JOBS)
-    message(STATUS "Have ${AVAILABLE_PHYSICAL_MEMORY} megabytes of memory. Limiting concurrent linkers jobs to ${PARALLEL_LINK_JOBS} and compiler jobs to ${PARALLEL_COMPILE_JOBS}")
+    message(STATUS "${CMAKE_CURRENT_SOURCE_DIR}: Have ${AVAILABLE_PHYSICAL_MEMORY} megabytes of memory. Limiting concurrent linkers jobs to ${PARALLEL_LINK_JOBS} and compiler jobs to ${PARALLEL_COMPILE_JOBS}")
 endif ()
-
-if (LLVM_PARALLEL_LINK_JOBS)
-    set_property(GLOBAL APPEND PROPERTY JOB_POOLS link_job_pool=${PARALLEL_LINK_JOBS})
-    set(CMAKE_JOB_POOL_LINK link_job_pool)
-endif ()
-
--- a/cmake/test_cpu.cmake
+++ b/cmake/test_cpu.cmake
@ -27,6 +27,9 @@ if (HAVE_SSE41)
    set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}")
 endif ()

+if (ARCH_PPC64LE)
+    set (COMPILER_FLAGS "${COMPILER_FLAGS} -maltivec -D__SSE2__=1 -DNO_WARN_X86_INTRINSICS")
+endif ()

 # gcc -dM -E -msse4.2 - < /dev/null | sort > gcc-dump-sse42
 #define __SSE4_2__ 1
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@ -1,11 +1,11 @@
 # Third-party libraries may have substandard code.

 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-stringop-overflow")
+    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-stringop-overflow -Wno-implicit-function-declaration -Wno-return-type -Wno-array-bounds -Wno-bool-compare -Wno-int-conversion -Wno-switch")
    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-implicit-fallthrough -Wno-class-memaccess -Wno-sign-compare -std=c++1z")
 elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-format -Wno-parentheses-equality")
-    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-format -std=c++1z")
+    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-format -Wno-parentheses-equality -Wno-tautological-constant-compare -Wno-tautological-constant-out-of-range-compare -Wno-implicit-function-declaration -Wno-return-type -Wno-pointer-bool-conversion -Wno-enum-conversion -Wno-int-conversion -Wno-switch")
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-format -Wno-inconsistent-missing-override -std=c++1z")
 endif ()

 if (USE_INTERNAL_BOOST_LIBRARY)
@ -54,6 +54,8 @@ if (USE_INTERNAL_UNWIND_LIBRARY)
 endif ()

 if (USE_INTERNAL_ZLIB_LIBRARY)
+    set (ZLIB_ENABLE_TESTS 0 CACHE INTERNAL "")
+    set (SKIP_INSTALL_ALL 1 CACHE INTERNAL "")
    set (ZLIB_COMPAT 1 CACHE INTERNAL "") # also enables WITH_GZFILEOP
    set (WITH_NATIVE_INSTRUCTIONS ${ARCH_NATIVE} CACHE INTERNAL "")
    if (OS_FREEBSD OR ARCH_I386)
@ -74,15 +76,15 @@ if (USE_INTERNAL_ZLIB_LIBRARY)
       target_compile_definitions (zlibstatic PUBLIC X86_64 UNALIGNED_OK)
    endif ()

-    set_target_properties(example PROPERTIES EXCLUDE_FROM_ALL 1)
-    if (TARGET example64)
-        set_target_properties(example64 PROPERTIES EXCLUDE_FROM_ALL 1)
-    endif ()
+    #set_target_properties(example PROPERTIES EXCLUDE_FROM_ALL 1)
+    #if (TARGET example64)
+    #    set_target_properties(example64 PROPERTIES EXCLUDE_FROM_ALL 1)
+    #endif ()

-    set_target_properties(minigzip PROPERTIES EXCLUDE_FROM_ALL 1)
-    if (TARGET minigzip64)
-        set_target_properties(minigzip64 PROPERTIES EXCLUDE_FROM_ALL 1)
-    endif ()
+    #set_target_properties(minigzip PROPERTIES EXCLUDE_FROM_ALL 1)
+    #if (TARGET minigzip64)
+    #    set_target_properties(minigzip64 PROPERTIES EXCLUDE_FROM_ALL 1)
+    #endif ()
 endif ()

 if (USE_INTERNAL_CCTZ_LIBRARY)
@ -105,8 +107,12 @@ if (USE_INTERNAL_SSL_LIBRARY)
    if (NOT MAKE_STATIC_LIBRARIES)
        set (BUILD_SHARED 1)
    endif ()
-    set (USE_SHARED ${USE_STATIC_LIBRARIES})
-    set (LIBRESSL_SKIP_INSTALL 1)
+
+    # By default, ${CMAKE_INSTALL_PREFIX}/etc/ssl is selected - that is not what we need.
+    # We need to use system wide ssl directory.
+    set (OPENSSLDIR "/etc/ssl")
+
+    set (LIBRESSL_SKIP_INSTALL 1 CACHE INTERNAL "")
    add_subdirectory (ssl)
    target_include_directories(${OPENSSL_CRYPTO_LIBRARY} SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR})
    target_include_directories(${OPENSSL_SSL_LIBRARY} SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR})
@ -125,6 +131,10 @@ if (USE_INTERNAL_RDKAFKA_LIBRARY)
    target_include_directories(rdkafka BEFORE PRIVATE ${OPENSSL_INCLUDE_DIR})
 endif ()

+if (USE_RDKAFKA)
+    add_subdirectory (cppkafka-cmake)
+endif()
+
 if (ENABLE_ODBC AND USE_INTERNAL_ODBC_LIBRARY)
    add_subdirectory (unixodbc-cmake)
 endif ()
@ -139,6 +149,7 @@ if (USE_INTERNAL_CAPNP_LIBRARY)
 endif ()

 if (USE_INTERNAL_POCO_LIBRARY)
+    set (POCO_VERBOSE_MESSAGES 0 CACHE INTERNAL "")
    set (save_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
    set (save_CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
    set (_save ${ENABLE_TESTS})
@ -160,13 +171,16 @@ if (USE_INTERNAL_POCO_LIBRARY)
    endif ()
 endif ()

-if (USE_INTERNAL_GTEST_LIBRARY)
+if(USE_INTERNAL_GTEST_LIBRARY)
    # Google Test from sources
    add_subdirectory(${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest ${CMAKE_CURRENT_BINARY_DIR}/googletest)
    # avoid problems with <regexp.h>
    target_compile_definitions (gtest INTERFACE GTEST_HAS_POSIX_RE=0)
    target_include_directories (gtest SYSTEM INTERFACE ${ClickHouse_SOURCE_DIR}/contrib/googletest/include)
-endif ()
+elseif(GTEST_SRC_DIR)
+    add_subdirectory(${GTEST_SRC_DIR}/googletest ${CMAKE_CURRENT_BINARY_DIR}/googletest)
+    target_compile_definitions(gtest INTERFACE GTEST_HAS_POSIX_RE=0)
+endif()

 if (USE_INTERNAL_LLVM_LIBRARY)
    file(GENERATE OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/empty.cpp CONTENT " ")
@ -201,13 +215,14 @@ if (USE_INTERNAL_LIBXML2_LIBRARY)
    add_subdirectory(libxml2-cmake)
 endif ()

+if (USE_INTERNAL_PROTOBUF_LIBRARY)
+    set(protobuf_BUILD_TESTS OFF CACHE INTERNAL "" FORCE)
+    set(protobuf_BUILD_SHARED_LIBS OFF CACHE INTERNAL "" FORCE)
+    set(protobuf_WITH_ZLIB 0 CACHE INTERNAL "" FORCE) # actually will use zlib, but skip find
+    add_subdirectory(protobuf/cmake)
+endif ()
+
 if (USE_INTERNAL_HDFS3_LIBRARY)
-    include(${ClickHouse_SOURCE_DIR}/cmake/find_protobuf.cmake)
-    if (USE_INTERNAL_PROTOBUF_LIBRARY)
-        set(protobuf_BUILD_TESTS OFF CACHE INTERNAL "" FORCE)
-        set(protobuf_BUILD_SHARED_LIBS OFF CACHE INTERNAL "" FORCE)
-        add_subdirectory(protobuf/cmake)
-    endif ()
    add_subdirectory(libhdfs3-cmake)
 endif ()

--- a/contrib/cppkafka
+++ b/contrib/cppkafka
@ -0,0 +1 @@
+Subproject commit 520465510efef7704346cf8d140967c4abb057c1
--- a/contrib/cppkafka-cmake/CMakeLists.txt
+++ b/contrib/cppkafka-cmake/CMakeLists.txt
@ -0,0 +1,31 @@
+set(CPPKAFKA_DIR ${CMAKE_SOURCE_DIR}/contrib/cppkafka)
+
+set(SRCS
+  ${CPPKAFKA_DIR}/src/configuration.cpp
+  ${CPPKAFKA_DIR}/src/topic_configuration.cpp
+  ${CPPKAFKA_DIR}/src/configuration_option.cpp
+  ${CPPKAFKA_DIR}/src/exceptions.cpp
+  ${CPPKAFKA_DIR}/src/topic.cpp
+  ${CPPKAFKA_DIR}/src/buffer.cpp
+  ${CPPKAFKA_DIR}/src/queue.cpp
+  ${CPPKAFKA_DIR}/src/message.cpp
+  ${CPPKAFKA_DIR}/src/message_timestamp.cpp
+  ${CPPKAFKA_DIR}/src/message_internal.cpp
+  ${CPPKAFKA_DIR}/src/topic_partition.cpp
+  ${CPPKAFKA_DIR}/src/topic_partition_list.cpp
+  ${CPPKAFKA_DIR}/src/metadata.cpp
+  ${CPPKAFKA_DIR}/src/group_information.cpp
+  ${CPPKAFKA_DIR}/src/error.cpp
+  ${CPPKAFKA_DIR}/src/event.cpp
+
+  ${CPPKAFKA_DIR}/src/kafka_handle_base.cpp
+  ${CPPKAFKA_DIR}/src/producer.cpp
+  ${CPPKAFKA_DIR}/src/consumer.cpp
+)
+
+add_library(cppkafka ${LINK_MODE} ${SRCS})
+
+target_link_libraries(cppkafka PRIVATE ${RDKAFKA_LIBRARY})
+target_include_directories(cppkafka PRIVATE ${CPPKAFKA_DIR}/include/cppkafka)
+target_include_directories(cppkafka PRIVATE ${Boost_INCLUDE_DIRS})
+target_include_directories(cppkafka SYSTEM PUBLIC ${CPPKAFKA_DIR}/include)
--- a/contrib/libdivide/libdivide.h
+++ b/contrib/libdivide/libdivide.h
@ -341,6 +341,13 @@ static inline __m128i libdivide_get_0000FFFF(void) {
 #pragma clang diagnostic pop
 #endif

+/// This is a bug in gcc-8, _MM_SHUFFLE was forgotten, though in trunk it is ok https://github.com/gcc-mirror/gcc/blob/master/gcc/config/rs6000/xmmintrin.h#L61
+#if defined(__PPC__)
+#ifndef _MM_SHUFFLE
+#define _MM_SHUFFLE(w,x,y,z) (((w) << 6) | ((x) << 4) | ((y) << 2) | (z))
+#endif
+#endif
+
 static inline __m128i libdivide_s64_signbits(__m128i v) {
    //we want to compute v >> 63, that is, _mm_srai_epi64(v, 63).  But there is no 64 bit shift right arithmetic instruction in SSE2.  So we have to fake it by first duplicating the high 32 bit values, and then using a 32 bit shift.  Another option would be to use _mm_srli_epi64(v, 63) and then subtract that from 0, but that approach appears to be substantially slower for unknown reasons
    __m128i hiBitsDuped = _mm_shuffle_epi32(v, _MM_SHUFFLE(3, 3, 1, 1));
--- a/contrib/libhdfs3-cmake/CMake/CodeCoverage.cmake
+++ b/contrib/libhdfs3-cmake/CMake/CodeCoverage.cmake
@ -1,48 +0,0 @@
-# Check prereqs
-FIND_PROGRAM(GCOV_PATH gcov)
-FIND_PROGRAM(LCOV_PATH lcov)
-FIND_PROGRAM(GENHTML_PATH genhtml)
-
-IF(NOT GCOV_PATH)
-    MESSAGE(FATAL_ERROR "gcov not found! Aborting...")
-ENDIF(NOT GCOV_PATH)
-
-IF(NOT CMAKE_BUILD_TYPE STREQUAL Debug)
-    MESSAGE(WARNING "Code coverage results with an optimised (non-Debug) build may be misleading")
-ENDIF(NOT CMAKE_BUILD_TYPE STREQUAL Debug)
-
-#Setup compiler options
-ADD_DEFINITIONS(-fprofile-arcs -ftest-coverage)
-
-SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs ")
-SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fprofile-arcs ")
-
-IF(NOT LCOV_PATH)
-    MESSAGE(FATAL_ERROR "lcov not found! Aborting...")
-ENDIF(NOT LCOV_PATH)
-
-IF(NOT GENHTML_PATH)
-    MESSAGE(FATAL_ERROR "genhtml not found! Aborting...")
-ENDIF(NOT GENHTML_PATH)
-
-#Setup target
-ADD_CUSTOM_TARGET(ShowCoverage
-    #Capturing lcov counters and generating report
-    COMMAND ${LCOV_PATH} --directory . --capture --output-file CodeCoverage.info
-    COMMAND ${LCOV_PATH} --remove CodeCoverage.info '${CMAKE_CURRENT_BINARY_DIR}/*' 'test/*' 'mock/*' '/usr/*' '/opt/*' '*ext/rhel5_x86_64*' '*ext/osx*' --output-file CodeCoverage.info.cleaned
-    COMMAND ${GENHTML_PATH} -o CodeCoverageReport CodeCoverage.info.cleaned
-)
-
-
-ADD_CUSTOM_TARGET(ShowAllCoverage
-    #Capturing lcov counters and generating report
-    COMMAND ${LCOV_PATH} -a CodeCoverage.info.cleaned -a CodeCoverage.info.cleaned_withoutHA -o AllCodeCoverage.info
-    COMMAND sed -e 's|/.*/src|${CMAKE_SOURCE_DIR}/src|' -ig AllCodeCoverage.info
-    COMMAND ${GENHTML_PATH} -o AllCodeCoverageReport AllCodeCoverage.info
-)
-
-ADD_CUSTOM_TARGET(ResetCoverage
-    #Cleanup lcov
-    COMMAND ${LCOV_PATH} --directory . --zerocounters
-)
-	
--- a/contrib/libhdfs3-cmake/CMake/FindBoost.cmake
+++ b/contrib/libhdfs3-cmake/CMake/FindBoost.cmake
--- a/contrib/libhdfs3-cmake/CMake/Options.cmake
+++ b/contrib/libhdfs3-cmake/CMake/Options.cmake
@ -1,38 +1,15 @@
-OPTION(ENABLE_COVERAGE "enable code coverage" OFF)
-OPTION(ENABLE_DEBUG "enable debug build" OFF)
 OPTION(ENABLE_SSE "enable SSE4.2 buildin function" ON)
-OPTION(ENABLE_FRAME_POINTER "enable frame pointer on 64bit system with flag -fno-omit-frame-pointer, on 32bit system, it is always enabled" ON)
-OPTION(ENABLE_LIBCPP "using libc++ instead of libstdc++, only valid for clang compiler" OFF)
-OPTION(ENABLE_BOOST "using boost instead of native compiler c++0x support" OFF)

 INCLUDE (CheckFunctionExists)
 CHECK_FUNCTION_EXISTS(dladdr HAVE_DLADDR)
 CHECK_FUNCTION_EXISTS(nanosleep HAVE_NANOSLEEP)

-IF(ENABLE_DEBUG STREQUAL ON)
-    SET(CMAKE_BUILD_TYPE Debug CACHE 
-        STRING "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel." FORCE)
-    SET(CMAKE_CXX_FLAGS_DEBUG "-g -O0" CACHE STRING "compiler flags for debug" FORCE)
-    SET(CMAKE_C_FLAGS_DEBUG "-g -O0" CACHE STRING "compiler flags for debug" FORCE)
-ELSE(ENABLE_DEBUG STREQUAL ON)
-    SET(CMAKE_BUILD_TYPE RelWithDebInfo CACHE 
-        STRING "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel." FORCE)
-ENDIF(ENABLE_DEBUG STREQUAL ON)
-
 SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-strict-aliasing")
 SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-strict-aliasing")

-IF(ENABLE_COVERAGE STREQUAL ON)
-    INCLUDE(CodeCoverage)
-ENDIF(ENABLE_COVERAGE STREQUAL ON)
-
-IF(ENABLE_FRAME_POINTER STREQUAL ON)
-    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
-ENDIF(ENABLE_FRAME_POINTER STREQUAL ON) 
-
-IF(ENABLE_SSE STREQUAL ON)
+IF(ENABLE_SSE STREQUAL ON AND NOT ARCH_PPC64LE AND NOT ARCH_AARCH64 AND NOT ARCH_ARM)
    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2")
-ENDIF(ENABLE_SSE STREQUAL ON) 
+ENDIF(ENABLE_SSE STREQUAL ON AND NOT ARCH_PPC64LE AND NOT ARCH_AARCH64 AND NOT ARCH_ARM)

 IF(NOT TEST_HDFS_PREFIX)
 SET(TEST_HDFS_PREFIX "./" CACHE STRING "default directory prefix used for test." FORCE)
@ -41,76 +18,7 @@ ENDIF(NOT TEST_HDFS_PREFIX)
 ADD_DEFINITIONS(-DTEST_HDFS_PREFIX="${TEST_HDFS_PREFIX}")
 ADD_DEFINITIONS(-D__STDC_FORMAT_MACROS)
 ADD_DEFINITIONS(-D_GNU_SOURCE)
-
-IF(OS_MACOSX AND CMAKE_COMPILER_IS_GNUCXX)
-    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-bind_at_load")
-ENDIF(OS_MACOSX AND CMAKE_COMPILER_IS_GNUCXX)
-
-IF(OS_LINUX)
-    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--export-dynamic")
-ENDIF(OS_LINUX)
-
-SET(BOOST_ROOT ${CMAKE_PREFIX_PATH})
-IF(ENABLE_BOOST STREQUAL ON)
-    MESSAGE(STATUS "using boost instead of native compiler c++0x support.")
-    FIND_PACKAGE(Boost 1.50 REQUIRED)
-    SET(NEED_BOOST true CACHE INTERNAL "boost is required")
-ELSE(ENABLE_BOOST STREQUAL ON)
-    SET(NEED_BOOST false CACHE INTERNAL "boost is required")
-ENDIF(ENABLE_BOOST STREQUAL ON)
-
-IF(CMAKE_COMPILER_IS_GNUCXX)
-    IF(ENABLE_LIBCPP STREQUAL ON)
-        MESSAGE(FATAL_ERROR "Unsupport using GCC compiler with libc++")
-    ENDIF(ENABLE_LIBCPP STREQUAL ON)
-    
-    IF((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR EQUAL 4) AND OS_MACOSX)
-    	SET(NEED_GCCEH true CACHE INTERNAL "Explicitly link with gcc_eh")
-    	MESSAGE(STATUS "link with -lgcc_eh for TLS")
-    ENDIF((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR EQUAL 4) AND OS_MACOSX)
-  
-    IF((GCC_COMPILER_VERSION_MAJOR LESS 4) OR ((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 4)))
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
-        IF(NOT ENABLE_BOOST STREQUAL ON)
-            MESSAGE(STATUS "gcc version is older than 4.6.0, boost is required.")
-            FIND_PACKAGE(Boost 1.50 REQUIRED)
-            SET(NEED_BOOST true CACHE INTERNAL "boost is required")
-        ENDIF(NOT ENABLE_BOOST STREQUAL ON)
-    ELSEIF((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 7))
-        IF(NOT ENABLE_BOOST STREQUAL ON)
-            MESSAGE(STATUS "gcc version is older than 4.6.0, boost is required.")
-            FIND_PACKAGE(Boost 1.50 REQUIRED)
-            SET(NEED_BOOST true CACHE INTERNAL "boost is required")
-        ENDIF(NOT ENABLE_BOOST STREQUAL ON)
-        MESSAGE(STATUS "adding c++0x support for gcc compiler")
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x")
-    ELSE((GCC_COMPILER_VERSION_MAJOR LESS 4) OR ((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 4)))
-        MESSAGE(STATUS "adding c++0x support for gcc compiler")
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x")
-    ENDIF((GCC_COMPILER_VERSION_MAJOR LESS 4) OR ((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 4)))
-    
-    IF(NEED_BOOST)
-        IF((Boost_MAJOR_VERSION LESS 1) OR ((Boost_MAJOR_VERSION EQUAL 1) AND (Boost_MINOR_VERSION LESS 50)))
-            MESSAGE(FATAL_ERROR "boost 1.50+ is required")
-        ENDIF()
-    ELSE(NEED_BOOST)
-        IF(HAVE_NANOSLEEP)
-           SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_NANOSLEEP") 
-        ELSE(HAVE_NANOSLEEP)
-            MESSAGE(FATAL_ERROR "nanosleep() is required")
-        ENDIF(HAVE_NANOSLEEP)
-    ENDIF(NEED_BOOST)
-    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
-ELSEIF(CMAKE_COMPILER_IS_CLANG)
-    MESSAGE(STATUS "adding c++0x support for clang compiler")
-    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x")
-    SET(CMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LANGUAGE_STANDARD "c++0x")
-    IF(ENABLE_LIBCPP STREQUAL ON)
-        MESSAGE(STATUS "using libc++ instead of libstdc++")
-        SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
-        SET(CMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LIBRARY "libc++")
-    ENDIF(ENABLE_LIBCPP STREQUAL ON)
-ENDIF(CMAKE_COMPILER_IS_GNUCXX)
+ADD_DEFINITIONS(-D_GLIBCXX_USE_NANOSLEEP)

 TRY_COMPILE(STRERROR_R_RETURN_INT
 	${CMAKE_CURRENT_BINARY_DIR}
@ -138,32 +46,8 @@ TRY_COMPILE(HAVE_NESTED_EXCEPTION
    CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'"
 	OUTPUT_VARIABLE OUTPUT)

-FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include <boost/chrono.hpp>")
-TRY_COMPILE(HAVE_BOOST_CHRONO
-    ${CMAKE_CURRENT_BINARY_DIR}
-    ${CMAKE_CURRENT_BINARY_DIR}/test.cpp 
-    CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'"
-    -DINCLUDE_DIRECTORIES=${Boost_INCLUDE_DIR}
-    OUTPUT_VARIABLE OUTPUT)
+SET(HAVE_BOOST_CHRONO 0)
+SET(HAVE_BOOST_ATOMIC 0)

-FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include <chrono>")
-TRY_COMPILE(HAVE_STD_CHRONO
-    ${CMAKE_CURRENT_BINARY_DIR}
-    ${CMAKE_CURRENT_BINARY_DIR}/test.cpp 
-    CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'"
-    OUTPUT_VARIABLE OUTPUT)
-
-FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include <boost/atomic.hpp>")
-TRY_COMPILE(HAVE_BOOST_ATOMIC
-    ${CMAKE_CURRENT_BINARY_DIR}
-    ${CMAKE_CURRENT_BINARY_DIR}/test.cpp 
-    CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'"
-    -DINCLUDE_DIRECTORIES=${Boost_INCLUDE_DIR}
-    OUTPUT_VARIABLE OUTPUT)
-    
-FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include <atomic>")
-TRY_COMPILE(HAVE_STD_ATOMIC
-    ${CMAKE_CURRENT_BINARY_DIR}
-    ${CMAKE_CURRENT_BINARY_DIR}/test.cpp 
-    CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'"
-    OUTPUT_VARIABLE OUTPUT)
+SET(HAVE_STD_CHRONO 1)
+SET(HAVE_STD_ATOMIC 1)
--- a/contrib/libmetrohash/CMakeLists.txt
+++ b/contrib/libmetrohash/CMakeLists.txt
@ -1,5 +1,5 @@
 if (HAVE_SSE42) # Not used. Pretty easy to port.
-    set (SOURCES_SSE42_ONLY src/metrohash128crc.cpp)
+    set (SOURCES_SSE42_ONLY src/metrohash128crc.cpp src/metrohash128crc.h)
 endif ()

 add_library(metrohash
--- a/contrib/libmetrohash/LICENSE
+++ b/contrib/libmetrohash/LICENSE
@ -1,22 +1,201 @@
-The MIT License (MIT)
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/

-Copyright (c) 2015 J. Andrew Rogers
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION

-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
+   1. Definitions.

-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.

-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.

+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
--- a/contrib/libmetrohash/README.md
+++ b/contrib/libmetrohash/README.md
@ -5,12 +5,44 @@ MetroHash is a set of state-of-the-art hash functions for *non-cryptographic* us
 * Fastest general-purpose functions for bulk hashing. 
 * Fastest general-purpose functions for small, variable length keys. 
 * Robust statistical bias profile, similar to the MD5 cryptographic hash.
+* Hashes can be constructed incrementally (**new**)
 * 64-bit, 128-bit, and 128-bit CRC variants currently available.
 * Optimized for modern x86-64 microarchitectures.
 * Elegant, compact, readable functions.
 
 You can read more about the design and history [here](http://www.jandrewrogers.com/2015/05/27/metrohash/).

+## News
+
+### 23 October 2018
+
+The project has been re-licensed under Apache License v2.0. The purpose of this license change is consistency with the imminent release of MetroHash v2.0, which is also licensed under the Apache license.
+
+### 27 July 2015
+
+Two new 64-bit and 128-bit algorithms add the ability to construct hashes incrementally. In addition to supporting incremental construction, the algorithms are slightly superior to the prior versions. 
+
+A big change is that these new algorithms are implemented as C++ classes that support both incremental and stateless hashing. These classes also have a static method for verifying the implementation against the test vectors built into the classes. Implementations are now fully contained by their respective headers e.g. "metrohash128.h".
+
+*Note: an incremental version of the 128-bit CRC version is on its way but is not included in this push.*
+
+**Usage Example For Stateless Hashing**
+
+`MetroHash128::Hash(key, key_length, hash_ptr, seed)`
+
+**Usage Example For Incremental Hashing**
+
+`MetroHash128 hasher;`  
+`hasher.Update(partial_key, partial_key_length);`  
+`...`  
+`hasher.Update(partial_key, partial_key_length);`  
+`hasher.Finalize(hash_ptr);`  
+
+An `Initialize(seed)` method allows the hasher objects to be reused.
+
+
+### 27 May 2015
+
 Six hash functions have been included in the initial release:

 * 64-bit hash functions, "metrohash64_1" and "metrohash64_2"
--- a/contrib/libmetrohash/VERSION
+++ b/contrib/libmetrohash/VERSION
@ -1,7 +1,4 @@
-origin: git@github.com:jandrewrogers/MetroHash.git
-commit d9dee18a54a8a6766e24c1950b814ac7ca9d1a89
-Merge: 761e8a4 3d06b24
+origin: https://github.com/jandrewrogers/MetroHash.git
+commit 690a521d9beb2e1050cc8f273fdabc13b31bf8f6 tag: v1.1.3
 Author: J. Andrew Rogers <andrew@jarbox.org>
-Date:   Sat Jun 6 16:12:06 2015 -0700
-
-    modified README
+Date:   Tue Oct 23 09:49:53 2018 -0700
--- a/contrib/libmetrohash/src/metrohash.h
+++ b/contrib/libmetrohash/src/metrohash.h
@ -1,73 +1,24 @@
 // metrohash.h
 //
-// The MIT License (MIT)
+// Copyright 2015-2018 J. Andrew Rogers
 //
-// Copyright (c) 2015 J. Andrew Rogers
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+//     http://www.apache.org/licenses/LICENSE-2.0
 //
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.

 #ifndef METROHASH_METROHASH_H
 #define METROHASH_METROHASH_H

-#include <stdint.h>
-#include <string.h>
-
-// MetroHash 64-bit hash functions
-void metrohash64_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out);
-void metrohash64_2(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out);
-
-// MetroHash 128-bit hash functions
-void metrohash128_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out);
-void metrohash128_2(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out);
-
-// MetroHash 128-bit hash functions using CRC instruction
-void metrohash128crc_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out);
-void metrohash128crc_2(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out);
-
-
-/* rotate right idiom recognized by compiler*/
-inline static uint64_t rotate_right(uint64_t v, unsigned k)
-{
-    return (v >> k) | (v << (64 - k));
-}
-
-// unaligned reads, fast and safe on Nehalem and later microarchitectures
-inline static uint64_t read_u64(const void * const ptr)
-{
-    return static_cast<uint64_t>(*reinterpret_cast<const uint64_t*>(ptr));
-}
-
-inline static uint64_t read_u32(const void * const ptr)
-{
-    return static_cast<uint64_t>(*reinterpret_cast<const uint32_t*>(ptr));
-}
-
-inline static uint64_t read_u16(const void * const ptr)
-{
-    return static_cast<uint64_t>(*reinterpret_cast<const uint16_t*>(ptr));
-}
-
-inline static uint64_t read_u8 (const void * const ptr)
-{
-    return static_cast<uint64_t>(*reinterpret_cast<const uint8_t *>(ptr));
-}
-
+#include "metrohash64.h"
+#include "metrohash128.h"
+#include "metrohash128crc.h"

 #endif // #ifndef METROHASH_METROHASH_H
--- a/contrib/libmetrohash/src/metrohash128.cpp
+++ b/contrib/libmetrohash/src/metrohash128.cpp
@ -1,29 +1,260 @@
 // metrohash128.cpp
 //
-// The MIT License (MIT)
+// Copyright 2015-2018 J. Andrew Rogers
 //
-// Copyright (c) 2015 J. Andrew Rogers
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+//     http://www.apache.org/licenses/LICENSE-2.0
 //
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string.h>
+#include "platform.h"
+#include "metrohash128.h"
+
+const char * MetroHash128::test_string = "012345678901234567890123456789012345678901234567890123456789012";
+
+const uint8_t MetroHash128::test_seed_0[16] =   {
+                                                0xC7, 0x7C, 0xE2, 0xBF, 0xA4, 0xED, 0x9F, 0x9B,
+                                                0x05, 0x48, 0xB2, 0xAC, 0x50, 0x74, 0xA2, 0x97
+                                                };
+
+const uint8_t MetroHash128::test_seed_1[16] =   {
+                                                0x45, 0xA3, 0xCD, 0xB8, 0x38, 0x19, 0x9D, 0x7F,
+                                                0xBD, 0xD6, 0x8D, 0x86, 0x7A, 0x14, 0xEC, 0xEF
+                                                };
+
+
+
+MetroHash128::MetroHash128(const uint64_t seed)
+{
+    Initialize(seed);
+}
+
+
+void MetroHash128::Initialize(const uint64_t seed)
+{
+    // initialize internal hash registers
+    state.v[0] = (static_cast<uint64_t>(seed) - k0) * k3;
+    state.v[1] = (static_cast<uint64_t>(seed) + k1) * k2;
+    state.v[2] = (static_cast<uint64_t>(seed) + k0) * k2;
+    state.v[3] = (static_cast<uint64_t>(seed) - k1) * k3;
+
+    // initialize total length of input
+    bytes = 0;
+}
+
+
+void MetroHash128::Update(const uint8_t * const buffer, const uint64_t length)
+{
+    const uint8_t * ptr = reinterpret_cast<const uint8_t*>(buffer);
+    const uint8_t * const end = ptr + length;
+
+    // input buffer may be partially filled
+    if (bytes % 32)
+    {
+        uint64_t fill = 32 - (bytes % 32);
+        if (fill > length)
+            fill = length;
+
+        memcpy(input.b + (bytes % 32), ptr, static_cast<size_t>(fill));
+        ptr   += fill;
+        bytes += fill;
+        
+        // input buffer is still partially filled
+        if ((bytes % 32) != 0) return;
+
+        // process full input buffer
+        state.v[0] += read_u64(&input.b[ 0]) * k0; state.v[0] = rotate_right(state.v[0],29) + state.v[2];
+        state.v[1] += read_u64(&input.b[ 8]) * k1; state.v[1] = rotate_right(state.v[1],29) + state.v[3];
+        state.v[2] += read_u64(&input.b[16]) * k2; state.v[2] = rotate_right(state.v[2],29) + state.v[0];
+        state.v[3] += read_u64(&input.b[24]) * k3; state.v[3] = rotate_right(state.v[3],29) + state.v[1];
+    }
+    
+    // bulk update
+    bytes += (end - ptr);
+    while (ptr <= (end - 32))
+    {
+        // process directly from the source, bypassing the input buffer
+        state.v[0] += read_u64(ptr) * k0; ptr += 8; state.v[0] = rotate_right(state.v[0],29) + state.v[2];
+        state.v[1] += read_u64(ptr) * k1; ptr += 8; state.v[1] = rotate_right(state.v[1],29) + state.v[3];
+        state.v[2] += read_u64(ptr) * k2; ptr += 8; state.v[2] = rotate_right(state.v[2],29) + state.v[0];
+        state.v[3] += read_u64(ptr) * k3; ptr += 8; state.v[3] = rotate_right(state.v[3],29) + state.v[1];
+    }
+    
+    // store remaining bytes in input buffer
+    if (ptr < end)
+        memcpy(input.b, ptr, end - ptr);
+}
+
+
+void MetroHash128::Finalize(uint8_t * const hash)
+{
+    // finalize bulk loop, if used
+    if (bytes >= 32)
+    {
+        state.v[2] ^= rotate_right(((state.v[0] + state.v[3]) * k0) + state.v[1], 21) * k1;
+        state.v[3] ^= rotate_right(((state.v[1] + state.v[2]) * k1) + state.v[0], 21) * k0;
+        state.v[0] ^= rotate_right(((state.v[0] + state.v[2]) * k0) + state.v[3], 21) * k1;
+        state.v[1] ^= rotate_right(((state.v[1] + state.v[3]) * k1) + state.v[2], 21) * k0;
+    }
+    
+    // process any bytes remaining in the input buffer
+    const uint8_t * ptr = reinterpret_cast<const uint8_t*>(input.b);
+    const uint8_t * const end = ptr + (bytes % 32);
+    
+    if ((end - ptr) >= 16)
+    {
+        state.v[0] += read_u64(ptr) * k2; ptr += 8; state.v[0] = rotate_right(state.v[0],33) * k3;
+        state.v[1] += read_u64(ptr) * k2; ptr += 8; state.v[1] = rotate_right(state.v[1],33) * k3;
+        state.v[0] ^= rotate_right((state.v[0] * k2) + state.v[1], 45) * k1;
+        state.v[1] ^= rotate_right((state.v[1] * k3) + state.v[0], 45) * k0;
+    }
+
+    if ((end - ptr) >= 8)
+    {
+        state.v[0] += read_u64(ptr) * k2; ptr += 8; state.v[0] = rotate_right(state.v[0],33) * k3;
+        state.v[0] ^= rotate_right((state.v[0] * k2) + state.v[1], 27) * k1;
+    }
+
+    if ((end - ptr) >= 4)
+    {
+        state.v[1] += read_u32(ptr) * k2; ptr += 4; state.v[1] = rotate_right(state.v[1],33) * k3;
+        state.v[1] ^= rotate_right((state.v[1] * k3) + state.v[0], 46) * k0;
+    }
+
+    if ((end - ptr) >= 2)
+    {
+        state.v[0] += read_u16(ptr) * k2; ptr += 2; state.v[0] = rotate_right(state.v[0],33) * k3;
+        state.v[0] ^= rotate_right((state.v[0] * k2) + state.v[1], 22) * k1;
+    }
+
+    if ((end - ptr) >= 1)
+    {
+        state.v[1] += read_u8 (ptr) * k2; state.v[1] = rotate_right(state.v[1],33) * k3;
+        state.v[1] ^= rotate_right((state.v[1] * k3) + state.v[0], 58) * k0;
+    }
+    
+    state.v[0] += rotate_right((state.v[0] * k0) + state.v[1], 13);
+    state.v[1] += rotate_right((state.v[1] * k1) + state.v[0], 37);
+    state.v[0] += rotate_right((state.v[0] * k2) + state.v[1], 13);
+    state.v[1] += rotate_right((state.v[1] * k3) + state.v[0], 37);
+
+    bytes = 0;
+
+    // do any endian conversion here
+
+    memcpy(hash, state.v, 16);
+}
+
+
+void MetroHash128::Hash(const uint8_t * buffer, const uint64_t length, uint8_t * const hash, const uint64_t seed)
+{
+    const uint8_t * ptr = reinterpret_cast<const uint8_t*>(buffer);
+    const uint8_t * const end = ptr + length;
+
+    uint64_t v[4];
+
+    v[0] = (static_cast<uint64_t>(seed) - k0) * k3;
+    v[1] = (static_cast<uint64_t>(seed) + k1) * k2;
+
+    if (length >= 32)
+    {
+        v[2] = (static_cast<uint64_t>(seed) + k0) * k2;
+        v[3] = (static_cast<uint64_t>(seed) - k1) * k3;
+
+        do
+        {
+            v[0] += read_u64(ptr) * k0; ptr += 8; v[0] = rotate_right(v[0],29) + v[2];
+            v[1] += read_u64(ptr) * k1; ptr += 8; v[1] = rotate_right(v[1],29) + v[3];
+            v[2] += read_u64(ptr) * k2; ptr += 8; v[2] = rotate_right(v[2],29) + v[0];
+            v[3] += read_u64(ptr) * k3; ptr += 8; v[3] = rotate_right(v[3],29) + v[1];
+        }
+        while (ptr <= (end - 32));
+
+        v[2] ^= rotate_right(((v[0] + v[3]) * k0) + v[1], 21) * k1;
+        v[3] ^= rotate_right(((v[1] + v[2]) * k1) + v[0], 21) * k0;
+        v[0] ^= rotate_right(((v[0] + v[2]) * k0) + v[3], 21) * k1;
+        v[1] ^= rotate_right(((v[1] + v[3]) * k1) + v[2], 21) * k0;
+    }
+
+    if ((end - ptr) >= 16)
+    {
+        v[0] += read_u64(ptr) * k2; ptr += 8; v[0] = rotate_right(v[0],33) * k3;
+        v[1] += read_u64(ptr) * k2; ptr += 8; v[1] = rotate_right(v[1],33) * k3;
+        v[0] ^= rotate_right((v[0] * k2) + v[1], 45) * k1;
+        v[1] ^= rotate_right((v[1] * k3) + v[0], 45) * k0;
+    }
+
+    if ((end - ptr) >= 8)
+    {
+        v[0] += read_u64(ptr) * k2; ptr += 8; v[0] = rotate_right(v[0],33) * k3;
+        v[0] ^= rotate_right((v[0] * k2) + v[1], 27) * k1;
+    }
+
+    if ((end - ptr) >= 4)
+    {
+        v[1] += read_u32(ptr) * k2; ptr += 4; v[1] = rotate_right(v[1],33) * k3;
+        v[1] ^= rotate_right((v[1] * k3) + v[0], 46) * k0;
+    }
+
+    if ((end - ptr) >= 2)
+    {
+        v[0] += read_u16(ptr) * k2; ptr += 2; v[0] = rotate_right(v[0],33) * k3;
+        v[0] ^= rotate_right((v[0] * k2) + v[1], 22) * k1;
+    }
+
+    if ((end - ptr) >= 1)
+    {
+        v[1] += read_u8 (ptr) * k2; v[1] = rotate_right(v[1],33) * k3;
+        v[1] ^= rotate_right((v[1] * k3) + v[0], 58) * k0;
+    }
+
+    v[0] += rotate_right((v[0] * k0) + v[1], 13);
+    v[1] += rotate_right((v[1] * k1) + v[0], 37);
+    v[0] += rotate_right((v[0] * k2) + v[1], 13);
+    v[1] += rotate_right((v[1] * k3) + v[0], 37);
+
+    // do any endian conversion here
+
+    memcpy(hash, v, 16);
+}
+
+
+bool MetroHash128::ImplementationVerified()
+{
+    uint8_t hash[16];
+    const uint8_t * key = reinterpret_cast<const uint8_t *>(MetroHash128::test_string);
+
+    // verify one-shot implementation
+    MetroHash128::Hash(key, strlen(MetroHash128::test_string), hash, 0);
+    if (memcmp(hash, MetroHash128::test_seed_0, 16) != 0) return false;
+
+    MetroHash128::Hash(key, strlen(MetroHash128::test_string), hash, 1);
+    if (memcmp(hash, MetroHash128::test_seed_1, 16) != 0) return false;
+
+    // verify incremental implementation
+    MetroHash128 metro;
+    
+    metro.Initialize(0);
+    metro.Update(reinterpret_cast<const uint8_t *>(MetroHash128::test_string), strlen(MetroHash128::test_string));
+    metro.Finalize(hash);
+    if (memcmp(hash, MetroHash128::test_seed_0, 16) != 0) return false;
+
+    metro.Initialize(1);
+    metro.Update(reinterpret_cast<const uint8_t *>(MetroHash128::test_string), strlen(MetroHash128::test_string));
+    metro.Finalize(hash);
+    if (memcmp(hash, MetroHash128::test_seed_1, 16) != 0) return false;
+
+    return true;
+}

-#include "metrohash.h"

 void metrohash128_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out)
 {
@ -97,6 +328,8 @@ void metrohash128_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t *
    v[0] += rotate_right((v[0] * k2) + v[1], 13);
    v[1] += rotate_right((v[1] * k3) + v[0], 37);

+    // do any endian conversion here
+
    memcpy(out, v, 16);
 }

@ -173,6 +406,8 @@ void metrohash128_2(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t *
    v[0] += rotate_right((v[0] * k2) + v[1], 33);
    v[1] += rotate_right((v[1] * k3) + v[0], 33);

+    // do any endian conversion here
+
    memcpy(out, v, 16);
 }

--- a/contrib/libmetrohash/src/metrohash128.h
+++ b/contrib/libmetrohash/src/metrohash128.h
@ -0,0 +1,72 @@
+// metrohash128.h
+//
+// Copyright 2015-2018 J. Andrew Rogers
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef METROHASH_METROHASH_128_H
+#define METROHASH_METROHASH_128_H
+
+#include <stdint.h>
+
+class MetroHash128
+{
+public:
+    static const uint32_t bits = 128;
+
+    // Constructor initializes the same as Initialize()
+    MetroHash128(const uint64_t seed=0);
+    
+    // Initializes internal state for new hash with optional seed
+    void Initialize(const uint64_t seed=0);
+    
+    // Update the hash state with a string of bytes. If the length
+    // is sufficiently long, the implementation switches to a bulk
+    // hashing algorithm directly on the argument buffer for speed.
+    void Update(const uint8_t * buffer, const uint64_t length);
+    
+    // Constructs the final hash and writes it to the argument buffer.
+    // After a hash is finalized, this instance must be Initialized()-ed
+    // again or the behavior of Update() and Finalize() is undefined.
+    void Finalize(uint8_t * const hash);
+    
+    // A non-incremental function implementation. This can be significantly
+    // faster than the incremental implementation for some usage patterns.
+    static void Hash(const uint8_t * buffer, const uint64_t length, uint8_t * const hash, const uint64_t seed=0);
+
+    // Does implementation correctly execute test vectors?
+    static bool ImplementationVerified();
+
+    // test vectors -- Hash(test_string, seed=0) => test_seed_0
+    static const char * test_string;
+    static const uint8_t test_seed_0[16];
+    static const uint8_t test_seed_1[16];
+
+private:
+    static const uint64_t k0 = 0xC83A91E1;
+    static const uint64_t k1 = 0x8648DBDB;
+    static const uint64_t k2 = 0x7BDEC03B;
+    static const uint64_t k3 = 0x2F5870A5;
+    
+    struct { uint64_t v[4]; } state;
+    struct { uint8_t b[32]; } input;
+    uint64_t bytes;
+};
+
+
+// Legacy 128-bit hash functions -- do not use
+void metrohash128_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out);
+void metrohash128_2(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out);
+
+
+#endif // #ifndef METROHASH_METROHASH_128_H
--- a/contrib/libmetrohash/src/metrohash128crc.cpp
+++ b/contrib/libmetrohash/src/metrohash128crc.cpp
@ -1,31 +1,24 @@
 // metrohash128crc.cpp
 //
-// The MIT License (MIT)
+// Copyright 2015-2018 J. Andrew Rogers
 //
-// Copyright (c) 2015 J. Andrew Rogers
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+//     http://www.apache.org/licenses/LICENSE-2.0
 //
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.


-#include "metrohash.h"
 #include <nmmintrin.h>
+#include <string.h>
+#include "metrohash.h"
+#include "platform.h"


 void metrohash128crc_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out)
--- a/contrib/libmetrohash/src/metrohash128crc.h
+++ b/contrib/libmetrohash/src/metrohash128crc.h
@ -0,0 +1,27 @@
+// metrohash128crc.h
+//
+// Copyright 2015-2018 J. Andrew Rogers
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef METROHASH_METROHASH_128_CRC_H
+#define METROHASH_METROHASH_128_CRC_H
+
+#include <stdint.h>
+
+// Legacy 128-bit hash functions
+void metrohash128crc_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out);
+void metrohash128crc_2(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out);
+
+
+#endif // #ifndef METROHASH_METROHASH_128_CRC_H
--- a/contrib/libmetrohash/src/metrohash64.cpp
+++ b/contrib/libmetrohash/src/metrohash64.cpp
@ -1,29 +1,257 @@
 // metrohash64.cpp
 //
-// The MIT License (MIT)
+// Copyright 2015-2018 J. Andrew Rogers
 //
-// Copyright (c) 2015 J. Andrew Rogers
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+//     http://www.apache.org/licenses/LICENSE-2.0
 //
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "platform.h"
+#include "metrohash64.h"
+
+#include <cstring>
+
+const char * MetroHash64::test_string = "012345678901234567890123456789012345678901234567890123456789012";
+
+const uint8_t MetroHash64::test_seed_0[8] =   { 0x6B, 0x75, 0x3D, 0xAE, 0x06, 0x70, 0x4B, 0xAD };
+const uint8_t MetroHash64::test_seed_1[8] =   { 0x3B, 0x0D, 0x48, 0x1C, 0xF4, 0xB9, 0xB8, 0xDF };
+
+
+
+MetroHash64::MetroHash64(const uint64_t seed)
+{
+    Initialize(seed);
+}
+
+
+void MetroHash64::Initialize(const uint64_t seed)
+{
+    vseed = (static_cast<uint64_t>(seed) + k2) * k0;
+
+    // initialize internal hash registers
+    state.v[0] = vseed;
+    state.v[1] = vseed;
+    state.v[2] = vseed;
+    state.v[3] = vseed;
+
+    // initialize total length of input
+    bytes = 0;
+}
+
+
+void MetroHash64::Update(const uint8_t * const buffer, const uint64_t length)
+{
+    const uint8_t * ptr = reinterpret_cast<const uint8_t*>(buffer);
+    const uint8_t * const end = ptr + length;
+
+    // input buffer may be partially filled
+    if (bytes % 32)
+    {
+        uint64_t fill = 32 - (bytes % 32);
+        if (fill > length)
+            fill = length;
+
+        memcpy(input.b + (bytes % 32), ptr, static_cast<size_t>(fill));
+        ptr   += fill;
+        bytes += fill;
+        
+        // input buffer is still partially filled
+        if ((bytes % 32) != 0) return;
+
+        // process full input buffer
+        state.v[0] += read_u64(&input.b[ 0]) * k0; state.v[0] = rotate_right(state.v[0],29) + state.v[2];
+        state.v[1] += read_u64(&input.b[ 8]) * k1; state.v[1] = rotate_right(state.v[1],29) + state.v[3];
+        state.v[2] += read_u64(&input.b[16]) * k2; state.v[2] = rotate_right(state.v[2],29) + state.v[0];
+        state.v[3] += read_u64(&input.b[24]) * k3; state.v[3] = rotate_right(state.v[3],29) + state.v[1];
+    }
+    
+    // bulk update
+    bytes += static_cast<uint64_t>(end - ptr);
+    while (ptr <= (end - 32))
+    {
+        // process directly from the source, bypassing the input buffer
+        state.v[0] += read_u64(ptr) * k0; ptr += 8; state.v[0] = rotate_right(state.v[0],29) + state.v[2];
+        state.v[1] += read_u64(ptr) * k1; ptr += 8; state.v[1] = rotate_right(state.v[1],29) + state.v[3];
+        state.v[2] += read_u64(ptr) * k2; ptr += 8; state.v[2] = rotate_right(state.v[2],29) + state.v[0];
+        state.v[3] += read_u64(ptr) * k3; ptr += 8; state.v[3] = rotate_right(state.v[3],29) + state.v[1];
+    }
+    
+    // store remaining bytes in input buffer
+    if (ptr < end)
+        memcpy(input.b, ptr, static_cast<size_t>(end - ptr));
+}
+
+
+void MetroHash64::Finalize(uint8_t * const hash)
+{
+    // finalize bulk loop, if used
+    if (bytes >= 32)
+    {
+        state.v[2] ^= rotate_right(((state.v[0] + state.v[3]) * k0) + state.v[1], 37) * k1;
+        state.v[3] ^= rotate_right(((state.v[1] + state.v[2]) * k1) + state.v[0], 37) * k0;
+        state.v[0] ^= rotate_right(((state.v[0] + state.v[2]) * k0) + state.v[3], 37) * k1;
+        state.v[1] ^= rotate_right(((state.v[1] + state.v[3]) * k1) + state.v[2], 37) * k0;
+
+        state.v[0] = vseed + (state.v[0] ^ state.v[1]);
+    }
+    
+    // process any bytes remaining in the input buffer
+    const uint8_t * ptr = reinterpret_cast<const uint8_t*>(input.b);
+    const uint8_t * const end = ptr + (bytes % 32);
+    
+    if ((end - ptr) >= 16)
+    {
+        state.v[1]  = state.v[0] + (read_u64(ptr) * k2); ptr += 8; state.v[1] = rotate_right(state.v[1],29) * k3;
+        state.v[2]  = state.v[0] + (read_u64(ptr) * k2); ptr += 8; state.v[2] = rotate_right(state.v[2],29) * k3;
+        state.v[1] ^= rotate_right(state.v[1] * k0, 21) + state.v[2];
+        state.v[2] ^= rotate_right(state.v[2] * k3, 21) + state.v[1];
+        state.v[0] += state.v[2];
+    }
+
+    if ((end - ptr) >= 8)
+    {
+        state.v[0] += read_u64(ptr) * k3; ptr += 8;
+        state.v[0] ^= rotate_right(state.v[0], 55) * k1;
+    }
+
+    if ((end - ptr) >= 4)
+    {
+        state.v[0] += read_u32(ptr) * k3; ptr += 4;
+        state.v[0] ^= rotate_right(state.v[0], 26) * k1;
+    }
+
+    if ((end - ptr) >= 2)
+    {
+        state.v[0] += read_u16(ptr) * k3; ptr += 2;
+        state.v[0] ^= rotate_right(state.v[0], 48) * k1;
+    }
+
+    if ((end - ptr) >= 1)
+    {
+        state.v[0] += read_u8 (ptr) * k3;
+        state.v[0] ^= rotate_right(state.v[0], 37) * k1;
+    }
+    
+    state.v[0] ^= rotate_right(state.v[0], 28);
+    state.v[0] *= k0;
+    state.v[0] ^= rotate_right(state.v[0], 29);
+
+    bytes = 0;
+
+    // do any endian conversion here
+
+    memcpy(hash, state.v, 8);
+}
+
+
+void MetroHash64::Hash(const uint8_t * buffer, const uint64_t length, uint8_t * const hash, const uint64_t seed)
+{
+    const uint8_t * ptr = reinterpret_cast<const uint8_t*>(buffer);
+    const uint8_t * const end = ptr + length;
+
+    uint64_t h = (static_cast<uint64_t>(seed) + k2) * k0;
+
+    if (length >= 32)
+    {
+        uint64_t v[4];
+        v[0] = h;
+        v[1] = h;
+        v[2] = h;
+        v[3] = h;
+
+        do
+        {
+            v[0] += read_u64(ptr) * k0; ptr += 8; v[0] = rotate_right(v[0],29) + v[2];
+            v[1] += read_u64(ptr) * k1; ptr += 8; v[1] = rotate_right(v[1],29) + v[3];
+            v[2] += read_u64(ptr) * k2; ptr += 8; v[2] = rotate_right(v[2],29) + v[0];
+            v[3] += read_u64(ptr) * k3; ptr += 8; v[3] = rotate_right(v[3],29) + v[1];
+        }
+        while (ptr <= (end - 32));
+
+        v[2] ^= rotate_right(((v[0] + v[3]) * k0) + v[1], 37) * k1;
+        v[3] ^= rotate_right(((v[1] + v[2]) * k1) + v[0], 37) * k0;
+        v[0] ^= rotate_right(((v[0] + v[2]) * k0) + v[3], 37) * k1;
+        v[1] ^= rotate_right(((v[1] + v[3]) * k1) + v[2], 37) * k0;
+        h += v[0] ^ v[1];
+    }
+
+    if ((end - ptr) >= 16)
+    {
+        uint64_t v0 = h + (read_u64(ptr) * k2); ptr += 8; v0 = rotate_right(v0,29) * k3;
+        uint64_t v1 = h + (read_u64(ptr) * k2); ptr += 8; v1 = rotate_right(v1,29) * k3;
+        v0 ^= rotate_right(v0 * k0, 21) + v1;
+        v1 ^= rotate_right(v1 * k3, 21) + v0;
+        h += v1;
+    }
+
+    if ((end - ptr) >= 8)
+    {
+        h += read_u64(ptr) * k3; ptr += 8;
+        h ^= rotate_right(h, 55) * k1;
+    }
+
+    if ((end - ptr) >= 4)
+    {
+        h += read_u32(ptr) * k3; ptr += 4;
+        h ^= rotate_right(h, 26) * k1;
+    }
+
+    if ((end - ptr) >= 2)
+    {
+        h += read_u16(ptr) * k3; ptr += 2;
+        h ^= rotate_right(h, 48) * k1;
+    }
+
+    if ((end - ptr) >= 1)
+    {
+        h += read_u8 (ptr) * k3;
+        h ^= rotate_right(h, 37) * k1;
+    }
+
+    h ^= rotate_right(h, 28);
+    h *= k0;
+    h ^= rotate_right(h, 29);
+
+    memcpy(hash, &h, 8);
+}
+
+
+bool MetroHash64::ImplementationVerified()
+{
+    uint8_t hash[8];
+    const uint8_t * key = reinterpret_cast<const uint8_t *>(MetroHash64::test_string);
+
+    // verify one-shot implementation
+    MetroHash64::Hash(key, strlen(MetroHash64::test_string), hash, 0);
+    if (memcmp(hash, MetroHash64::test_seed_0, 8) != 0) return false;
+
+    MetroHash64::Hash(key, strlen(MetroHash64::test_string), hash, 1);
+    if (memcmp(hash, MetroHash64::test_seed_1, 8) != 0) return false;
+
+    // verify incremental implementation
+    MetroHash64 metro;
+    
+    metro.Initialize(0);
+    metro.Update(reinterpret_cast<const uint8_t *>(MetroHash64::test_string), strlen(MetroHash64::test_string));
+    metro.Finalize(hash);
+    if (memcmp(hash, MetroHash64::test_seed_0, 8) != 0) return false;
+
+    metro.Initialize(1);
+    metro.Update(reinterpret_cast<const uint8_t *>(MetroHash64::test_string), strlen(MetroHash64::test_string));
+    metro.Finalize(hash);
+    if (memcmp(hash, MetroHash64::test_seed_1, 8) != 0) return false;
+
+    return true;
+}

-#include "metrohash.h"

 void metrohash64_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out)
 {
--- a/contrib/libmetrohash/src/metrohash64.h
+++ b/contrib/libmetrohash/src/metrohash64.h
@ -0,0 +1,73 @@
+// metrohash64.h
+//
+// Copyright 2015-2018 J. Andrew Rogers
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef METROHASH_METROHASH_64_H
+#define METROHASH_METROHASH_64_H
+
+#include <stdint.h>
+
+class MetroHash64
+{
+public:
+    static const uint32_t bits = 64;
+
+    // Constructor initializes the same as Initialize()
+    MetroHash64(const uint64_t seed=0);
+    
+    // Initializes internal state for new hash with optional seed
+    void Initialize(const uint64_t seed=0);
+    
+    // Update the hash state with a string of bytes. If the length
+    // is sufficiently long, the implementation switches to a bulk
+    // hashing algorithm directly on the argument buffer for speed.
+    void Update(const uint8_t * buffer, const uint64_t length);
+    
+    // Constructs the final hash and writes it to the argument buffer.
+    // After a hash is finalized, this instance must be Initialized()-ed
+    // again or the behavior of Update() and Finalize() is undefined.
+    void Finalize(uint8_t * const hash);
+    
+    // A non-incremental function implementation. This can be significantly
+    // faster than the incremental implementation for some usage patterns.
+    static void Hash(const uint8_t * buffer, const uint64_t length, uint8_t * const hash, const uint64_t seed=0);
+
+    // Does implementation correctly execute test vectors?
+    static bool ImplementationVerified();
+
+    // test vectors -- Hash(test_string, seed=0) => test_seed_0
+    static const char * test_string;
+    static const uint8_t test_seed_0[8];
+    static const uint8_t test_seed_1[8];
+
+private:
+    static const uint64_t k0 = 0xD6D018F5;
+    static const uint64_t k1 = 0xA2AA033B;
+    static const uint64_t k2 = 0x62992FC1;
+    static const uint64_t k3 = 0x30BC5B29;
+    
+    struct { uint64_t v[4]; } state;
+    struct { uint8_t b[32]; } input;
+    uint64_t bytes;
+    uint64_t vseed;
+};
+
+
+// Legacy 64-bit hash functions -- do not use
+void metrohash64_1(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out);
+void metrohash64_2(const uint8_t * key, uint64_t len, uint32_t seed, uint8_t * out);
+
+
+#endif // #ifndef METROHASH_METROHASH_64_H
--- a/contrib/libmetrohash/src/platform.h
+++ b/contrib/libmetrohash/src/platform.h
@ -0,0 +1,50 @@
+// platform.h
+//
+// Copyright 2015-2018 J. Andrew Rogers
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef METROHASH_PLATFORM_H
+#define METROHASH_PLATFORM_H
+
+#include <stdint.h>
+
+// rotate right idiom recognized by most compilers
+inline static uint64_t rotate_right(uint64_t v, unsigned k)
+{
+    return (v >> k) | (v << (64 - k));
+}
+
+// unaligned reads, fast and safe on Nehalem and later microarchitectures
+inline static uint64_t read_u64(const void * const ptr)
+{
+    return static_cast<uint64_t>(*reinterpret_cast<const uint64_t*>(ptr));
+}
+
+inline static uint64_t read_u32(const void * const ptr)
+{
+    return static_cast<uint64_t>(*reinterpret_cast<const uint32_t*>(ptr));
+}
+
+inline static uint64_t read_u16(const void * const ptr)
+{
+    return static_cast<uint64_t>(*reinterpret_cast<const uint16_t*>(ptr));
+}
+
+inline static uint64_t read_u8 (const void * const ptr)
+{
+    return static_cast<uint64_t>(*reinterpret_cast<const uint8_t *>(ptr));
+}
+
+
+#endif // #ifndef METROHASH_PLATFORM_H
--- a/contrib/libmetrohash/src/testvector.h
+++ b/contrib/libmetrohash/src/testvector.h
@ -1,27 +1,18 @@
 // testvector.h
 //
-// The MIT License (MIT)
+// Copyright 2015-2018 J. Andrew Rogers
 //
-// Copyright (c) 2015 J. Andrew Rogers
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
 //
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
+//     http://www.apache.org/licenses/LICENSE-2.0
 //
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.

 #ifndef METROHASH_TESTVECTOR_H
 #define METROHASH_TESTVECTOR_H
@ -46,6 +37,8 @@ struct TestVectorData

 static const char * test_key_63 = "012345678901234567890123456789012345678901234567890123456789012";

+// The hash assumes a little-endian architecture. Treating the hash results
+// as an array of uint64_t should enable conversion for big-endian implementations.
 const TestVectorData TestVector [] = 
 {
 	// seed = 0
--- a/contrib/librdkafka
+++ b/contrib/librdkafka
@ -1 +1 @@
-Subproject commit 7478b5ef16aadd6543fe38bc6a2deb895c70da98
+Subproject commit 363dcad5a23dc29381cc626620e68ae418b3af19
--- a/contrib/librdkafka-cmake/CMakeLists.txt
+++ b/contrib/librdkafka-cmake/CMakeLists.txt
@ -1,60 +1,65 @@
 set(RDKAFKA_SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/librdkafka/src)

 set(SRCS
-${RDKAFKA_SOURCE_DIR}/crc32c.c
-${RDKAFKA_SOURCE_DIR}/rdaddr.c
-${RDKAFKA_SOURCE_DIR}/rdavl.c
-${RDKAFKA_SOURCE_DIR}/rdbuf.c
-${RDKAFKA_SOURCE_DIR}/rdcrc32.c
-${RDKAFKA_SOURCE_DIR}/rdkafka.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_assignor.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_broker.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_buf.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_cgrp.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_conf.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_event.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_feature.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_lz4.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_metadata.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_metadata_cache.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_msg.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_msgset_reader.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_msgset_writer.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_offset.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_op.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_partition.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_pattern.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_queue.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_range_assignor.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_request.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_roundrobin_assignor.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_sasl.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_plain.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_subscription.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_timer.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_topic.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_transport.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_interceptor.c
-${RDKAFKA_SOURCE_DIR}/rdkafka_header.c
-${RDKAFKA_SOURCE_DIR}/rdlist.c
-${RDKAFKA_SOURCE_DIR}/rdlog.c
-${RDKAFKA_SOURCE_DIR}/rdmurmur2.c
-${RDKAFKA_SOURCE_DIR}/rdports.c
-${RDKAFKA_SOURCE_DIR}/rdrand.c
-${RDKAFKA_SOURCE_DIR}/rdregex.c
-${RDKAFKA_SOURCE_DIR}/rdstring.c
-${RDKAFKA_SOURCE_DIR}/rdunittest.c
-${RDKAFKA_SOURCE_DIR}/rdvarint.c
-${RDKAFKA_SOURCE_DIR}/snappy.c
-${RDKAFKA_SOURCE_DIR}/tinycthread.c
-${RDKAFKA_SOURCE_DIR}/xxhash.c
-${RDKAFKA_SOURCE_DIR}/lz4.c
-${RDKAFKA_SOURCE_DIR}/lz4frame.c
-${RDKAFKA_SOURCE_DIR}/lz4hc.c
-${RDKAFKA_SOURCE_DIR}/rdgz.c
+  ${RDKAFKA_SOURCE_DIR}/crc32c.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_zstd.c
+  ${RDKAFKA_SOURCE_DIR}/rdaddr.c
+  ${RDKAFKA_SOURCE_DIR}/rdavl.c
+  ${RDKAFKA_SOURCE_DIR}/rdbuf.c
+  ${RDKAFKA_SOURCE_DIR}/rdcrc32.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_assignor.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_background.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_broker.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_buf.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_cgrp.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_conf.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_event.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_feature.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_idempotence.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_lz4.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_metadata.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_metadata_cache.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_msg.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_msgset_reader.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_msgset_writer.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_offset.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_op.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_partition.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_pattern.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_queue.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_range_assignor.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_request.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_roundrobin_assignor.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_sasl.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_plain.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_subscription.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_timer.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_topic.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_transport.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_interceptor.c
+  ${RDKAFKA_SOURCE_DIR}/rdkafka_header.c
+  ${RDKAFKA_SOURCE_DIR}/rdlist.c
+  ${RDKAFKA_SOURCE_DIR}/rdlog.c
+  ${RDKAFKA_SOURCE_DIR}/rdmurmur2.c
+  ${RDKAFKA_SOURCE_DIR}/rdports.c
+  ${RDKAFKA_SOURCE_DIR}/rdrand.c
+  ${RDKAFKA_SOURCE_DIR}/rdregex.c
+  ${RDKAFKA_SOURCE_DIR}/rdstring.c
+  ${RDKAFKA_SOURCE_DIR}/rdunittest.c
+  ${RDKAFKA_SOURCE_DIR}/rdvarint.c
+  ${RDKAFKA_SOURCE_DIR}/snappy.c
+  ${RDKAFKA_SOURCE_DIR}/tinycthread.c
+  ${RDKAFKA_SOURCE_DIR}/tinycthread_extra.c
+  ${RDKAFKA_SOURCE_DIR}/xxhash.c
+  ${RDKAFKA_SOURCE_DIR}/lz4.c
+  ${RDKAFKA_SOURCE_DIR}/lz4frame.c
+  ${RDKAFKA_SOURCE_DIR}/lz4hc.c
+  ${RDKAFKA_SOURCE_DIR}/rdgz.c
 )

 add_library(rdkafka ${LINK_MODE} ${SRCS})
-target_include_directories(rdkafka PRIVATE include)
-target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR})
-target_link_libraries(rdkafka PUBLIC ${ZLIB_LIBRARIES} ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY})
+target_include_directories(rdkafka SYSTEM PUBLIC include)
+target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR})         # Because weird logic with "include_next" is used.
+target_include_directories(rdkafka SYSTEM PRIVATE ${ZSTD_INCLUDE_DIR}/common)   # Because wrong path to "zstd_errors.h" is used.
+target_link_libraries(rdkafka PUBLIC ${ZLIB_LIBRARIES} ${ZSTD_LIBRARY} ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY})
--- a/contrib/librdkafka-cmake/config.h
+++ b/contrib/librdkafka-cmake/config.h
@ -1,4 +1,4 @@
-// Automatically generated by ./configure 
+// Automatically generated by ./configure
 #ifndef _CONFIG_H_
 #define _CONFIG_H_
 #define ARCH "x86_64"
@ -51,6 +51,8 @@
 //#define WITH_PLUGINS 1
 // zlib
 #define WITH_ZLIB 1
+// zstd
+#define WITH_ZSTD 1
 // WITH_SNAPPY
 #define WITH_SNAPPY 1
 // WITH_SOCKEM
@ -60,7 +62,9 @@
 // WITH_SASL_SCRAM
 //#define WITH_SASL_SCRAM 1
 // crc32chw
+#if !defined(__PPC__)
 #define WITH_CRC32C_HW 1
+#endif
 // regex
 #define HAVE_REGEX 1
 // strndup
@ -71,4 +75,8 @@
 #define HAVE_PTHREAD_SETNAME_GNU 1
 // python
 //#define HAVE_PYTHON 1
+// C11 threads
+#if (__STDC_VERSION__ >= 201112L) && !defined(__STDC_NO_THREADS__)
+#  define WITH_C11THREADS 1
+#endif
 #endif /* _CONFIG_H_ */
--- a/contrib/librdkafka-cmake/include/librdkafka/rdkafka.h
+++ b/contrib/librdkafka-cmake/include/librdkafka/rdkafka.h
@ -0,0 +1,5 @@
+#if __has_include(<rdkafka.h>) // maybe bundled
+#    include_next <rdkafka.h> // Y_IGNORE
+#else // system
+#    include_next <librdkafka/rdkafka.h>
+#endif
--- a/contrib/libunwind/CMakeLists.txt
+++ b/contrib/libunwind/CMakeLists.txt
@ -1,56 +1,110 @@
-
 enable_language(ASM)

-add_library(unwind
-src/mi/init.c
-src/mi/flush_cache.c
-src/mi/mempool.c
-src/mi/strerror.c
-src/x86_64/is_fpreg.c
-src/x86_64/regname.c
-src/mi/_ReadULEB.c
-src/mi/_ReadSLEB.c
-src/mi/backtrace.c
-src/mi/dyn-cancel.c
-src/mi/dyn-info-list.c
-src/mi/dyn-register.c
-src/mi/Ldyn-extract.c
-src/mi/Lfind_dynamic_proc_info.c
-src/mi/Lget_accessors.c
-src/mi/Lget_proc_info_by_ip.c
-src/mi/Lget_proc_name.c
-src/mi/Lput_dynamic_unwind_info.c
-src/mi/Ldestroy_addr_space.c
-src/mi/Lget_reg.c
-src/mi/Lset_reg.c
-src/mi/Lget_fpreg.c
-src/mi/Lset_fpreg.c
-src/mi/Lset_caching_policy.c
-src/x86_64/setcontext.S
-src/x86_64/Lcreate_addr_space.c
-src/x86_64/Lget_save_loc.c
-src/x86_64/Lglobal.c
-src/x86_64/Linit.c
-src/x86_64/Linit_local.c
-src/x86_64/Linit_remote.c
-src/x86_64/Lget_proc_info.c
-src/x86_64/Lregs.c
-src/x86_64/Lresume.c
-src/x86_64/Lstash_frame.c
-src/x86_64/Lstep.c
-src/x86_64/Ltrace.c
-src/x86_64/getcontext.S
-src/dwarf/Lexpr.c
-src/dwarf/Lfde.c
-src/dwarf/Lfind_proc_info-lsb.c
-src/dwarf/Lparser.c
-src/dwarf/Lpe.c
-src/dwarf/global.c
-src/elf64.c
+if (ARCH_PPC64LE)
+    add_library(unwind
+    src/mi/init.c
+    src/mi/flush_cache.c
+    src/mi/mempool.c
+    src/mi/strerror.c
+    src/mi/_ReadULEB.c
+    src/mi/_ReadSLEB.c
+    src/mi/backtrace.c
+    src/mi/dyn-cancel.c
+    src/mi/dyn-info-list.c
+    src/mi/dyn-register.c
+    src/mi/Ldyn-extract.c
+    src/mi/Lfind_dynamic_proc_info.c
+    src/mi/Lget_accessors.c
+    src/mi/Lget_proc_info_by_ip.c
+    src/mi/Lget_proc_name.c
+    src/mi/Lput_dynamic_unwind_info.c
+    src/mi/Ldestroy_addr_space.c
+    src/mi/Lget_reg.c
+    src/mi/Lset_reg.c
+    src/mi/Lget_fpreg.c
+    src/mi/Lset_fpreg.c
+    src/mi/Lset_caching_policy.c
+    src/dwarf/Lexpr.c
+    src/dwarf/Lfde.c
+    src/dwarf/Lfind_proc_info-lsb.c
+    src/dwarf/Lparser.c
+    src/dwarf/Lpe.c
+    src/dwarf/global.c
+    src/elf64.c
+    src/os-linux.c

-src/os-linux.c
-src/x86_64/Los-linux.c
-)
+    src/ppc64/is_fpreg.c
+    src/ppc64/regname.c
+    src/ppc64/get_func_addr.c
+    src/ppc/Linit_local.c
+    src/ppc/Linit_remote.c
+    src/ppc/Lis_signal_frame.c
+    src/ppc/longjmp.S
+    src/ppc/Lreg_states_iterate.c
+    src/ppc/siglongjmp.S
+    src/ppc64/setcontext.S
+    src/ppc64/Lcreate_addr_space.c
+    src/ppc64/Lglobal.c
+    src/ppc64/Linit.c
+    src/ppc64/Lreg_states_iterate.c
+    src/ppc64/Lregs.c
+    src/ppc64/Lresume.c
+    src/ppc64/Lstep.c
+    src/ppc64/regname.c
+    src/ppc64/setcontext.S
+    )
+else ()
+    add_library(unwind
+    src/mi/init.c
+    src/mi/flush_cache.c
+    src/mi/mempool.c
+    src/mi/strerror.c
+    src/mi/_ReadULEB.c
+    src/mi/_ReadSLEB.c
+    src/mi/backtrace.c
+    src/mi/dyn-cancel.c
+    src/mi/dyn-info-list.c
+    src/mi/dyn-register.c
+    src/mi/Ldyn-extract.c
+    src/mi/Lfind_dynamic_proc_info.c
+    src/mi/Lget_accessors.c
+    src/mi/Lget_proc_info_by_ip.c
+    src/mi/Lget_proc_name.c
+    src/mi/Lput_dynamic_unwind_info.c
+    src/mi/Ldestroy_addr_space.c
+    src/mi/Lget_reg.c
+    src/mi/Lset_reg.c
+    src/mi/Lget_fpreg.c
+    src/mi/Lset_fpreg.c
+    src/mi/Lset_caching_policy.c
+    src/dwarf/Lexpr.c
+    src/dwarf/Lfde.c
+    src/dwarf/Lfind_proc_info-lsb.c
+    src/dwarf/Lparser.c
+    src/dwarf/Lpe.c
+    src/dwarf/global.c
+    src/elf64.c
+    src/os-linux.c
+
+    src/x86_64/is_fpreg.c
+    src/x86_64/regname.c
+    src/x86_64/setcontext.S
+    src/x86_64/Lcreate_addr_space.c
+    src/x86_64/Lget_save_loc.c
+    src/x86_64/Lglobal.c
+    src/x86_64/Linit.c
+    src/x86_64/Linit_local.c
+    src/x86_64/Linit_remote.c
+    src/x86_64/Lget_proc_info.c
+    src/x86_64/Lregs.c
+    src/x86_64/Lresume.c
+    src/x86_64/Lstash_frame.c
+    src/x86_64/Lstep.c
+    src/x86_64/Ltrace.c
+    src/x86_64/getcontext.S
+    src/x86_64/Los-linux.c
+    )
+endif()

 find_file (HAVE_ATOMIC_OPS_H "atomic_ops.h")
 configure_file (config/config.h.in ${CMAKE_CURRENT_BINARY_DIR}/config/config.h)
--- a/contrib/libunwind/include/tdep-ppc64/libunwind_i.h
+++ b/contrib/libunwind/include/tdep-ppc64/libunwind_i.h
@ -37,6 +37,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.  */

 #include <stdlib.h>
 #include <libunwind.h>
+#include <libunwind-ppc64.h>

 #include "elf64.h"
 #include "mempool.h"
--- a/contrib/libunwind/src/ppc/Greg_states_iterate.c
+++ b/contrib/libunwind/src/ppc/Greg_states_iterate.c
@ -25,7 +25,7 @@ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.  */

-#include "unwind_i.h"
+#include "../ppc64/unwind_i.h"

 PROTECTED int
 unw_reg_states_iterate (unw_cursor_t *cursor,
--- a/contrib/mariadb-connector-c
+++ b/contrib/mariadb-connector-c
@ -1 +1 @@
-Subproject commit a0fd36cc5a5313414a5a2ebe9322577a29b4782a
+Subproject commit d85d0e98999cd9e28ceb66645999b4a9ce85370e
--- a/contrib/murmurhash/include/murmurhash2.h
+++ b/contrib/murmurhash/include/murmurhash2.h
@ -2,8 +2,7 @@
 // MurmurHash2 was written by Austin Appleby, and is placed in the public
 // domain. The author hereby disclaims copyright to this source code.

-#ifndef _MURMURHASH2_H_
-#define _MURMURHASH2_H_
+#pragma once

 //-----------------------------------------------------------------------------
 // Platform-specific functions and macros
@ -30,6 +29,3 @@ uint64_t MurmurHash64B      (const void * key, int len, uint64_t seed);
 uint32_t MurmurHash2A       (const void * key, int len, uint32_t seed);
 uint32_t MurmurHashNeutral2 (const void * key, int len, uint32_t seed);
 uint32_t MurmurHashAligned2 (const void * key, int len, uint32_t seed);
-
-#endif // _MURMURHASH2_H_
-
--- a/contrib/murmurhash/include/murmurhash3.h
+++ b/contrib/murmurhash/include/murmurhash3.h
@ -2,8 +2,7 @@
 // MurmurHash3 was written by Austin Appleby, and is placed in the public
 // domain. The author hereby disclaims copyright to this source code.

-#ifndef _MURMURHASH3_H_
-#define _MURMURHASH3_H_
+#pragma once

 //-----------------------------------------------------------------------------
 // Platform-specific functions and macros
@ -33,5 +32,3 @@ void MurmurHash3_x86_128 ( const void * key, int len, uint32_t seed, void * out
 void MurmurHash3_x64_128 ( const void * key, int len, uint32_t seed, void * out );

 //-----------------------------------------------------------------------------
-
-#endif // _MURMURHASH3_H_
--- a/contrib/pdqsort
+++ b/contrib/pdqsort
@ -0,0 +1 @@
+Subproject commit 08879029ab8dcb80a70142acb709e3df02de5d37
--- a/contrib/ssl
+++ b/contrib/ssl
@ -1 +1 @@
-Subproject commit dbbbcdbbd17785566f8f9c107b714f9e213d7293
+Subproject commit ba8de796195ff9d8bb0249ce289b83226b848b77
--- a/dbms/CMakeLists.txt
+++ b/dbms/CMakeLists.txt
@ -2,15 +2,27 @@ if (USE_INCLUDE_WHAT_YOU_USE)
    set (CMAKE_CXX_INCLUDE_WHAT_YOU_USE ${IWYU_PATH})
 endif ()

-include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/find_vectorclass.cmake)
+if(COMPILER_PIPE)
+    set(MAX_COMPILER_MEMORY 2500)
+else()
+    set(MAX_COMPILER_MEMORY 1500)
+endif()
+if(MAKE_STATIC_LIBRARIES)
+    set(MAX_LINKER_MEMORY 3500)
+else()
+    set(MAX_LINKER_MEMORY 2500)
+endif()
+include(../cmake/limit_jobs.cmake)
+
+include(cmake/find_vectorclass.cmake)

 set (CONFIG_VERSION ${CMAKE_CURRENT_BINARY_DIR}/src/Common/config_version.h)
 set (CONFIG_COMMON ${CMAKE_CURRENT_BINARY_DIR}/src/Common/config.h)

 include (cmake/version.cmake)
-message (STATUS "Will build ${VERSION_FULL}")
-configure_file (${CMAKE_CURRENT_SOURCE_DIR}/src/Common/config.h.in ${CONFIG_COMMON})
-configure_file (${CMAKE_CURRENT_SOURCE_DIR}/src/Common/config_version.h.in ${CONFIG_VERSION})
+message (STATUS "Will build ${VERSION_FULL} revision ${VERSION_REVISION}")
+configure_file (src/Common/config.h.in ${CONFIG_COMMON})
+configure_file (src/Common/config_version.h.in ${CONFIG_VERSION})

 if (NOT MSVC)
    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra")
@ -21,13 +33,35 @@ if (NOT NO_WERROR)
    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror")
 endif ()

-# Add some warnings that are not available even with -Wall -Wextra.
+# Add some warnings that are not available even with -Wall -Wextra -Wpedantic.
+
+option (WEVERYTHING "Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang." ON)

 if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra-semi -Wcomma -Winconsistent-missing-destructor-override -Wunused-exception-parameter -Wshadow-uncaptured-local")
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wpedantic -Wno-vla-extension -Wno-zero-length-array -Wno-gnu-anonymous-struct -Wno-nested-anon-types")

-    if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6)
-        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wredundant-parens -Wzero-as-null-pointer-constant")
+    set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wshadow -Wshadow-uncaptured-local -Wextra-semi -Wcomma -Winconsistent-missing-destructor-override -Wunused-exception-parameter -Wcovered-switch-default -Wold-style-cast -Wrange-loop-analysis -Wunused-member-function -Wunreachable-code -Wunreachable-code-return -Wnewline-eof -Wembedded-directive -Wgnu-case-range -Wunused-macros -Wconditional-uninitialized -Wdeprecated -Wundef -Wreserved-id-macro -Wredundant-parens -Wzero-as-null-pointer-constant")
+
+    if (WEVERYTHING)
+        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-missing-noreturn -Wno-padded -Wno-switch-enum -Wno-shadow-field-in-constructor -Wno-deprecated-dynamic-exception-spec -Wno-float-equal -Wno-weak-vtables -Wno-shift-sign-overflow -Wno-sign-conversion -Wno-conversion -Wno-exit-time-destructors -Wno-undefined-func-template -Wno-documentation-unknown-command -Wno-missing-variable-declarations -Wno-unused-template -Wno-global-constructors -Wno-c99-extensions -Wno-missing-prototypes -Wno-weak-template-vtables -Wno-zero-length-array -Wno-gnu-anonymous-struct -Wno-nested-anon-types -Wno-double-promotion -Wno-disabled-macro-expansion -Wno-used-but-marked-unused -Wno-vla-extension -Wno-vla -Wno-packed")
+
+        # TODO Enable conversion, sign-conversion, double-promotion warnings.
+    endif ()
+
+    if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7)
+        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+
+        if (WEVERYTHING)
+            set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-return-std-move-in-c++11")
+        endif ()
+    endif ()
+
+    if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8)
+        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra-semi-stmt -Wshadow-field -Wstring-plus-int")
+
+        if (WEVERYTHING)
+            set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+        endif ()
    endif ()
 endif ()

@ -53,7 +87,7 @@ add_subdirectory (src)
 set(dbms_headers)
 set(dbms_sources)

-include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
+include(../cmake/dbms_glob_sources.cmake)

 add_headers_and_sources(clickhouse_common_io src/Common)
 add_headers_and_sources(clickhouse_common_io src/Common/HashTable)
@ -142,10 +176,6 @@ if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELW
        PROPERTIES COMPILE_FLAGS -g0)
 endif ()

-if (NOT ARCH_ARM AND CPUID_LIBRARY)
-    set (LINK_LIBRARIES_ONLY_ON_X86_64 ${CPUID_LIBRARY})
-endif()
-
 target_link_libraries (clickhouse_common_io
        PUBLIC
    common
@ -153,8 +183,6 @@ target_link_libraries (clickhouse_common_io
    string_utils
    widechar_width
    ${LINK_LIBRARIES_ONLY_ON_X86_64}
-    ${LZ4_LIBRARY}
-    ${ZSTD_LIBRARY}
    ${DOUBLE_CONVERSION_LIBRARIES}
    pocoext
        PUBLIC
@ -172,11 +200,27 @@ target_link_libraries (clickhouse_common_io
    ${Boost_SYSTEM_LIBRARY}
        PRIVATE
    apple_rt
+        PUBLIC
+    Threads::Threads
+        PRIVATE
    ${CMAKE_DL_LIBS}
 )

+target_include_directories(clickhouse_common_io SYSTEM BEFORE PUBLIC ${PDQSORT_INCLUDE_DIR})
+
+target_include_directories(clickhouse_common_io SYSTEM BEFORE PUBLIC ${RE2_INCLUDE_DIR})
+
+if(CPUID_LIBRARY)
+    target_link_libraries(clickhouse_common_io PRIVATE ${CPUID_LIBRARY})
+endif()
+
+if(CPUINFO_LIBRARY)
+    target_link_libraries(clickhouse_common_io PRIVATE ${CPUINFO_LIBRARY})
+endif()
+
 target_link_libraries (dbms
        PRIVATE
+    clickhouse_compression
    clickhouse_parsers
    clickhouse_common_config
        PUBLIC
@ -192,11 +236,9 @@ target_link_libraries (dbms
    ${Boost_PROGRAM_OPTIONS_LIBRARY}
        PUBLIC
    ${Boost_SYSTEM_LIBRARY}
+    Threads::Threads
 )

-if (NOT USE_INTERNAL_RE2_LIBRARY)
-    target_include_directories (dbms SYSTEM BEFORE PRIVATE ${RE2_INCLUDE_DIR})
-endif ()

 if (NOT USE_INTERNAL_BOOST_LIBRARY)
    target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
@ -216,7 +258,6 @@ if (USE_POCO_SQLODBC)
    endif()
 endif()

-#if (Poco_Data_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY)
 if (Poco_Data_FOUND)
    target_include_directories (clickhouse_common_io SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIR})
    target_include_directories (dbms SYSTEM PRIVATE ${Poco_Data_INCLUDE_DIR})
@ -243,6 +284,7 @@ target_link_libraries (dbms PRIVATE ${Poco_Foundation_LIBRARY})

 if (USE_ICU)
    target_link_libraries (dbms PRIVATE ${ICU_LIBRARIES})
+    target_include_directories (dbms SYSTEM PRIVATE ${ICU_INCLUDE_DIRS})
 endif ()

 if (USE_CAPNP)
@ -254,6 +296,7 @@ endif ()

 if (USE_RDKAFKA)
    target_link_libraries (dbms PRIVATE ${RDKAFKA_LIBRARY})
+    target_link_libraries (dbms PRIVATE ${CPPKAFKA_LIBRARY})
    if (NOT USE_INTERNAL_RDKAFKA_LIBRARY)
        target_include_directories (dbms SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR})
    endif ()
@ -264,17 +307,15 @@ target_link_libraries(dbms PRIVATE ${OPENSSL_CRYPTO_LIBRARY} Threads::Threads)
 target_include_directories (dbms SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR})
 target_include_directories (dbms SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})

-if (USE_HDFS)
-    target_link_libraries (dbms PRIVATE ${HDFS3_LIBRARY})
-    target_include_directories (dbms SYSTEM BEFORE PRIVATE ${HDFS3_INCLUDE_DIR})
-endif()
+if (USE_PROTOBUF)
+    target_link_libraries (dbms PRIVATE ${Protobuf_LIBRARY})
+    target_include_directories (dbms SYSTEM BEFORE PRIVATE ${Protobuf_INCLUDE_DIR})
+endif ()

-if (NOT USE_INTERNAL_LZ4_LIBRARY)
-    target_include_directories (dbms SYSTEM BEFORE PRIVATE ${LZ4_INCLUDE_DIR})
-endif ()
-if (NOT USE_INTERNAL_ZSTD_LIBRARY)
-    target_include_directories (dbms SYSTEM BEFORE PRIVATE ${ZSTD_INCLUDE_DIR})
-endif ()
+if (USE_HDFS)
+    target_link_libraries (clickhouse_common_io PRIVATE ${HDFS3_LIBRARY})
+    target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${HDFS3_INCLUDE_DIR})
+endif()

 if (USE_JEMALLOC)
    target_include_directories (dbms SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR}) # used in Interpreters/AsynchronousMetrics.cpp
@ -291,12 +332,7 @@ target_include_directories (clickhouse_common_io BEFORE PRIVATE ${COMMON_INCLUDE
 add_subdirectory (programs)
 add_subdirectory (tests)

-if (GLIBC_COMPATIBILITY AND NOT CLICKHOUSE_SPLIT_BINARY)
-    MESSAGE(STATUS "Some symbols from glibc will be replaced for compatibility")
-    target_link_libraries(dbms PUBLIC glibc-compatibility)
-endif()
-
-if (ENABLE_TESTS)
+if (ENABLE_TESTS AND USE_GTEST)
    macro (grep_gtest_sources BASE_DIR DST_VAR)
        # Cold match files that are not in tests/ directories
        file(GLOB_RECURSE "${DST_VAR}" RELATIVE "${BASE_DIR}" "gtest*.cpp")
--- a/dbms/cmake/version.cmake
+++ b/dbms/cmake/version.cmake
@ -1,11 +1,11 @@
 # This strings autochanged from release_lib.sh:
-set(VERSION_REVISION 54412 CACHE STRING "") # changed manually for tests
-set(VERSION_MAJOR 18 CACHE STRING "")
-set(VERSION_MINOR 16 CACHE STRING "")
-set(VERSION_PATCH 0 CACHE STRING "")
-set(VERSION_GITHASH b9b48c646c253358340bd39fd57754e92f88cd8a CACHE STRING "")
-set(VERSION_DESCRIBE v18.16.0-testing CACHE STRING "")
-set(VERSION_STRING 18.16.0 CACHE STRING "")
+set(VERSION_REVISION 54413)
+set(VERSION_MAJOR 19)
+set(VERSION_MINOR 1)
+set(VERSION_PATCH 6)
+set(VERSION_GITHASH f73b337a93d534671b2187660398b8573fc1d464)
+set(VERSION_DESCRIBE v19.1.6-testing)
+set(VERSION_STRING 19.1.6)
 # end of autochange

 set(VERSION_EXTRA "" CACHE STRING "")
@ -19,8 +19,8 @@ if (VERSION_EXTRA)
    string(CONCAT VERSION_STRING ${VERSION_STRING} "." ${VERSION_EXTRA})
 endif ()

-set (VERSION_NAME "${PROJECT_NAME}" CACHE STRING "")
-set (VERSION_FULL "${VERSION_NAME} ${VERSION_STRING}" CACHE STRING "")
-set (VERSION_SO "${VERSION_STRING}" CACHE STRING "")
+set (VERSION_NAME "${PROJECT_NAME}")
+set (VERSION_FULL "${VERSION_NAME} ${VERSION_STRING}")
+set (VERSION_SO "${VERSION_STRING}")

 math (EXPR VERSION_INTEGER "${VERSION_PATCH} + ${VERSION_MINOR}*1000 + ${VERSION_MAJOR}*1000000")
--- a/dbms/programs/CMakeLists.txt
+++ b/dbms/programs/CMakeLists.txt
@ -28,11 +28,18 @@ add_subdirectory (copier)
 add_subdirectory (format)
 add_subdirectory (clang)
 add_subdirectory (obfuscator)
-add_subdirectory (odbc-bridge)
+
+if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
+    add_subdirectory (odbc-bridge)
+endif ()

 if (CLICKHOUSE_SPLIT_BINARY)
    set (CLICKHOUSE_ALL_TARGETS clickhouse-server clickhouse-client clickhouse-local clickhouse-benchmark clickhouse-performance-test
-            clickhouse-extract-from-config clickhouse-compressor clickhouse-format clickhouse-copier clickhouse-odbc-bridge)
+            clickhouse-extract-from-config clickhouse-compressor clickhouse-format clickhouse-copier)
+
+    if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
+        list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-odbc-bridge)
+    endif ()

    if (USE_EMBEDDED_COMPILER)
        list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-clang clickhouse-lld)
@ -85,9 +92,6 @@ else ()
    if (USE_EMBEDDED_COMPILER)
        target_link_libraries (clickhouse PRIVATE clickhouse-compiler-lib)
    endif ()
-    if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
-        target_link_libraries (clickhouse PRIVATE clickhouse-odbc-bridge-lib)
-    endif()

    set (CLICKHOUSE_BUNDLE)
    if (ENABLE_CLICKHOUSE_SERVER)
@ -135,15 +139,14 @@ else ()
        install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-format DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
        list(APPEND CLICKHOUSE_BUNDLE clickhouse-format)
    endif ()
-    if (ENABLE_CLICKHOUSE_COPIER)
+    if (ENABLE_CLICKHOUSE_OBFUSCATOR)
        add_custom_target (clickhouse-obfuscator ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-obfuscator DEPENDS clickhouse)
        install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-obfuscator DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
        list(APPEND CLICKHOUSE_BUNDLE clickhouse-obfuscator)
    endif ()
    if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
-        add_custom_target (clickhouse-odbc-bridge ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-odbc-bridge DEPENDS clickhouse)
-        install (FILES ${CMAKE_CURRENT_BINARY_DIR}/clickhouse-odbc-bridge DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
-        list(APPEND CLICKHOUSE_BUNDLE clickhouse-odbc-bridge)
+        # just to be able to run integration tests
+        add_custom_target (clickhouse-odbc-bridge-copy ALL COMMAND ${CMAKE_COMMAND} -E create_symlink ${CMAKE_CURRENT_BINARY_DIR}/odbc-bridge/clickhouse-odbc-bridge clickhouse-odbc-bridge DEPENDS clickhouse-odbc-bridge)
    endif ()


--- a/dbms/programs/benchmark/Benchmark.cpp
+++ b/dbms/programs/benchmark/Benchmark.cpp
@ -11,7 +11,7 @@
 #include <Poco/File.h>
 #include <Poco/Util/Application.h>
 #include <Common/Stopwatch.h>
-#include <common/ThreadPool.h>
+#include <Common/ThreadPool.h>
 #include <AggregateFunctions/ReservoirSampler.h>
 #include <AggregateFunctions/registerAggregateFunctions.h>
 #include <boost/program_options.hpp>
@ -229,7 +229,7 @@ private:
                report(info_per_interval);
                delay_watch.restart();
            }
-        };
+        }

        return true;
    }
@ -324,7 +324,7 @@ private:

        double seconds = watch.elapsedSeconds();

-        std::lock_guard<std::mutex> lock(mutex);
+        std::lock_guard lock(mutex);
        info_per_interval.add(seconds, progress.rows, progress.bytes, info.rows, info.bytes);
        info_total.add(seconds, progress.rows, progress.bytes, info.rows, info.bytes);
    }
@ -332,7 +332,7 @@ private:

    void report(Stats & info)
    {
-        std::lock_guard<std::mutex> lock(mutex);
+        std::lock_guard lock(mutex);

        /// Avoid zeros, nans or exceptions
        if (0 == info.queries)
@ -369,7 +369,7 @@ private:
    {
        WriteBufferFromFile json_out(filename);

-        std::lock_guard<std::mutex> lock(mutex);
+        std::lock_guard lock(mutex);

        auto print_key_value = [&](auto key, auto value, bool with_comma = true)
        {
@ -503,6 +503,4 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
        std::cerr << getCurrentExceptionMessage(print_stacktrace, true) << std::endl;
        return getCurrentExceptionCode();
    }
-
-    return 0;
 }
--- a/dbms/programs/benchmark/CMakeLists.txt
+++ b/dbms/programs/benchmark/CMakeLists.txt
@ -5,4 +5,5 @@ target_include_directories (clickhouse-benchmark-lib SYSTEM PRIVATE ${PCG_RANDOM
 if (CLICKHOUSE_SPLIT_BINARY)
    add_executable (clickhouse-benchmark clickhouse-benchmark.cpp)
    target_link_libraries (clickhouse-benchmark PRIVATE clickhouse-benchmark-lib clickhouse_aggregate_functions)
+    install (TARGETS clickhouse-benchmark ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
 endif ()
--- a/dbms/programs/clang/CMakeLists.txt
+++ b/dbms/programs/clang/CMakeLists.txt
@ -27,7 +27,7 @@ elseif (EXISTS ${INTERNAL_COMPILER_BIN_ROOT}${INTERNAL_COMPILER_EXECUTABLE})
 endif ()

 if (COPY_HEADERS_COMPILER AND OS_LINUX)
-    add_custom_target (copy-headers env CLANG=${COPY_HEADERS_COMPILER} BUILD_PATH=${ClickHouse_BINARY_DIR} DESTDIR=${ClickHouse_SOURCE_DIR} ${ClickHouse_SOURCE_DIR}/copy_headers.sh ${ClickHouse_SOURCE_DIR} ${TMP_HEADERS_DIR} DEPENDS ${COPY_HEADERS_DEPENDS} WORKING_DIRECTORY ${ClickHouse_SOURCE_DIR} SOURCES ${ClickHouse_SOURCE_DIR}/copy_headers.sh)
+    add_custom_target (copy-headers [ -f ${TMP_HEADERS_DIR}/dbms/src/Interpreters/SpecializedAggregator.h ] || env CLANG=${COPY_HEADERS_COMPILER} BUILD_PATH=${ClickHouse_BINARY_DIR} DESTDIR=${ClickHouse_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/copy_headers.sh ${ClickHouse_SOURCE_DIR} ${TMP_HEADERS_DIR} DEPENDS ${COPY_HEADERS_DEPENDS} WORKING_DIRECTORY ${ClickHouse_SOURCE_DIR} SOURCES copy_headers.sh)

    if (USE_INTERNAL_LLVM_LIBRARY)
        set (CLANG_HEADERS_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm/clang/lib/Headers")
--- a/dbms/programs/clang/copy_headers.sh
+++ b/dbms/programs/clang/copy_headers.sh
@ -19,7 +19,7 @@ set -e
 #
 # sudo ./copy_headers.sh . /usr/share/clickhouse/headers/

-SOURCE_PATH=${1:-.}
+SOURCE_PATH=${1:-../../..}
 DST=${2:-$SOURCE_PATH/../headers}
 BUILD_PATH=${BUILD_PATH=${3:-$SOURCE_PATH/build}}

--- a/dbms/programs/client/CMakeLists.txt
+++ b/dbms/programs/client/CMakeLists.txt
@ -1,5 +1,5 @@
 add_library (clickhouse-client-lib ${LINK_MODE} Client.cpp)
-target_link_libraries (clickhouse-client-lib PRIVATE clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_link_libraries (clickhouse-client-lib PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
 if (READLINE_INCLUDE_DIR)
    target_include_directories (clickhouse-client-lib SYSTEM PRIVATE ${READLINE_INCLUDE_DIR})
 endif ()
@ -7,6 +7,7 @@ endif ()
 if (CLICKHOUSE_SPLIT_BINARY)
    add_executable (clickhouse-client clickhouse-client.cpp)
    target_link_libraries (clickhouse-client PRIVATE clickhouse-client-lib)
+    install (TARGETS clickhouse-client ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
 endif ()

 install (FILES clickhouse-client.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-client COMPONENT clickhouse-client RENAME config.xml)
--- a/dbms/programs/client/Client.cpp
+++ b/dbms/programs/client/Client.cpp
@ -56,6 +56,7 @@
 #include <Parsers/formatAST.h>
 #include <Parsers/parseQuery.h>
 #include <Interpreters/Context.h>
+#include <Interpreters/InterpreterSetQuery.h>
 #include <Client/Connection.h>
 #include <Common/InterruptListener.h>
 #include <Functions/registerFunctions.h>
@ -74,8 +75,8 @@
 /// http://en.wikipedia.org/wiki/ANSI_escape_code

 /// Similar codes \e[s, \e[u don't work in VT100 and Mosh.
-#define SAVE_CURSOR_POSITION "\e7"
-#define RESTORE_CURSOR_POSITION "\e8"
+#define SAVE_CURSOR_POSITION "\033""7"
+#define RESTORE_CURSOR_POSITION "\033""8"

 #define CLEAR_TO_END_OF_LINE "\033[K"

@ -219,6 +220,9 @@ private:
        APPLY_FOR_SETTINGS(EXTRACT_SETTING)
 #undef EXTRACT_SETTING

+        /// Set path for format schema files
+        if (config().has("format_schema_path"))
+            context.setFormatSchemaPath(Poco::Path(config().getString("format_schema_path")).toString());
    }


@ -554,10 +558,10 @@ private:

    void loop()
    {
-        String query;
-        String prev_query;
+        String input;
+        String prev_input;

-        while (char * line_ = readline(query.empty() ? prompt().c_str() : ":-] "))
+        while (char * line_ = readline(input.empty() ? prompt().c_str() : ":-] "))
        {
            String line = line_;
            free(line_);
@ -577,17 +581,17 @@ private:
            if (ends_with_backslash)
                line = line.substr(0, ws - 1);

-            query += line;
+            input += line;

            if (!ends_with_backslash && (ends_with_semicolon || has_vertical_output_suffix || (!config().has("multiline") && !hasDataInSTDIN())))
            {
-                if (query != prev_query)
+                if (input != prev_input)
                {
                    /// Replace line breaks with spaces to prevent the following problem.
                    /// Every line of multi-line query is saved to history file as a separate line.
                    /// If the user restarts the client then after pressing the "up" button
                    /// every line of the query will be displayed separately.
-                    std::string logged_query = query;
+                    std::string logged_query = input;
                    std::replace(logged_query.begin(), logged_query.end(), '\n', ' ');
                    add_history(logged_query.c_str());

@ -596,18 +600,18 @@ private:
                        throwFromErrno("Cannot append history to file " + history_file, ErrorCodes::CANNOT_APPEND_HISTORY);
 #endif

-                    prev_query = query;
+                    prev_input = input;
                }

                if (has_vertical_output_suffix)
-                    query = query.substr(0, query.length() - 2);
+                    input = input.substr(0, input.length() - 2);

                try
                {
                    /// Determine the terminal size.
                    ioctl(0, TIOCGWINSZ, &terminal_size);

-                    if (!process(query))
+                    if (!process(input))
                        break;
                }
                catch (const Exception & e)
@ -633,11 +637,11 @@ private:
                    connect();
                }

-                query = "";
+                input = "";
            }
            else
            {
-                query += '\n';
+                input += '\n';
            }
        }
    }
@ -666,11 +670,15 @@ private:
        const bool test_mode = config().has("testmode");
        if (config().has("multiquery"))
        {
+            {   /// disable logs if expects errors
+                TestHint test_hint(test_mode, text);
+                if (test_hint.clientError() || test_hint.serverError())
+                    process("SET send_logs_level = 'none'");
+            }
+
            /// Several queries separated by ';'.
            /// INSERT data is ended by the end of line, not ';'.

-            String query;
-
            const char * begin = text.data();
            const char * end = begin + text.size();

@ -702,19 +710,19 @@ private:
                    insert->end = pos;
                }

-                query = text.substr(begin - text.data(), pos - begin);
+                String str = text.substr(begin - text.data(), pos - begin);

                begin = pos;
                while (isWhitespaceASCII(*begin) || *begin == ';')
                    ++begin;

-                TestHint test_hint(test_mode, query);
+                TestHint test_hint(test_mode, str);
                expected_client_error = test_hint.clientError();
                expected_server_error = test_hint.serverError();

                try
                {
-                    if (!processSingleQuery(query, ast) && !ignore_error)
+                    if (!processSingleQuery(str, ast) && !ignore_error)
                        return false;
                }
                catch (...)
@ -722,7 +730,7 @@ private:
                    last_exception = std::make_unique<Exception>(getCurrentExceptionMessage(true), getCurrentExceptionCode());
                    actual_client_error = last_exception->code();
                    if (!ignore_error && (!actual_client_error || actual_client_error != expected_client_error))
-                        std::cerr << "Error on processing query: " << query << std::endl << last_exception->message();
+                        std::cerr << "Error on processing query: " << str << std::endl << last_exception->message();
                    got_exception = true;
                }

@ -856,7 +864,7 @@ private:
    }


-    /// Process the query that doesn't require transfering data blocks to the server.
+    /// Process the query that doesn't require transferring data blocks to the server.
    void processOrdinaryQuery()
    {
        connection->sendQuery(query, query_id, QueryProcessingStage::Complete, &context.getSettingsRef(), nullptr, true);
@ -865,7 +873,7 @@ private:
    }


-    /// Process the query that requires transfering data blocks to the server.
+    /// Process the query that requires transferring data blocks to the server.
    void processInsertQuery()
    {
        /// Send part of query without data, because data will be sent separately.
@ -898,8 +906,6 @@ private:
        ParserQuery parser(end, true);
        ASTPtr res;

-        const auto ignore_error = config().getBool("ignore-error", false);
-
        if (is_interactive || ignore_error)
        {
            String message;
@ -1134,7 +1140,7 @@ private:
    }


-    /// Process Log packets, exit when recieve Exception or EndOfStream
+    /// Process Log packets, exit when receive Exception or EndOfStream
    bool receiveEndOfQuery()
    {
        while (true)
@ -1204,6 +1210,10 @@ private:
                    const auto & id = typeid_cast<const ASTIdentifier &>(*query_with_output->format);
                    current_format = id.name;
                }
+                if (query_with_output->settings_ast)
+                {
+                    InterpreterSetQuery(query_with_output->settings_ast, context).executeForCurrentContext();
+                }
            }

            if (has_vertical_output_suffix)
@ -1532,12 +1542,19 @@ public:
        po::options_description main_description("Main options", line_length, min_description_length);
        main_description.add_options()
            ("help", "produce help message")
-            ("config-file,c", po::value<std::string>(), "config-file path")
+            ("config-file,C", po::value<std::string>(), "config-file path")
+            ("config,c", po::value<std::string>(), "config-file path (another shorthand)")
            ("host,h", po::value<std::string>()->default_value("localhost"), "server host")
            ("port", po::value<int>()->default_value(9000), "server port")
            ("secure,s", "Use TLS connection")
            ("user,u", po::value<std::string>()->default_value("default"), "user")
-            ("password", po::value<std::string>(), "password")
+            /** If "--password [value]" is used but the value is omitted, the bad argument exception will be thrown.
+              * implicit_value is used to avoid this exception (to allow user to type just "--password")
+              * Since currently boost provides no way to check if a value has been set implicitly for an option,
+              * the "\n" is used to distinguish this case because there is hardly a chance an user would use "\n"
+              * as the password.
+              */
+            ("password", po::value<std::string>()->implicit_value("\n"), "password")
            ("ask-password", "ask-password")
            ("query_id", po::value<std::string>(), "query_id")
            ("query,q", po::value<std::string>(), "query")
@ -1575,13 +1592,11 @@ public:
            ("structure", po::value<std::string>(), "structure")
            ("types", po::value<std::string>(), "types")
        ;
-
        /// Parse main commandline options.
        po::parsed_options parsed = po::command_line_parser(
            common_arguments.size(), common_arguments.data()).options(main_description).run();
        po::variables_map options;
        po::store(parsed, options);
-
        if (options.count("version") || options.count("V"))
        {
            showClientVersion();
@ -1610,10 +1625,10 @@ public:
        for (size_t i = 0; i < external_tables_arguments.size(); ++i)
        {
            /// Parse commandline options related to external tables.
-            po::parsed_options parsed = po::command_line_parser(
+            po::parsed_options parsed_tables = po::command_line_parser(
                external_tables_arguments[i].size(), external_tables_arguments[i].data()).options(external_description).run();
            po::variables_map external_options;
-            po::store(parsed, external_options);
+            po::store(parsed_tables, external_options);

            try
            {
@ -1639,9 +1654,14 @@ public:
        APPLY_FOR_SETTINGS(EXTRACT_SETTING)
 #undef EXTRACT_SETTING

+        if (options.count("config-file") && options.count("config"))
+            throw Exception("Two or more configuration files referenced in arguments", ErrorCodes::BAD_ARGUMENTS);
+
        /// Save received data into the internal config.
        if (options.count("config-file"))
            config().setString("config-file", options["config-file"].as<std::string>());
+        if (options.count("config"))
+            config().setString("config-file", options["config"].as<std::string>());
        if (options.count("host") && !options["host"].defaulted())
            config().setString("host", options["host"].as<std::string>());
        if (options.count("query_id"))
@ -1700,11 +1720,11 @@ public:

 int mainEntryClickHouseClient(int argc, char ** argv)
 {
-    DB::Client client;
-
    try
    {
+        DB::Client client;
        client.init(argc, argv);
+        return client.run();
    }
    catch (const boost::program_options::error & e)
    {
@ -1716,6 +1736,4 @@ int mainEntryClickHouseClient(int argc, char ** argv)
        std::cerr << DB::getCurrentExceptionMessage(true) << std::endl;
        return 1;
    }
-
-    return client.run();
 }
--- a/dbms/programs/client/ConnectionParameters.h
+++ b/dbms/programs/client/ConnectionParameters.h
@ -8,7 +8,7 @@
 #include <Common/Exception.h>
 #include <IO/ConnectionTimeouts.h>

-#include <common/SetTerminalEcho.h>
+#include <common/setTerminalEcho.h>
 #include <ext/scope_guard.h>

 #include <Poco/Util/AbstractConfiguration.h>
@ -25,12 +25,12 @@ namespace ErrorCodes
 struct ConnectionParameters
 {
    String host;
-    UInt16 port;
+    UInt16 port{};
    String default_database;
    String user;
    String password;
-    Protocol::Secure security;
-    Protocol::Compression compression;
+    Protocol::Secure security = Protocol::Secure::Disable;
+    Protocol::Compression compression = Protocol::Compression::Enable;
    ConnectionTimeouts timeouts;

    ConnectionParameters() {}
@ -48,27 +48,33 @@ struct ConnectionParameters
                is_secure ? DBMS_DEFAULT_SECURE_PORT : DBMS_DEFAULT_PORT));

        default_database = config.getString("database", "");
-        user = config.getString("user", "");
-
+        /// changed the default value to "default" to fix the issue when the user in the prompt is blank
+        user = config.getString("user", "default");
+        bool password_prompt = false;
        if (config.getBool("ask-password", false))
        {
            if (config.has("password"))
                throw Exception("Specified both --password and --ask-password. Remove one of them", ErrorCodes::BAD_ARGUMENTS);
-
-            std::cout << "Password for user " << user << ": ";
-            SetTerminalEcho(false);
-
-            SCOPE_EXIT({
-                SetTerminalEcho(true);
-            });
-            std::getline(std::cin, password);
-            std::cout << std::endl;
+            password_prompt = true;
        }
        else
        {
            password = config.getString("password", "");
+            /// if the value of --password is omitted, the password will be set implicitly to "\n"
+            if (password == "\n")
+                password_prompt = true;
        }
+        if (password_prompt)
+        {
+            std::cout << "Password for user (" << user << "): ";
+            setTerminalEcho(false);

+            SCOPE_EXIT({
+                setTerminalEcho(true);
+            });
+            std::getline(std::cin, password);
+            std::cout << std::endl;
+        }
        compression = config.getBool("compression", true)
            ? Protocol::Compression::Enable
            : Protocol::Compression::Disable;
--- a/dbms/programs/client/Suggest.h
+++ b/dbms/programs/client/Suggest.h
@ -56,7 +56,7 @@ private:
    {
        std::string prefix_str(prefix);
        std::tie(pos, end) = std::equal_range(words.begin(), words.end(), prefix_str,
-            [prefix_length](const std::string & s, const std::string & prefix) { return strncmp(s.c_str(), prefix.c_str(), prefix_length) < 0; });
+            [prefix_length](const std::string & s, const std::string & prefix_searched) { return strncmp(s.c_str(), prefix_searched.c_str(), prefix_length) < 0; });
    }

    /// Iterates through matched range.
--- a/dbms/programs/client/TestHint.h
+++ b/dbms/programs/client/TestHint.h
@ -5,6 +5,7 @@
 #include <iostream>
 #include <Core/Types.h>
 #include <Common/Exception.h>
+#include <Parsers/Lexer.h>


 namespace DB
@ -27,25 +28,27 @@ public:
        if (!enabled_)
            return;

-        /// TODO: This is absolutely wrong. Fragment may be contained inside string literal.
-        size_t pos = query.find("--");
+        Lexer lexer(query.data(), query.data() + query.size());

-        if (pos != String::npos && query.find("--", pos + 2) != String::npos)
-            return; /// It's not last comment. Hint belongs to commented query. /// TODO Absolutely wrong: there maybe the following comment for the next query.
-
-        if (pos != String::npos)
+        for (Token token = lexer.nextToken(); !token.isEnd(); token = lexer.nextToken())
        {
-            /// TODO: This is also wrong. Comment may already have ended by line break.
-            pos = query.find('{', pos + 2);
-
-            if (pos != String::npos)
+            if (token.type == TokenType::Comment)
            {
-                String hint = query.substr(pos + 1);
+                String comment(token.begin, token.begin + token.size());

-                /// TODO: And this is wrong for the same reason.
-                pos = hint.find('}');
-                hint.resize(pos);
-                parse(hint);
+                if (!comment.empty())
+                {
+                    size_t pos_start = comment.find('{', 0);
+                    if (pos_start != String::npos)
+                    {
+                        size_t pos_end = comment.find('}', pos_start);
+                        if (pos_end != String::npos)
+                        {
+                            String hint(comment.begin() + pos_start + 1, comment.begin() + pos_end);
+                            parse(hint);
+                        }
+                    }
+                }
            }
        }
    }
@ -90,11 +93,12 @@ private:
    {
        std::stringstream ss;
        ss << hint;
+        String item;
+
        while (!ss.eof())
        {
-            String item;
            ss >> item;
-            if (item.empty())
+            if (ss.eof())
                break;

            if (item == "serverError")
--- a/dbms/programs/compressor/CMakeLists.txt
+++ b/dbms/programs/compressor/CMakeLists.txt
@ -1,8 +1,9 @@
 add_library (clickhouse-compressor-lib ${LINK_MODE} Compressor.cpp)
-target_link_libraries (clickhouse-compressor-lib PRIVATE clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_link_libraries (clickhouse-compressor-lib PRIVATE clickhouse_compression clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})

 if (CLICKHOUSE_SPLIT_BINARY)
    # Also in utils
    add_executable (clickhouse-compressor clickhouse-compressor.cpp)
    target_link_libraries (clickhouse-compressor PRIVATE clickhouse-compressor-lib)
+    install (TARGETS clickhouse-compressor ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
 endif ()
--- a/dbms/programs/compressor/Compressor.cpp
+++ b/dbms/programs/compressor/Compressor.cpp
@ -1,21 +1,23 @@
 #include <iostream>
-
+#include <optional>
 #include <boost/program_options.hpp>

 #include <Common/Exception.h>
 #include <IO/WriteBufferFromFileDescriptor.h>
 #include <IO/ReadBufferFromFileDescriptor.h>
-#include <IO/CompressedWriteBuffer.h>
-#include <IO/CompressedReadBuffer.h>
+#include <Compression/CompressedWriteBuffer.h>
+#include <Compression/CompressedReadBuffer.h>
 #include <IO/WriteHelpers.h>
 #include <IO/copyData.h>

+#include <Compression/CompressionFactory.h>

 namespace DB
 {
    namespace ErrorCodes
    {
        extern const int TOO_LARGE_SIZE_COMPRESSED;
+        extern const int BAD_ARGUMENTS;
    }
 }

@ -61,7 +63,8 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
        ("block-size,b", boost::program_options::value<unsigned>()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size")
        ("hc", "use LZ4HC instead of LZ4")
        ("zstd", "use ZSTD instead of LZ4")
-        ("level", boost::program_options::value<int>(), "compression level")
+        ("codec", boost::program_options::value<std::vector<std::string>>()->multitoken(), "use codecs combination instead of LZ4")
+        ("level", boost::program_options::value<std::vector<int>>()->multitoken(), "compression levels for codecs specified via --codec")
        ("none", "use no compression instead of LZ4")
        ("stat", "print block statistics of compressed data")
    ;
@ -84,19 +87,45 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
        bool stat_mode = options.count("stat");
        bool use_none = options.count("none");
        unsigned block_size = options["block-size"].as<unsigned>();
+        std::vector<std::string> codecs;
+        if (options.count("codec"))
+            codecs = options["codec"].as<std::vector<std::string>>();

-        DB::CompressionMethod method = DB::CompressionMethod::LZ4;
+        if ((use_lz4hc || use_zstd || use_none) && !codecs.empty())
+            throw DB::Exception("Wrong options, codec flags like --zstd and --codec options are mutually exclusive", DB::ErrorCodes::BAD_ARGUMENTS);
+
+        std::string method_family = "LZ4";

        if (use_lz4hc)
-            method = DB::CompressionMethod::LZ4HC;
+            method_family = "LZ4HC";
        else if (use_zstd)
-            method = DB::CompressionMethod::ZSTD;
+            method_family = "ZSTD";
        else if (use_none)
-            method = DB::CompressionMethod::NONE;
+            method_family = "NONE";
+
+        std::vector<int> levels;
+        if (options.count("level"))
+            levels = options["level"].as<std::vector<int>>();
+
+        DB::CompressionCodecPtr codec;
+        if (!codecs.empty())
+        {
+            if (levels.size() > codecs.size())
+                throw DB::Exception("Specified more levels than codecs", DB::ErrorCodes::BAD_ARGUMENTS);
+
+            std::vector<DB::CodecNameWithLevel> codec_names;
+            for (size_t i = 0; i < codecs.size(); ++i)
+            {
+                if (i < levels.size())
+                    codec_names.emplace_back(codecs[i], levels[i]);
+                else
+                    codec_names.emplace_back(codecs[i], std::nullopt);
+            }
+            codec = DB::CompressionCodecFactory::instance().get(codec_names);
+        }
+        else
+            codec = DB::CompressionCodecFactory::instance().get(method_family, levels.empty() ? std::nullopt : std::optional<int>(levels.back()));

-        DB::CompressionSettings settings(method, options.count("level")
-            ? options["level"].as<int>()
-            : DB::CompressionSettings::getDefaultLevel(method));

        DB::ReadBufferFromFileDescriptor rb(STDIN_FILENO);
        DB::WriteBufferFromFileDescriptor wb(STDOUT_FILENO);
@ -115,7 +144,7 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
        else
        {
            /// Compression
-            DB::CompressedWriteBuffer to(wb, settings, block_size);
+            DB::CompressedWriteBuffer to(wb, codec, block_size);
            DB::copyData(rb, to);
        }
    }
--- a/dbms/programs/compressor/README.md
+++ b/dbms/programs/compressor/README.md
@ -0,0 +1,27 @@
+## ClickHouse compressor
+
+Simple program for data compression and decompression.
+
+### Examples
+
+Compress data with LZ4:
+```
+$ ./clickhouse-compressor < input_file > output_file
+```
+
+Decompress data from LZ4 format:
+```
+$ ./clickhouse-compressor --decompress < input_file > output_file
+```
+
+Compress data with ZSTD at level 5:
+
+```
+$ ./clickhouse-compressor --codec ZSTD --level 5 < input_file > output_file
+```
+
+Compress data with ZSTD level 10, LZ4HC level 7 and LZ4.
+
+```
+$ ./clickhouse-compressor --codec ZSTD --level 5 --codec LZ4HC --level 7 --codec LZ4 < input_file > output_file
+```
--- a/dbms/programs/config_tools.h.in
+++ b/dbms/programs/config_tools.h.in
@ -12,3 +12,4 @@
 #cmakedefine01 ENABLE_CLICKHOUSE_COMPRESSOR
 #cmakedefine01 ENABLE_CLICKHOUSE_FORMAT
 #cmakedefine01 ENABLE_CLICKHOUSE_OBFUSCATOR
+#cmakedefine01 ENABLE_CLICKHOUSE_ODBC_BRIDGE
--- a/dbms/programs/copier/CMakeLists.txt
+++ b/dbms/programs/copier/CMakeLists.txt
@ -4,4 +4,5 @@ target_link_libraries (clickhouse-copier-lib PRIVATE clickhouse-server-lib click
 if (CLICKHOUSE_SPLIT_BINARY)
    add_executable (clickhouse-copier clickhouse-copier.cpp)
    target_link_libraries (clickhouse-copier clickhouse-copier-lib)
+    install (TARGETS clickhouse-copier ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
 endif ()
--- a/dbms/programs/copier/ClusterCopier.cpp
+++ b/dbms/programs/copier/ClusterCopier.cpp
@ -18,7 +18,7 @@
 #include <pcg_random.hpp>

 #include <common/logger_useful.h>
-#include <common/ThreadPool.h>
+#include <Common/ThreadPool.h>
 #include <daemon/OwnPatternFormatter.h>

 #include <Common/Exception.h>
@ -243,7 +243,7 @@ struct ClusterPartition
    UInt64 rows_copied = 0;
    UInt64 blocks_copied = 0;

-    size_t total_tries = 0;
+    UInt64 total_tries = 0;
 };


@ -340,7 +340,7 @@ struct TaskCluster
    String default_local_database;

    /// Limits number of simultaneous workers
-    size_t max_workers = 0;
+    UInt64 max_workers = 0;

    /// Base settings for pull and push
    Settings settings_common;
@ -773,11 +773,11 @@ public:
    }

    template <typename T>
-    decltype(auto) retry(T && func, size_t max_tries = 100)
+    decltype(auto) retry(T && func, UInt64 max_tries = 100)
    {
        std::exception_ptr exception;

-        for (size_t try_number = 1; try_number <= max_tries; ++try_number)
+        for (UInt64 try_number = 1; try_number <= max_tries; ++try_number)
        {
            try
            {
@ -880,7 +880,7 @@ public:
    }

    /// Compute set of partitions, assume set of partitions aren't changed during the processing
-    void discoverTablePartitions(TaskTable & task_table, size_t num_threads = 0)
+    void discoverTablePartitions(TaskTable & task_table, UInt64 num_threads = 0)
    {
        /// Fetch partitions list from a shard
        {
@ -985,7 +985,7 @@ public:

            /// Retry table processing
            bool table_is_done = false;
-            for (size_t num_table_tries = 0; num_table_tries < max_table_tries; ++num_table_tries)
+            for (UInt64 num_table_tries = 0; num_table_tries < max_table_tries; ++num_table_tries)
            {
                if (tryProcessTable(task_table))
                {
@ -1044,7 +1044,7 @@ protected:
        String workers_path = getWorkersPath();
        String current_worker_path = getCurrentWorkerNodePath();

-        size_t num_bad_version_errors = 0;
+        UInt64 num_bad_version_errors = 0;

        while (true)
        {
@ -1055,7 +1055,7 @@ protected:
            auto version = stat.version;
            zookeeper->get(workers_path, &stat);

-            if (static_cast<size_t>(stat.numChildren) >= task_cluster->max_workers)
+            if (static_cast<UInt64>(stat.numChildren) >= task_cluster->max_workers)
            {
                LOG_DEBUG(log, "Too many workers (" << stat.numChildren << ", maximum " << task_cluster->max_workers << ")"
                    << ". Postpone processing " << description);
@ -1163,7 +1163,7 @@ protected:
        }

        // If all task is finished and zxid is not changed then partition could not become dirty again
-        for (size_t shard_num = 0; shard_num < status_paths.size(); ++shard_num)
+        for (UInt64 shard_num = 0; shard_num < status_paths.size(); ++shard_num)
        {
            if (zxid1[shard_num] != zxid2[shard_num])
            {
@ -1280,7 +1280,7 @@ protected:

        LOG_DEBUG(log, "Execute distributed DROP PARTITION: " << query);
        /// Limit number of max executing replicas to 1
-        size_t num_shards = executeQueryOnCluster(cluster_push, query, nullptr, &settings_push, PoolMode::GET_ONE, 1);
+        UInt64 num_shards = executeQueryOnCluster(cluster_push, query, nullptr, &settings_push, PoolMode::GET_ONE, 1);

        if (num_shards < cluster_push->getShardCount())
        {
@ -1299,8 +1299,8 @@ protected:
    }


-    static constexpr size_t max_table_tries = 1000;
-    static constexpr size_t max_shard_partition_tries = 600;
+    static constexpr UInt64 max_table_tries = 1000;
+    static constexpr UInt64 max_shard_partition_tries = 600;

    bool tryProcessTable(TaskTable & task_table)
    {
@ -1317,7 +1317,7 @@ protected:

            Stopwatch watch;
            TasksShard expected_shards;
-            size_t num_failed_shards = 0;
+            UInt64 num_failed_shards = 0;

            ++cluster_partition.total_tries;

@ -1368,7 +1368,7 @@ protected:
                bool is_unprioritized_task = !previous_shard_is_instantly_finished && shard->priority.is_remote;
                PartitionTaskStatus task_status = PartitionTaskStatus::Error;
                bool was_error = false;
-                for (size_t try_num = 0; try_num < max_shard_partition_tries; ++try_num)
+                for (UInt64 try_num = 0; try_num < max_shard_partition_tries; ++try_num)
                {
                    task_status = tryProcessPartitionTask(partition, is_unprioritized_task);

@ -1434,8 +1434,8 @@ protected:
            }
        }

-        size_t required_partitions = task_table.cluster_partitions.size();
-        size_t finished_partitions = task_table.finished_cluster_partitions.size();
+        UInt64 required_partitions = task_table.cluster_partitions.size();
+        UInt64 finished_partitions = task_table.finished_cluster_partitions.size();
        bool table_is_done = finished_partitions >= required_partitions;

        if (!table_is_done)
@ -1645,7 +1645,7 @@ protected:
            String query = queryToString(create_query_push_ast);

            LOG_DEBUG(log, "Create destination tables. Query: " << query);
-            size_t shards = executeQueryOnCluster(task_table.cluster_push, query, create_query_push_ast, &task_cluster->settings_push,
+            UInt64 shards = executeQueryOnCluster(task_table.cluster_push, query, create_query_push_ast, &task_cluster->settings_push,
                                    PoolMode::GET_MANY);
            LOG_DEBUG(log, "Destination tables " << getDatabaseDotTable(task_table.table_push) << " have been created on " << shards
                                                 << " shards of " << task_table.cluster_push->getShardCount());
@ -1699,7 +1699,7 @@ protected:
                std::future<Coordination::ExistsResponse> future_is_dirty_checker;

                Stopwatch watch(CLOCK_MONOTONIC_COARSE);
-                constexpr size_t check_period_milliseconds = 500;
+                constexpr UInt64 check_period_milliseconds = 500;

                /// Will asynchronously check that ZooKeeper connection and is_dirty flag appearing while copy data
                auto cancel_check = [&] ()
@ -1917,16 +1917,16 @@ protected:
    /** Executes simple query (without output streams, for example DDL queries) on each shard of the cluster
      * Returns number of shards for which at least one replica executed query successfully
      */
-    size_t executeQueryOnCluster(
+    UInt64 executeQueryOnCluster(
        const ClusterPtr & cluster,
        const String & query,
        const ASTPtr & query_ast_ = nullptr,
        const Settings * settings = nullptr,
        PoolMode pool_mode = PoolMode::GET_ALL,
-        size_t max_successful_executions_per_shard = 0) const
+        UInt64 max_successful_executions_per_shard = 0) const
    {
        auto num_shards = cluster->getShardsInfo().size();
-        std::vector<size_t> per_shard_num_successful_replicas(num_shards, 0);
+        std::vector<UInt64> per_shard_num_successful_replicas(num_shards, 0);

        ASTPtr query_ast;
        if (query_ast_ == nullptr)
@ -1939,10 +1939,10 @@ protected:


        /// We need to execute query on one replica at least
-        auto do_for_shard = [&] (size_t shard_index)
+        auto do_for_shard = [&] (UInt64 shard_index)
        {
            const Cluster::ShardInfo & shard = cluster->getShardsInfo().at(shard_index);
-            size_t & num_successful_executions = per_shard_num_successful_replicas.at(shard_index);
+            UInt64 & num_successful_executions = per_shard_num_successful_replicas.at(shard_index);
            num_successful_executions = 0;

            auto increment_and_check_exit = [&] ()
@ -1951,12 +1951,12 @@ protected:
                return max_successful_executions_per_shard && num_successful_executions >= max_successful_executions_per_shard;
            };

-            size_t num_replicas = cluster->getShardsAddresses().at(shard_index).size();
-            size_t num_local_replicas = shard.getLocalNodeCount();
-            size_t num_remote_replicas = num_replicas - num_local_replicas;
+            UInt64 num_replicas = cluster->getShardsAddresses().at(shard_index).size();
+            UInt64 num_local_replicas = shard.getLocalNodeCount();
+            UInt64 num_remote_replicas = num_replicas - num_local_replicas;

            /// In that case we don't have local replicas, but do it just in case
-            for (size_t i = 0; i < num_local_replicas; ++i)
+            for (UInt64 i = 0; i < num_local_replicas; ++i)
            {
                auto interpreter = InterpreterFactory::get(query_ast, context);
                interpreter->execute();
@ -1997,16 +1997,16 @@ protected:
        };

        {
-            ThreadPool thread_pool(std::min(num_shards, getNumberOfPhysicalCPUCores()));
+            ThreadPool thread_pool(std::min<UInt64>(num_shards, getNumberOfPhysicalCPUCores()));

-            for (size_t shard_index = 0; shard_index < num_shards; ++shard_index)
+            for (UInt64 shard_index = 0; shard_index < num_shards; ++shard_index)
                thread_pool.schedule([=] { do_for_shard(shard_index); });

            thread_pool.wait();
        }

-        size_t successful_shards = 0;
-        for (size_t num_replicas : per_shard_num_successful_replicas)
+        UInt64 successful_shards = 0;
+        for (UInt64 num_replicas : per_shard_num_successful_replicas)
            successful_shards += (num_replicas > 0);

        return successful_shards;
--- a/dbms/programs/extract-from-config/CMakeLists.txt
+++ b/dbms/programs/extract-from-config/CMakeLists.txt
@ -4,4 +4,5 @@ target_link_libraries (clickhouse-extract-from-config-lib PRIVATE clickhouse_com
 if (CLICKHOUSE_SPLIT_BINARY)
    add_executable (clickhouse-extract-from-config clickhouse-extract-from-config.cpp)
    target_link_libraries (clickhouse-extract-from-config PRIVATE clickhouse-extract-from-config-lib)
+    install (TARGETS clickhouse-extract-from-config ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
 endif ()
--- a/dbms/programs/format/CMakeLists.txt
+++ b/dbms/programs/format/CMakeLists.txt
@ -3,4 +3,5 @@ target_link_libraries (clickhouse-format-lib PRIVATE dbms clickhouse_common_io c
 if (CLICKHOUSE_SPLIT_BINARY)
    add_executable (clickhouse-format clickhouse-format.cpp)
    target_link_libraries (clickhouse-format PRIVATE clickhouse-format-lib)
+    install (TARGETS clickhouse-format ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
 endif ()
--- a/dbms/programs/local/CMakeLists.txt
+++ b/dbms/programs/local/CMakeLists.txt
@ -4,4 +4,5 @@ target_link_libraries (clickhouse-local-lib PRIVATE clickhouse_common_io clickho
 if (CLICKHOUSE_SPLIT_BINARY)
    add_executable (clickhouse-local clickhouse-local.cpp)
    target_link_libraries (clickhouse-local PRIVATE clickhouse-local-lib)
+    install (TARGETS clickhouse-local ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
 endif ()
--- a/dbms/programs/local/LocalServer.cpp
+++ b/dbms/programs/local/LocalServer.cpp
@ -17,6 +17,7 @@
 #include <Common/Config/ConfigProcessor.h>
 #include <Common/escapeForFileName.h>
 #include <Common/ClickHouseRevision.h>
+#include <Common/ThreadStatus.h>
 #include <Common/config_version.h>
 #include <IO/ReadBufferFromString.h>
 #include <IO/WriteBufferFromString.h>
@ -66,11 +67,11 @@ void LocalServer::initialize(Poco::Util::Application & self)
    }
 }

-void LocalServer::applyCmdSettings(Context & context)
+void LocalServer::applyCmdSettings()
 {
 #define EXTRACT_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) \
        if (cmd_settings.NAME.changed) \
-            context.getSettingsRef().NAME = cmd_settings.NAME;
+            context->getSettingsRef().NAME = cmd_settings.NAME;
        APPLY_FOR_SETTINGS(EXTRACT_SETTING)
 #undef EXTRACT_SETTING
 }
@ -102,7 +103,7 @@ int LocalServer::main(const std::vector<std::string> & /*args*/)
 try
 {
    Logger * log = &logger();
-
+    ThreadStatus thread_status;
    UseSSL use_ssl;

    if (!config().has("query") && !config().has("table-structure")) /// Nothing to process
@ -137,7 +138,7 @@ try
    static KillingErrorHandler error_handler;
    Poco::ErrorHandler::set(&error_handler);

-    /// Don't initilaize DateLUT
+    /// Don't initialize DateLUT

    registerFunctions();
    registerAggregateFunctions();
@ -179,7 +180,7 @@ try
    std::string default_database = config().getString("default_database", "_local");
    context->addDatabase(default_database, std::make_shared<DatabaseMemory>(default_database));
    context->setCurrentDatabase(default_database);
-    applyCmdOptions(*context);
+    applyCmdOptions();

    if (!context->getPath().empty())
    {
@ -274,7 +275,7 @@ void LocalServer::processQueries()

    context->setUser("default", "", Poco::Net::SocketAddress{}, "");
    context->setCurrentQueryId("");
-    applyCmdSettings(*context);
+    applyCmdSettings();

    /// Use the same query_id (and thread group) for all queries
    CurrentThread::QueryScope query_scope_holder(*context);
@ -296,7 +297,7 @@ void LocalServer::processQueries()

        try
        {
-            executeQuery(read_buf, write_buf, /* allow_into_outfile = */ true, *context, {});
+            executeQuery(read_buf, write_buf, /* allow_into_outfile = */ true, *context, {}, {});
        }
        catch (...)
        {
@ -494,10 +495,10 @@ void LocalServer::init(int argc, char ** argv)
        config().setBool("ignore-error", true);
 }

-void LocalServer::applyCmdOptions(Context & context)
+void LocalServer::applyCmdOptions()
 {
-    context.setDefaultFormat(config().getString("output-format", config().getString("format", "TSV")));
-    applyCmdSettings(context);
+    context->setDefaultFormat(config().getString("output-format", config().getString("format", "TSV")));
+    applyCmdSettings();
 }

 }
--- a/dbms/programs/local/LocalServer.h
+++ b/dbms/programs/local/LocalServer.h
@ -34,8 +34,8 @@ private:
    std::string getInitialCreateTableQuery();

    void tryInitPath();
-    void applyCmdOptions(Context & context);
-    void applyCmdSettings(Context & context);
+    void applyCmdOptions();
+    void applyCmdSettings();
    void attachSystemTables();
    void processQueries();
    void setupUsers();
--- a/dbms/programs/main.cpp
+++ b/dbms/programs/main.cpp
@ -53,12 +53,9 @@ int mainEntryClickHouseFormat(int argc, char ** argv);
 #if ENABLE_CLICKHOUSE_COPIER || !defined(ENABLE_CLICKHOUSE_COPIER)
 int mainEntryClickHouseClusterCopier(int argc, char ** argv);
 #endif
-#if ENABLE_CLICKHOUSE_OBFUSCATOR
+#if ENABLE_CLICKHOUSE_OBFUSCATOR || !defined(ENABLE_CLICKHOUSE_OBFUSCATOR)
 int mainEntryClickHouseObfuscator(int argc, char ** argv);
 #endif
-#if ENABLE_CLICKHOUSE_ODBC_BRIDGE || !defined(ENABLE_CLICKHOUSE_ODBC_BRIDGE)
-int mainEntryClickHouseODBCBridge(int argc, char ** argv);
-#endif


 #if USE_EMBEDDED_COMPILER
@ -102,12 +99,9 @@ std::pair<const char *, MainFunc> clickhouse_applications[] =
 #if ENABLE_CLICKHOUSE_COPIER || !defined(ENABLE_CLICKHOUSE_COPIER)
    {"copier", mainEntryClickHouseClusterCopier},
 #endif
-#if ENABLE_CLICKHOUSE_OBFUSCATOR
+#if ENABLE_CLICKHOUSE_OBFUSCATOR || !defined(ENABLE_CLICKHOUSE_OBFUSCATOR)
    {"obfuscator", mainEntryClickHouseObfuscator},
 #endif
-#if ENABLE_CLICKHOUSE_ODBC_BRIDGE || !defined(ENABLE_CLICKHOUSE_ODBC_BRIDGE)
-    {"odbc-bridge", mainEntryClickHouseODBCBridge},
-#endif

 #if USE_EMBEDDED_COMPILER
    {"clang", mainEntryClickHouseClang},
--- a/dbms/programs/obfuscator/CMakeLists.txt
+++ b/dbms/programs/obfuscator/CMakeLists.txt
@ -5,4 +5,5 @@ if (CLICKHOUSE_SPLIT_BINARY)
    add_executable (clickhouse-obfuscator clickhouse-obfuscator.cpp)
    set_target_properties(clickhouse-obfuscator PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
    target_link_libraries (clickhouse-obfuscator PRIVATE clickhouse-obfuscator-lib)
+    install (TARGETS clickhouse-obfuscator ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
 endif ()
--- a/dbms/programs/obfuscator/Obfuscator.cpp
+++ b/dbms/programs/obfuscator/Obfuscator.cpp
@ -51,7 +51,7 @@ It is designed to retain the following properties of data:
 - probability distributions of length of strings;
 - probability of zero values of numbers; empty strings and arrays, NULLs;
 - data compression ratio when compressed with LZ77 and entropy family of codecs;
- continuouty (magnitude of difference) of time values across table; continuouty of floating point values.
+- continuity (magnitude of difference) of time values across table; continuity of floating point values.
 - date component of DateTime values;
 - UTF-8 validity of string values;
 - string values continue to look somewhat natural.
@ -123,7 +123,7 @@ UInt64 hash(Ts... xs)

 UInt64 maskBits(UInt64 x, size_t num_bits)
 {
-    return x & ((1 << num_bits) - 1);
+    return x & ((1ULL << num_bits) - 1);
 }


@ -149,7 +149,7 @@ UInt64 feistelNetwork(UInt64 x, size_t num_bits, UInt64 seed, size_t num_rounds
    UInt64 bits = maskBits(x, num_bits);
    for (size_t i = 0; i < num_rounds; ++i)
        bits = feistelRound(bits, num_bits, seed, i);
-    return (x & ~((1 << num_bits) - 1)) ^ bits;
+    return (x & ~((1ULL << num_bits) - 1)) ^ bits;
 }


@ -246,7 +246,7 @@ Float transformFloatMantissa(Float x, UInt64 seed)


 /// Transform difference from previous number by applying pseudorandom permutation to mantissa part of it.
-/// It allows to retain some continuouty property of source data.
+/// It allows to retain some continuity property of source data.
 template <typename Float>
 class FloatModel : public IModel
 {
@ -317,8 +317,8 @@ void transformFixedString(const UInt8 * src, UInt8 * dst, size_t size, UInt64 se

        if (size >= 16)
        {
-            char * dst = reinterpret_cast<char *>(std::min(pos, end - 16));
-            hash.get128(dst);
+            char * hash_dst = reinterpret_cast<char *>(std::min(pos, end - 16));
+            hash.get128(hash_dst);
        }
        else
        {
--- a/dbms/programs/odbc-bridge/CMakeLists.txt
+++ b/dbms/programs/odbc-bridge/CMakeLists.txt
@ -9,7 +9,7 @@ add_library (clickhouse-odbc-bridge-lib ${LINK_MODE}
    validateODBCConnectionString.cpp
 )

-target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE clickhouse_common_io daemon dbms)
+target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE daemon dbms clickhouse_common_io)
 target_include_directories (clickhouse-odbc-bridge-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)

 if (USE_POCO_SQLODBC)
@ -33,7 +33,11 @@ if (ENABLE_TESTS)
    add_subdirectory (tests)
 endif ()

-if (CLICKHOUSE_SPLIT_BINARY)
-    add_executable (clickhouse-odbc-bridge odbc-bridge.cpp)
-    target_link_libraries (clickhouse-odbc-bridge PRIVATE clickhouse-odbc-bridge-lib)
-endif ()
+# clickhouse-odbc-bridge is always a separate binary.
+# Reason: it must not export symbols from SSL, mariadb-client, etc. to not break ABI compatibility with ODBC drivers.
+# For this reason, we disabling -rdynamic linker flag. But we do it in strange way:
+SET(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "")
+
+add_executable (clickhouse-odbc-bridge odbc-bridge.cpp)
+target_link_libraries (clickhouse-odbc-bridge PRIVATE clickhouse-odbc-bridge-lib)
+install (TARGETS clickhouse-odbc-bridge RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
--- a/dbms/programs/odbc-bridge/getIdentifierQuote.h
+++ b/dbms/programs/odbc-bridge/getIdentifierQuote.h
@ -19,4 +19,4 @@ namespace DB

 std::string getIdentifierQuote(SQLHDBC hdbc);
 }
-#endif
+#endif
--- a/dbms/programs/odbc-bridge/validateODBCConnectionString.cpp
+++ b/dbms/programs/odbc-bridge/validateODBCConnectionString.cpp
@ -22,7 +22,7 @@ std::string validateODBCConnectionString(const std::string & connection_string)
    /// Connection string is a list of name, value pairs.
    /// name and value are separated by '='.
    /// names are case insensitive.
-    /// name=value pairs are sepated by ';'.
+    /// name=value pairs are separated by ';'.
    /// ASCII whitespace characters are skipped before and after delimiters.
    /// value may be optionally enclosed by {}
    /// in enclosed value, } is escaped as }}.
--- a/dbms/programs/performance-test/CMakeLists.txt
+++ b/dbms/programs/performance-test/CMakeLists.txt
@ -1,8 +1,21 @@
-add_library (clickhouse-performance-test-lib ${LINK_MODE} PerformanceTest.cpp)
+add_library (clickhouse-performance-test-lib ${LINK_MODE}
+  JSONString.cpp
+  StopConditionsSet.cpp
+  TestStopConditions.cpp
+  TestStats.cpp
+  ConfigPreprocessor.cpp
+  PerformanceTest.cpp
+  PerformanceTestInfo.cpp
+  executeQuery.cpp
+  applySubstitutions.cpp
+  ReportBuilder.cpp
+  PerformanceTestSuite.cpp
+)
 target_link_libraries (clickhouse-performance-test-lib PRIVATE dbms clickhouse_common_io clickhouse_common_config ${Boost_PROGRAM_OPTIONS_LIBRARY})
 target_include_directories (clickhouse-performance-test-lib SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR})

 if (CLICKHOUSE_SPLIT_BINARY)
    add_executable (clickhouse-performance-test clickhouse-performance-test.cpp)
    target_link_libraries (clickhouse-performance-test PRIVATE clickhouse-performance-test-lib)
+    install (TARGETS clickhouse-performance-test ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
 endif ()
--- a/dbms/programs/performance-test/ConfigPreprocessor.cpp
+++ b/dbms/programs/performance-test/ConfigPreprocessor.cpp
@ -0,0 +1,90 @@
+#include "ConfigPreprocessor.h"
+#include <Core/Types.h>
+#include <Poco/Path.h>
+#include <regex>
+namespace DB
+{
+std::vector<XMLConfigurationPtr> ConfigPreprocessor::processConfig(
+    const Strings & tests_tags,
+    const Strings & tests_names,
+    const Strings & tests_names_regexp,
+    const Strings & skip_tags,
+    const Strings & skip_names,
+    const Strings & skip_names_regexp) const
+{
+
+    std::vector<XMLConfigurationPtr> result;
+    for (const auto & path : paths)
+    {
+        result.emplace_back(new XMLConfiguration(path));
+        result.back()->setString("path", Poco::Path(path).absolute().toString());
+    }
+
+    /// Leave tests:
+    removeConfigurationsIf(result, FilterType::Tag, tests_tags, true);
+    removeConfigurationsIf(result, FilterType::Name, tests_names, true);
+    removeConfigurationsIf(result, FilterType::Name_regexp, tests_names_regexp, true);
+
+    /// Skip tests
+    removeConfigurationsIf(result, FilterType::Tag, skip_tags, false);
+    removeConfigurationsIf(result, FilterType::Name, skip_names, false);
+    removeConfigurationsIf(result, FilterType::Name_regexp, skip_names_regexp, false);
+    return result;
+}
+
+void ConfigPreprocessor::removeConfigurationsIf(
+    std::vector<XMLConfigurationPtr> & configs,
+    ConfigPreprocessor::FilterType filter_type,
+    const Strings & values,
+    bool leave) const
+{
+    auto checker = [&filter_type, &values, &leave] (XMLConfigurationPtr & config)
+    {
+        if (values.size() == 0)
+            return false;
+
+        bool remove_or_not = false;
+
+        if (filter_type == FilterType::Tag)
+        {
+            Strings tags_keys;
+            config->keys("tags", tags_keys);
+
+            Strings tags(tags_keys.size());
+            for (size_t i = 0; i != tags_keys.size(); ++i)
+                tags[i] = config->getString("tags.tag[" + std::to_string(i) + "]");
+
+            for (const std::string & config_tag : tags)
+            {
+                if (std::find(values.begin(), values.end(), config_tag) != values.end())
+                    remove_or_not = true;
+            }
+        }
+
+        if (filter_type == FilterType::Name)
+        {
+            remove_or_not = (std::find(values.begin(), values.end(), config->getString("name", "")) != values.end());
+        }
+
+        if (filter_type == FilterType::Name_regexp)
+        {
+            std::string config_name = config->getString("name", "");
+            auto regex_checker = [&config_name](const std::string & name_regexp)
+            {
+                std::regex pattern(name_regexp);
+                return std::regex_search(config_name, pattern);
+            };
+
+            remove_or_not = config->has("name") ? (std::find_if(values.begin(), values.end(), regex_checker) != values.end()) : false;
+        }
+
+        if (leave)
+            remove_or_not = !remove_or_not;
+        return remove_or_not;
+    };
+
+    auto new_end = std::remove_if(configs.begin(), configs.end(), checker);
+    configs.erase(new_end, configs.end());
+}
+
+}
--- a/dbms/programs/performance-test/ConfigPreprocessor.h
+++ b/dbms/programs/performance-test/ConfigPreprocessor.h
@ -0,0 +1,50 @@
+#pragma once
+
+#include <Poco/DOM/Document.h>
+#include <Poco/Util/XMLConfiguration.h>
+#include <Core/Types.h>
+#include <vector>
+#include <string>
+
+namespace DB
+{
+
+using XMLConfiguration = Poco::Util::XMLConfiguration;
+using XMLConfigurationPtr = Poco::AutoPtr<XMLConfiguration>;
+using XMLDocumentPtr = Poco::AutoPtr<Poco::XML::Document>;
+
+class ConfigPreprocessor
+{
+public:
+    ConfigPreprocessor(const Strings & paths_)
+        : paths(paths_)
+    {}
+
+    std::vector<XMLConfigurationPtr> processConfig(
+        const Strings & tests_tags,
+        const Strings & tests_names,
+        const Strings & tests_names_regexp,
+        const Strings & skip_tags,
+        const Strings & skip_names,
+        const Strings & skip_names_regexp) const;
+
+private:
+
+    enum class FilterType
+    {
+        Tag,
+        Name,
+        Name_regexp
+    };
+
+    /// Removes configurations that has a given value.
+    /// If leave is true, the logic is reversed.
+    void removeConfigurationsIf(
+        std::vector<XMLConfigurationPtr> & configs,
+        FilterType filter_type,
+        const Strings & values,
+        bool leave = false) const;
+
+    const Strings paths;
+};
+}
--- a/dbms/programs/performance-test/JSONString.cpp
+++ b/dbms/programs/performance-test/JSONString.cpp
@ -0,0 +1,66 @@
+#include "JSONString.h"
+
+#include <regex>
+#include <sstream>
+namespace DB
+{
+
+namespace
+{
+std::string pad(size_t padding)
+{
+    return std::string(padding * 4, ' ');
+}
+
+const std::regex NEW_LINE{"\n"};
+}
+
+void JSONString::set(const std::string & key, std::string value, bool wrap)
+{
+    if (value.empty())
+        value = "null";
+
+    bool reserved = (value[0] == '[' || value[0] == '{' || value == "null");
+    if (!reserved && wrap)
+        value = '"' + std::regex_replace(value, NEW_LINE, "\\n") + '"';
+
+    content[key] = value;
+}
+
+void JSONString::set(const std::string & key, const std::vector<JSONString> & run_infos)
+{
+    std::ostringstream value;
+    value << "[\n";
+
+    for (size_t i = 0; i < run_infos.size(); ++i)
+    {
+        value << pad(padding + 1) + run_infos[i].asString(padding + 2);
+        if (i != run_infos.size() - 1)
+            value << ',';
+
+        value << "\n";
+    }
+
+    value << pad(padding) << ']';
+    content[key] = value.str();
+}
+
+std::string JSONString::asString(size_t cur_padding) const
+{
+    std::ostringstream repr;
+    repr << "{";
+
+    for (auto it = content.begin(); it != content.end(); ++it)
+    {
+        if (it != content.begin())
+            repr << ',';
+        /// construct "key": "value" string with padding
+        repr << "\n" << pad(cur_padding) << '"' << it->first << '"' << ": " << it->second;
+    }
+
+    repr << "\n" << pad(cur_padding - 1) << '}';
+    return repr.str();
+}
+
+
+}
--- a/dbms/programs/performance-test/JSONString.h
+++ b/dbms/programs/performance-test/JSONString.h
@ -0,0 +1,40 @@
+#pragma once
+#include <Core/Types.h>
+
+#include <sys/stat.h>
+#include <type_traits>
+#include <vector>
+#include <map>
+
+namespace DB
+{
+
+/// NOTE The code is totally wrong.
+class JSONString
+{
+private:
+    std::map<std::string, std::string> content;
+    size_t padding;
+
+public:
+    explicit JSONString(size_t padding_ = 1) : padding(padding_) {}
+
+    void set(const std::string & key, std::string value, bool wrap = true);
+
+    template <typename T>
+    std::enable_if_t<std::is_arithmetic_v<T>> set(const std::string key, T value)
+    {
+        set(key, std::to_string(value), /*wrap= */ false);
+    }
+
+    void set(const std::string & key, const std::vector<JSONString> & run_infos);
+
+    std::string asString() const
+    {
+        return asString(padding);
+    }
+
+    std::string asString(size_t cur_padding) const;
+};
+
+}
--- a/dbms/programs/performance-test/PerformanceTest.cpp
+++ b/dbms/programs/performance-test/PerformanceTest.cpp
--- a/dbms/programs/performance-test/PerformanceTest.h
+++ b/dbms/programs/performance-test/PerformanceTest.h
@ -0,0 +1,62 @@
+#pragma once
+
+#include <Client/Connection.h>
+#include <Common/InterruptListener.h>
+#include <common/logger_useful.h>
+#include <Poco/Util/XMLConfiguration.h>
+
+#include "PerformanceTestInfo.h"
+
+namespace DB
+{
+
+using XMLConfiguration = Poco::Util::XMLConfiguration;
+using XMLConfigurationPtr = Poco::AutoPtr<XMLConfiguration>;
+using QueriesWithIndexes = std::vector<std::pair<std::string, size_t>>;
+
+class PerformanceTest
+{
+public:
+    PerformanceTest(
+        const XMLConfigurationPtr & config_,
+        Connection & connection_,
+        InterruptListener & interrupt_listener_,
+        const PerformanceTestInfo & test_info_,
+        Context & context_);
+
+    bool checkPreconditions() const;
+    void prepare() const;
+    std::vector<TestStats> execute();
+    void finish() const;
+
+    const PerformanceTestInfo & getTestInfo() const
+    {
+        return test_info;
+    }
+
+    bool checkSIGINT() const
+    {
+        return got_SIGINT;
+    }
+
+private:
+    void runQueries(
+        const QueriesWithIndexes & queries_with_indexes,
+        std::vector<TestStats> & statistics_by_run);
+
+    UInt64 calculateMaxExecTime() const;
+
+private:
+    XMLConfigurationPtr config;
+    Connection & connection;
+    InterruptListener & interrupt_listener;
+
+    PerformanceTestInfo test_info;
+    Context & context;
+
+    Poco::Logger * log;
+
+    bool got_SIGINT = false;
+};
+
+}
--- a/dbms/programs/performance-test/PerformanceTestInfo.cpp
+++ b/dbms/programs/performance-test/PerformanceTestInfo.cpp
@ -0,0 +1,285 @@
+#include "PerformanceTestInfo.h"
+#include <Common/getMultipleKeysFromConfig.h>
+#include <IO/ReadBufferFromFile.h>
+#include <IO/ReadHelpers.h>
+#include <IO/WriteBufferFromFile.h>
+#include <boost/filesystem.hpp>
+#include "applySubstitutions.h"
+#include <iostream>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+extern const int BAD_ARGUMENTS;
+}
+
+namespace
+{
+
+void extractSettings(
+    const XMLConfigurationPtr & config,
+    const std::string & key,
+    const Strings & settings_list,
+    std::map<std::string, std::string> & settings_to_apply)
+{
+    for (const std::string & setup : settings_list)
+    {
+        if (setup == "profile")
+            continue;
+
+        std::string value = config->getString(key + "." + setup);
+        if (value.empty())
+            value = "true";
+
+        settings_to_apply[setup] = value;
+    }
+}
+
+void checkMetricsInput(const Strings & metrics, ExecutionType exec_type)
+{
+    Strings loop_metrics = {
+        "min_time", "quantiles", "total_time",
+        "queries_per_second", "rows_per_second",
+        "bytes_per_second"};
+
+    Strings non_loop_metrics = {
+        "max_rows_per_second", "max_bytes_per_second",
+        "avg_rows_per_second", "avg_bytes_per_second"};
+
+    if (exec_type == ExecutionType::Loop)
+    {
+        for (const std::string & metric : metrics)
+        {
+            auto non_loop_pos =
+                std::find(non_loop_metrics.begin(), non_loop_metrics.end(), metric);
+
+            if (non_loop_pos != non_loop_metrics.end())
+               throw Exception("Wrong type of metric for loop execution type (" + metric + ")",
+                   ErrorCodes::BAD_ARGUMENTS);
+        }
+    }
+    else
+    {
+        for (const std::string & metric : metrics)
+        {
+            auto loop_pos = std::find(loop_metrics.begin(), loop_metrics.end(), metric);
+            if (loop_pos != loop_metrics.end())
+                throw Exception(
+                    "Wrong type of metric for non-loop execution type (" + metric + ")",
+                    ErrorCodes::BAD_ARGUMENTS);
+        }
+    }
+}
+
+}
+
+
+namespace fs = boost::filesystem;
+
+PerformanceTestInfo::PerformanceTestInfo(
+    XMLConfigurationPtr config,
+    const std::string & profiles_file_)
+    : profiles_file(profiles_file_)
+{
+    test_name = config->getString("name");
+    path = config->getString("path");
+    applySettings(config);
+    extractQueries(config);
+    processSubstitutions(config);
+    getExecutionType(config);
+    getStopConditions(config);
+    getMetrics(config);
+    extractAuxiliaryQueries(config);
+}
+
+void PerformanceTestInfo::applySettings(XMLConfigurationPtr config)
+{
+    if (config->has("settings"))
+    {
+        std::map<std::string, std::string> settings_to_apply;
+        Strings config_settings;
+        config->keys("settings", config_settings);
+
+        auto settings_contain = [&config_settings] (const std::string & setting)
+        {
+            auto position = std::find(config_settings.begin(), config_settings.end(), setting);
+            return position != config_settings.end();
+
+        };
+        /// Preprocess configuration file
+        if (settings_contain("profile"))
+        {
+            if (!profiles_file.empty())
+            {
+                std::string profile_name = config->getString("settings.profile");
+                XMLConfigurationPtr profiles_config(new XMLConfiguration(profiles_file));
+
+                Strings profile_settings;
+                profiles_config->keys("profiles." + profile_name, profile_settings);
+
+                extractSettings(profiles_config, "profiles." + profile_name, profile_settings, settings_to_apply);
+            }
+        }
+
+        extractSettings(config, "settings", config_settings, settings_to_apply);
+
+        /// This macro goes through all settings in the Settings.h
+        /// and, if found any settings in test's xml configuration
+        /// with the same name, sets its value to settings
+        std::map<std::string, std::string>::iterator it;
+#define EXTRACT_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION)   \
+        it = settings_to_apply.find(#NAME);                 \
+        if (it != settings_to_apply.end())                  \
+            settings.set(#NAME, settings_to_apply[#NAME]);
+
+        APPLY_FOR_SETTINGS(EXTRACT_SETTING)
+
+#undef EXTRACT_SETTING
+
+        if (settings_contain("average_rows_speed_precision"))
+            TestStats::avg_rows_speed_precision =
+                config->getDouble("settings.average_rows_speed_precision");
+
+        if (settings_contain("average_bytes_speed_precision"))
+            TestStats::avg_bytes_speed_precision =
+                config->getDouble("settings.average_bytes_speed_precision");
+    }
+}
+
+void PerformanceTestInfo::extractQueries(XMLConfigurationPtr config)
+{
+    if (config->has("query"))
+        queries = getMultipleValuesFromConfig(*config, "", "query");
+
+    if (config->has("query_file"))
+    {
+        const std::string filename = config->getString("query_file");
+        if (filename.empty())
+            throw Exception("Empty file name", ErrorCodes::BAD_ARGUMENTS);
+
+        bool tsv = fs::path(filename).extension().string() == ".tsv";
+
+        ReadBufferFromFile query_file(filename);
+        std::string query;
+
+        if (tsv)
+        {
+            while (!query_file.eof())
+            {
+                readEscapedString(query, query_file);
+                assertChar('\n', query_file);
+                queries.push_back(query);
+            }
+        }
+        else
+        {
+            readStringUntilEOF(query, query_file);
+            queries.push_back(query);
+        }
+    }
+
+    if (queries.empty())
+        throw Exception("Did not find any query to execute: " + test_name,
+            ErrorCodes::BAD_ARGUMENTS);
+}
+
+void PerformanceTestInfo::processSubstitutions(XMLConfigurationPtr config)
+{
+    if (config->has("substitutions"))
+    {
+        /// Make "subconfig" of inner xml block
+        ConfigurationPtr substitutions_view(config->createView("substitutions"));
+        constructSubstitutions(substitutions_view, substitutions);
+
+        auto queries_pre_format = queries;
+        queries.clear();
+        for (const auto & query : queries_pre_format)
+        {
+            auto formatted = formatQueries(query, substitutions);
+            queries.insert(queries.end(), formatted.begin(), formatted.end());
+        }
+    }
+}
+
+void PerformanceTestInfo::getExecutionType(XMLConfigurationPtr config)
+{
+    if (!config->has("type"))
+        throw Exception("Missing type property in config: " + test_name,
+            ErrorCodes::BAD_ARGUMENTS);
+
+    std::string config_exec_type = config->getString("type");
+    if (config_exec_type == "loop")
+        exec_type = ExecutionType::Loop;
+    else if (config_exec_type == "once")
+        exec_type = ExecutionType::Once;
+    else
+        throw Exception("Unknown type " + config_exec_type + " in :" + test_name,
+            ErrorCodes::BAD_ARGUMENTS);
+}
+
+
+void PerformanceTestInfo::getStopConditions(XMLConfigurationPtr config)
+{
+    TestStopConditions stop_conditions_template;
+    if (config->has("stop_conditions"))
+    {
+        ConfigurationPtr stop_conditions_config(config->createView("stop_conditions"));
+        stop_conditions_template.loadFromConfig(stop_conditions_config);
+    }
+
+    if (stop_conditions_template.empty())
+        throw Exception("No termination conditions were found in config",
+            ErrorCodes::BAD_ARGUMENTS);
+
+    times_to_run = config->getUInt("times_to_run", 1);
+
+    for (size_t i = 0; i < times_to_run * queries.size(); ++i)
+        stop_conditions_by_run.push_back(stop_conditions_template);
+
+}
+
+
+void PerformanceTestInfo::getMetrics(XMLConfigurationPtr config)
+{
+    ConfigurationPtr metrics_view(config->createView("metrics"));
+    metrics_view->keys(metrics);
+
+    if (config->has("main_metric"))
+    {
+        Strings main_metrics;
+        config->keys("main_metric", main_metrics);
+        if (main_metrics.size())
+            main_metric = main_metrics[0];
+    }
+
+    if (!main_metric.empty())
+    {
+        if (std::find(metrics.begin(), metrics.end(), main_metric) == metrics.end())
+            metrics.push_back(main_metric);
+    }
+    else
+    {
+        if (metrics.empty())
+            throw Exception("You shoud specify at least one metric",
+                ErrorCodes::BAD_ARGUMENTS);
+        main_metric = metrics[0];
+    }
+
+    if (metrics.size() > 0)
+        checkMetricsInput(metrics, exec_type);
+}
+
+void PerformanceTestInfo::extractAuxiliaryQueries(XMLConfigurationPtr config)
+{
+    if (config->has("create_query"))
+        create_queries = getMultipleValuesFromConfig(*config, "", "create_query");
+
+    if (config->has("fill_query"))
+        fill_queries = getMultipleValuesFromConfig(*config, "", "fill_query");
+
+    if (config->has("drop_query"))
+        drop_queries = getMultipleValuesFromConfig(*config, "", "drop_query");
+}
+
+}
--- a/dbms/programs/performance-test/PerformanceTestInfo.h
+++ b/dbms/programs/performance-test/PerformanceTestInfo.h
@ -0,0 +1,60 @@
+#pragma once
+#include <string>
+#include <vector>
+#include <map>
+#include <Interpreters/Settings.h>
+#include <Poco/Util/XMLConfiguration.h>
+#include <Poco/AutoPtr.h>
+
+#include "StopConditionsSet.h"
+#include "TestStopConditions.h"
+#include "TestStats.h"
+
+namespace DB
+{
+enum class ExecutionType
+{
+    Loop,
+    Once
+};
+
+using XMLConfiguration = Poco::Util::XMLConfiguration;
+using XMLConfigurationPtr = Poco::AutoPtr<XMLConfiguration>;
+using StringToVector = std::map<std::string, Strings>;
+
+/// Class containing all info to run performance test
+class PerformanceTestInfo
+{
+public:
+    PerformanceTestInfo(XMLConfigurationPtr config, const std::string & profiles_file_);
+
+    std::string test_name;
+    std::string path;
+    std::string main_metric;
+
+    Strings queries;
+    Strings metrics;
+
+    Settings settings;
+    ExecutionType exec_type;
+    StringToVector substitutions;
+    size_t times_to_run;
+
+    std::string profiles_file;
+    std::vector<TestStopConditions> stop_conditions_by_run;
+
+    Strings create_queries;
+    Strings fill_queries;
+    Strings drop_queries;
+
+private:
+    void applySettings(XMLConfigurationPtr config);
+    void extractQueries(XMLConfigurationPtr config);
+    void processSubstitutions(XMLConfigurationPtr config);
+    void getExecutionType(XMLConfigurationPtr config);
+    void getStopConditions(XMLConfigurationPtr config);
+    void getMetrics(XMLConfigurationPtr config);
+    void extractAuxiliaryQueries(XMLConfigurationPtr config);
+};
+
+}
--- a/dbms/programs/performance-test/PerformanceTestSuite.cpp
+++ b/dbms/programs/performance-test/PerformanceTestSuite.cpp
@ -0,0 +1,389 @@
+#include <algorithm>
+#include <iostream>
+#include <limits>
+#include <regex>
+#include <thread>
+#include <memory>
+
+#include <port/unistd.h>
+#include <sys/stat.h>
+
+#include <boost/filesystem.hpp>
+#include <boost/program_options.hpp>
+
+#include <Poco/Util/XMLConfiguration.h>
+#include <Poco/Logger.h>
+#include <Poco/ConsoleChannel.h>
+#include <Poco/FormattingChannel.h>
+#include <Poco/PatternFormatter.h>
+
+
+#include <common/logger_useful.h>
+#include <Client/Connection.h>
+#include <Core/Types.h>
+#include <Interpreters/Context.h>
+#include <IO/ConnectionTimeouts.h>
+#include <IO/UseSSL.h>
+#include <Interpreters/Settings.h>
+#include <Poco/AutoPtr.h>
+#include <Common/Exception.h>
+#include <Common/InterruptListener.h>
+
+#include "TestStopConditions.h"
+#include "TestStats.h"
+#include "ConfigPreprocessor.h"
+#include "PerformanceTest.h"
+#include "ReportBuilder.h"
+
+
+namespace fs = boost::filesystem;
+namespace po = boost::program_options;
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int BAD_ARGUMENTS;
+    extern const int FILE_DOESNT_EXIST;
+}
+
+/** Tests launcher for ClickHouse.
+ * The tool walks through given or default folder in order to find files with
+ * tests' descriptions and launches it.
+ */
+class PerformanceTestSuite
+{
+public:
+
+    PerformanceTestSuite(const std::string & host_,
+        const UInt16 port_,
+        const bool secure_,
+        const std::string & default_database_,
+        const std::string & user_,
+        const std::string & password_,
+        const bool lite_output_,
+        const std::string & profiles_file_,
+        Strings && input_files_,
+        Strings && tests_tags_,
+        Strings && skip_tags_,
+        Strings && tests_names_,
+        Strings && skip_names_,
+        Strings && tests_names_regexp_,
+        Strings && skip_names_regexp_,
+        const ConnectionTimeouts & timeouts)
+        : connection(host_, port_, default_database_, user_,
+            password_, timeouts, "performance-test", Protocol::Compression::Enable,
+            secure_ ? Protocol::Secure::Enable : Protocol::Secure::Disable)
+        , tests_tags(std::move(tests_tags_))
+        , tests_names(std::move(tests_names_))
+        , tests_names_regexp(std::move(tests_names_regexp_))
+        , skip_tags(std::move(skip_tags_))
+        , skip_names(std::move(skip_names_))
+        , skip_names_regexp(std::move(skip_names_regexp_))
+        , lite_output(lite_output_)
+        , profiles_file(profiles_file_)
+        , input_files(input_files_)
+        , log(&Poco::Logger::get("PerformanceTestSuite"))
+    {
+        if (input_files.size() < 1)
+            throw Exception("No tests were specified", ErrorCodes::BAD_ARGUMENTS);
+    }
+
+    /// This functionality seems strange.
+    //void initialize(Poco::Util::Application & self [[maybe_unused]])
+    //{
+    //    std::string home_path;
+    //    const char * home_path_cstr = getenv("HOME");
+    //    if (home_path_cstr)
+    //        home_path = home_path_cstr;
+    //    configReadClient(Poco::Util::Application::instance().config(), home_path);
+    //}
+
+    int run()
+    {
+        std::string name;
+        UInt64 version_major;
+        UInt64 version_minor;
+        UInt64 version_patch;
+        UInt64 version_revision;
+        connection.getServerVersion(name, version_major, version_minor, version_patch, version_revision);
+
+        std::stringstream ss;
+        ss << version_major << "." << version_minor << "." << version_patch;
+        server_version = ss.str();
+
+        report_builder = std::make_shared<ReportBuilder>(server_version);
+
+        processTestsConfigurations(input_files);
+
+        return 0;
+    }
+
+private:
+    Connection connection;
+
+    const Strings & tests_tags;
+    const Strings & tests_names;
+    const Strings & tests_names_regexp;
+    const Strings & skip_tags;
+    const Strings & skip_names;
+    const Strings & skip_names_regexp;
+
+    Context global_context = Context::createGlobal();
+    std::shared_ptr<ReportBuilder> report_builder;
+
+    std::string server_version;
+
+    InterruptListener interrupt_listener;
+
+    using XMLConfiguration = Poco::Util::XMLConfiguration;
+    using XMLConfigurationPtr = Poco::AutoPtr<XMLConfiguration>;
+
+    bool lite_output;
+    std::string profiles_file;
+
+    Strings input_files;
+    std::vector<XMLConfigurationPtr> tests_configurations;
+    Poco::Logger * log;
+
+    void processTestsConfigurations(const Strings & paths)
+    {
+        LOG_INFO(log, "Preparing test configurations");
+        ConfigPreprocessor config_prep(paths);
+        tests_configurations = config_prep.processConfig(
+            tests_tags,
+            tests_names,
+            tests_names_regexp,
+            skip_tags,
+            skip_names,
+            skip_names_regexp);
+
+        LOG_INFO(log, "Test configurations prepared");
+
+        if (tests_configurations.size())
+        {
+            Strings outputs;
+
+            for (auto & test_config : tests_configurations)
+            {
+                auto [output, signal] = runTest(test_config);
+                if (lite_output)
+                    std::cout << output;
+                else
+                    outputs.push_back(output);
+
+                if (signal)
+                    break;
+            }
+
+            if (!lite_output && outputs.size())
+            {
+                std::cout << "[" << std::endl;
+
+                for (size_t i = 0; i != outputs.size(); ++i)
+                {
+                    std::cout << outputs[i];
+                    if (i != outputs.size() - 1)
+                        std::cout << ",";
+
+                    std::cout << std::endl;
+                }
+
+                std::cout << "]" << std::endl;
+            }
+        }
+    }
+
+    std::pair<std::string, bool> runTest(XMLConfigurationPtr & test_config)
+    {
+        PerformanceTestInfo info(test_config, profiles_file);
+        LOG_INFO(log, "Config for test '" << info.test_name << "' parsed");
+        PerformanceTest current(test_config, connection, interrupt_listener, info, global_context);
+
+        current.checkPreconditions();
+        LOG_INFO(log, "Preconditions for test '" << info.test_name << "' are fullfilled");
+        LOG_INFO(log, "Preparing for run, have " << info.create_queries.size()
+            << " create queries and " << info.fill_queries.size() << " fill queries");
+        current.prepare();
+        LOG_INFO(log, "Prepared");
+        LOG_INFO(log, "Running test '" << info.test_name << "'");
+        auto result = current.execute();
+        LOG_INFO(log, "Test '" << info.test_name << "' finished");
+
+        LOG_INFO(log, "Running post run queries");
+        current.finish();
+        LOG_INFO(log, "Postqueries finished");
+
+        if (lite_output)
+            return {report_builder->buildCompactReport(info, result), current.checkSIGINT()};
+        else
+            return {report_builder->buildFullReport(info, result), current.checkSIGINT()};
+    }
+
+};
+
+}
+
+static void getFilesFromDir(const fs::path & dir, std::vector<std::string> & input_files, const bool recursive = false)
+{
+    Poco::Logger * log = &Poco::Logger::get("PerformanceTestSuite");
+    if (dir.extension().string() == ".xml")
+        LOG_WARNING(log, dir.string() + "' is a directory, but has .xml extension");
+
+    fs::directory_iterator end;
+    for (fs::directory_iterator it(dir); it != end; ++it)
+    {
+        const fs::path file = (*it);
+        if (recursive && fs::is_directory(file))
+            getFilesFromDir(file, input_files, recursive);
+        else if (!fs::is_directory(file) && file.extension().string() == ".xml")
+            input_files.push_back(file.string());
+    }
+}
+
+static std::vector<std::string> getInputFiles(const po::variables_map & options, Poco::Logger * log)
+{
+    std::vector<std::string> input_files;
+    bool recursive = options.count("recursive");
+
+    if (!options.count("input-files"))
+    {
+        LOG_INFO(log, "Trying to find test scenario files in the current folder...");
+        fs::path curr_dir(".");
+
+        getFilesFromDir(curr_dir, input_files, recursive);
+
+        if (input_files.empty())
+            throw DB::Exception("Did not find any xml files", DB::ErrorCodes::BAD_ARGUMENTS);
+        else
+            LOG_INFO(log, "Found " << input_files.size() << " files");
+    }
+    else
+    {
+        input_files = options["input-files"].as<std::vector<std::string>>();
+        LOG_INFO(log, "Found " + std::to_string(input_files.size()) + " input files");
+        std::vector<std::string> collected_files;
+
+        for (const std::string & filename : input_files)
+        {
+            fs::path file(filename);
+
+            if (!fs::exists(file))
+                throw DB::Exception("File '" + filename + "' does not exist", DB::ErrorCodes::FILE_DOESNT_EXIST);
+
+            if (fs::is_directory(file))
+            {
+                getFilesFromDir(file, collected_files, recursive);
+            }
+            else
+            {
+                if (file.extension().string() != ".xml")
+                    throw DB::Exception("File '" + filename + "' does not have .xml extension", DB::ErrorCodes::BAD_ARGUMENTS);
+                collected_files.push_back(filename);
+            }
+        }
+
+        input_files = std::move(collected_files);
+    }
+    std::sort(input_files.begin(), input_files.end());
+    return input_files;
+}
+
+int mainEntryClickHousePerformanceTest(int argc, char ** argv)
+try
+{
+    using po::value;
+    using Strings = DB::Strings;
+
+
+    po::options_description desc("Allowed options");
+    desc.add_options()
+        ("help", "produce help message")
+        ("lite", "use lite version of output")
+        ("profiles-file", value<std::string>()->default_value(""), "Specify a file with global profiles")
+        ("host,h", value<std::string>()->default_value("localhost"), "")
+        ("port", value<UInt16>()->default_value(9000), "")
+        ("secure,s", "Use TLS connection")
+        ("database", value<std::string>()->default_value("default"), "")
+        ("user", value<std::string>()->default_value("default"), "")
+        ("password", value<std::string>()->default_value(""), "")
+        ("log-level", value<std::string>()->default_value("information"), "Set log level")
+        ("tags", value<Strings>()->multitoken(), "Run only tests with tag")
+        ("skip-tags", value<Strings>()->multitoken(), "Do not run tests with tag")
+        ("names", value<Strings>()->multitoken(), "Run tests with specific name")
+        ("skip-names", value<Strings>()->multitoken(), "Do not run tests with name")
+        ("names-regexp", value<Strings>()->multitoken(), "Run tests with names matching regexp")
+        ("skip-names-regexp", value<Strings>()->multitoken(), "Do not run tests with names matching regexp")
+        ("recursive,r", "Recurse in directories to find all xml's");
+
+    /// These options will not be displayed in --help
+    po::options_description hidden("Hidden options");
+    hidden.add_options()
+        ("input-files", value<std::vector<std::string>>(), "");
+
+    /// But they will be legit, though. And they must be given without name
+    po::positional_options_description positional;
+    positional.add("input-files", -1);
+
+    po::options_description cmdline_options;
+    cmdline_options.add(desc).add(hidden);
+
+    po::variables_map options;
+    po::store(
+        po::command_line_parser(argc, argv).
+        options(cmdline_options).positional(positional).run(), options);
+    po::notify(options);
+
+    Poco::AutoPtr<Poco::PatternFormatter> formatter(new Poco::PatternFormatter("%Y.%m.%d %H:%M:%S.%F <%p> %s: %t"));
+    Poco::AutoPtr<Poco::ConsoleChannel> console_chanel(new Poco::ConsoleChannel);
+    Poco::AutoPtr<Poco::FormattingChannel> channel(new Poco::FormattingChannel(formatter, console_chanel));
+
+    Poco::Logger::root().setLevel(options["log-level"].as<std::string>());
+    Poco::Logger::root().setChannel(channel);
+
+    Poco::Logger * log = &Poco::Logger::get("PerformanceTestSuite");
+    if (options.count("help"))
+    {
+        std::cout << "Usage: " << argv[0] << " [options] [test_file ...] [tests_folder]\n";
+        std::cout << desc << "\n";
+        return 0;
+    }
+
+    Strings input_files = getInputFiles(options, log);
+
+    Strings tests_tags = options.count("tags") ? options["tags"].as<Strings>() : Strings({});
+    Strings skip_tags = options.count("skip-tags") ? options["skip-tags"].as<Strings>() : Strings({});
+    Strings tests_names = options.count("names") ? options["names"].as<Strings>() : Strings({});
+    Strings skip_names = options.count("skip-names") ? options["skip-names"].as<Strings>() : Strings({});
+    Strings tests_names_regexp = options.count("names-regexp") ? options["names-regexp"].as<Strings>() : Strings({});
+    Strings skip_names_regexp = options.count("skip-names-regexp") ? options["skip-names-regexp"].as<Strings>() : Strings({});
+
+    auto timeouts = DB::ConnectionTimeouts::getTCPTimeoutsWithoutFailover(DB::Settings());
+
+    DB::UseSSL use_ssl;
+
+    DB::PerformanceTestSuite performance_test_suite(
+        options["host"].as<std::string>(),
+        options["port"].as<UInt16>(),
+        options.count("secure"),
+        options["database"].as<std::string>(),
+        options["user"].as<std::string>(),
+        options["password"].as<std::string>(),
+        options.count("lite") > 0,
+        options["profiles-file"].as<std::string>(),
+        std::move(input_files),
+        std::move(tests_tags),
+        std::move(skip_tags),
+        std::move(tests_names),
+        std::move(skip_names),
+        std::move(tests_names_regexp),
+        std::move(skip_names_regexp),
+        timeouts);
+    return performance_test_suite.run();
+}
+catch (...)
+{
+    std::cout << DB::getCurrentExceptionMessage(/*with stacktrace = */ true) << std::endl;
+    int code = DB::getCurrentExceptionCode();
+    return code ? code : 1;
+}
--- a/dbms/programs/performance-test/ReportBuilder.cpp
+++ b/dbms/programs/performance-test/ReportBuilder.cpp
@ -0,0 +1,196 @@
+#include "ReportBuilder.h"
+
+#include <algorithm>
+#include <regex>
+#include <sstream>
+#include <thread>
+
+#include <Common/getNumberOfPhysicalCPUCores.h>
+#include <Common/getFQDNOrHostName.h>
+#include <common/getMemoryAmount.h>
+
+#include "JSONString.h"
+
+namespace DB
+{
+
+namespace
+{
+const std::regex QUOTE_REGEX{"\""};
+}
+
+ReportBuilder::ReportBuilder(const std::string & server_version_)
+    : server_version(server_version_)
+    , hostname(getFQDNOrHostName())
+    , num_cores(getNumberOfPhysicalCPUCores())
+    , num_threads(std::thread::hardware_concurrency())
+    , ram(getMemoryAmount())
+{
+}
+
+std::string ReportBuilder::getCurrentTime() const
+{
+    return DateLUT::instance().timeToString(time(nullptr));
+}
+
+std::string ReportBuilder::buildFullReport(
+    const PerformanceTestInfo & test_info,
+    std::vector<TestStats> & stats) const
+{
+    JSONString json_output;
+
+    json_output.set("hostname", hostname);
+    json_output.set("num_cores", num_cores);
+    json_output.set("num_threads", num_threads);
+    json_output.set("ram", ram);
+    json_output.set("server_version", server_version);
+    json_output.set("time", getCurrentTime());
+    json_output.set("test_name", test_info.test_name);
+    json_output.set("path", test_info.path);
+    json_output.set("main_metric", test_info.main_metric);
+
+    auto has_metric = [&test_info] (const std::string & metric_name)
+    {
+        return std::find(test_info.metrics.begin(),
+            test_info.metrics.end(), metric_name) != test_info.metrics.end();
+    };
+
+    if (test_info.substitutions.size())
+    {
+        JSONString json_parameters(2); /// here, 2 is the size of \t padding
+
+        for (auto it = test_info.substitutions.begin(); it != test_info.substitutions.end(); ++it)
+        {
+            std::string parameter = it->first;
+            Strings values = it->second;
+
+            std::ostringstream array_string;
+            array_string << "[";
+            for (size_t i = 0; i != values.size(); ++i)
+            {
+                array_string << '"' << std::regex_replace(values[i], QUOTE_REGEX, "\\\"") << '"';
+                if (i != values.size() - 1)
+                {
+                    array_string << ", ";
+                }
+            }
+            array_string << ']';
+
+            json_parameters.set(parameter, array_string.str());
+        }
+
+        json_output.set("parameters", json_parameters.asString());
+    }
+
+    std::vector<JSONString> run_infos;
+    for (size_t query_index = 0; query_index < test_info.queries.size(); ++query_index)
+    {
+        for (size_t number_of_launch = 0; number_of_launch < test_info.times_to_run; ++number_of_launch)
+        {
+            size_t stat_index = number_of_launch * test_info.queries.size() + query_index;
+            TestStats & statistics = stats[stat_index];
+
+            if (!statistics.ready)
+                continue;
+
+            JSONString runJSON;
+
+            auto query = std::regex_replace(test_info.queries[query_index], QUOTE_REGEX, "\\\"");
+            runJSON.set("query", query);
+            if (!statistics.exception.empty())
+                runJSON.set("exception", statistics.exception);
+
+            if (test_info.exec_type == ExecutionType::Loop)
+            {
+                /// in seconds
+                if (has_metric("min_time"))
+                    runJSON.set("min_time", statistics.min_time / double(1000));
+
+                if (has_metric("quantiles"))
+                {
+                    JSONString quantiles(4); /// here, 4 is the size of \t padding
+                    for (double percent = 10; percent <= 90; percent += 10)
+                    {
+                        std::string quantile_key = std::to_string(percent / 100.0);
+                        while (quantile_key.back() == '0')
+                            quantile_key.pop_back();
+
+                        quantiles.set(quantile_key,
+                            statistics.sampler.quantileInterpolated(percent / 100.0));
+                    }
+                    quantiles.set("0.95",
+                        statistics.sampler.quantileInterpolated(95 / 100.0));
+                    quantiles.set("0.99",
+                        statistics.sampler.quantileInterpolated(99 / 100.0));
+                    quantiles.set("0.999",
+                        statistics.sampler.quantileInterpolated(99.9 / 100.0));
+                    quantiles.set("0.9999",
+                        statistics.sampler.quantileInterpolated(99.99 / 100.0));
+
+                    runJSON.set("quantiles", quantiles.asString());
+                }
+
+                if (has_metric("total_time"))
+                    runJSON.set("total_time", statistics.total_time);
+
+                if (has_metric("queries_per_second"))
+                    runJSON.set("queries_per_second",
+                        double(statistics.queries) / statistics.total_time);
+
+                if (has_metric("rows_per_second"))
+                    runJSON.set("rows_per_second",
+                        double(statistics.total_rows_read) / statistics.total_time);
+
+                if (has_metric("bytes_per_second"))
+                    runJSON.set("bytes_per_second",
+                        double(statistics.total_bytes_read) / statistics.total_time);
+            }
+            else
+            {
+                if (has_metric("max_rows_per_second"))
+                    runJSON.set("max_rows_per_second", statistics.max_rows_speed);
+
+                if (has_metric("max_bytes_per_second"))
+                    runJSON.set("max_bytes_per_second", statistics.max_bytes_speed);
+
+                if (has_metric("avg_rows_per_second"))
+                    runJSON.set("avg_rows_per_second", statistics.avg_rows_speed_value);
+
+                if (has_metric("avg_bytes_per_second"))
+                    runJSON.set("avg_bytes_per_second", statistics.avg_bytes_speed_value);
+            }
+
+            run_infos.push_back(runJSON);
+        }
+    }
+
+    json_output.set("runs", run_infos);
+
+    return json_output.asString();
+}
+
+std::string ReportBuilder::buildCompactReport(
+    const PerformanceTestInfo & test_info,
+    std::vector<TestStats> & stats) const
+{
+
+    std::ostringstream output;
+
+    for (size_t query_index = 0; query_index < test_info.queries.size(); ++query_index)
+    {
+        for (size_t number_of_launch = 0; number_of_launch < test_info.times_to_run; ++number_of_launch)
+        {
+            if (test_info.queries.size() > 1)
+                output << "query \"" << test_info.queries[query_index] << "\", ";
+
+            output << "run " << std::to_string(number_of_launch + 1) << ": ";
+            output << test_info.main_metric << " = ";
+            size_t index = number_of_launch * test_info.queries.size() + query_index;
+            output << stats[index].getStatisticByName(test_info.main_metric);
+            output << "\n";
+        }
+    }
+    return output.str();
+}
+
+}
--- a/dbms/programs/performance-test/ReportBuilder.h
+++ b/dbms/programs/performance-test/ReportBuilder.h
@ -0,0 +1,32 @@
+#pragma once
+#include "PerformanceTestInfo.h"
+#include <vector>
+#include <string>
+
+namespace DB
+{
+
+class ReportBuilder
+{
+public:
+    explicit ReportBuilder(const std::string & server_version_);
+    std::string buildFullReport(
+        const PerformanceTestInfo & test_info,
+        std::vector<TestStats> & stats) const;
+
+    std::string buildCompactReport(
+        const PerformanceTestInfo & test_info,
+        std::vector<TestStats> & stats) const;
+private:
+    std::string server_version;
+    std::string hostname;
+    size_t num_cores;
+    size_t num_threads;
+    size_t ram;
+
+private:
+    std::string getCurrentTime() const;
+
+};
+
+}
--- a/dbms/programs/performance-test/StopConditionsSet.cpp
+++ b/dbms/programs/performance-test/StopConditionsSet.cpp
@ -0,0 +1,63 @@
+#include "StopConditionsSet.h"
+#include <Common/Exception.h>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+extern const int LOGICAL_ERROR;
+}
+
+void StopConditionsSet::loadFromConfig(const ConfigurationPtr & stop_conditions_view)
+{
+    Strings keys;
+    stop_conditions_view->keys(keys);
+
+    for (const std::string & key : keys)
+    {
+        if (key == "total_time_ms")
+            total_time_ms.value = stop_conditions_view->getUInt64(key);
+        else if (key == "rows_read")
+            rows_read.value = stop_conditions_view->getUInt64(key);
+        else if (key == "bytes_read_uncompressed")
+            bytes_read_uncompressed.value = stop_conditions_view->getUInt64(key);
+        else if (key == "iterations")
+            iterations.value = stop_conditions_view->getUInt64(key);
+        else if (key == "min_time_not_changing_for_ms")
+            min_time_not_changing_for_ms.value = stop_conditions_view->getUInt64(key);
+        else if (key == "max_speed_not_changing_for_ms")
+            max_speed_not_changing_for_ms.value = stop_conditions_view->getUInt64(key);
+        else if (key == "average_speed_not_changing_for_ms")
+            average_speed_not_changing_for_ms.value = stop_conditions_view->getUInt64(key);
+        else
+            throw Exception("Met unkown stop condition: " + key, ErrorCodes::LOGICAL_ERROR);
+    }
+    ++initialized_count;
+}
+
+void StopConditionsSet::reset()
+{
+    total_time_ms.fulfilled = false;
+    rows_read.fulfilled = false;
+    bytes_read_uncompressed.fulfilled = false;
+    iterations.fulfilled = false;
+    min_time_not_changing_for_ms.fulfilled = false;
+    max_speed_not_changing_for_ms.fulfilled = false;
+    average_speed_not_changing_for_ms.fulfilled = false;
+
+    fulfilled_count = 0;
+}
+
+void StopConditionsSet::report(UInt64 value, StopConditionsSet::StopCondition & condition)
+{
+    if (condition.value && !condition.fulfilled && value >= condition.value)
+    {
+        condition.fulfilled = true;
+        ++fulfilled_count;
+    }
+}
+
+
+
+}
--- a/Show More
+++ b/Show More
				`@ -0,0 +1 @@`
				`Subproject commit 520465510efef7704346cf8d140967c4abb057c1`
				`@ -0,0 +1 @@`
				`Subproject commit 08879029ab8dcb80a70142acb709e3df02de5d37`