Merge branch 'master' of github.com:yandex/ClickHouse into group_by_in_order_optimization

.github/ISSUE_TEMPLATE/backward-compatibility.md

@@ -17,7 +17,7 @@ A clear and concise description of what works not as it is supposed to.
 * Which interface to use, if matters
 * Non-default settings, if any
 * `CREATE TABLE` statements for all tables involved
-* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
+* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
 * Queries to run that lead to unexpected result
 
 **Error message and/or stacktrace**

.github/ISSUE_TEMPLATE/bug-report.md

@@ -17,7 +17,7 @@ A clear and concise description of what works not as it is supposed to.
 * Which interface to use, if matters
 * Non-default settings, if any
 * `CREATE TABLE` statements for all tables involved
-* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
+* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
 * Queries to run that lead to unexpected result
 
 **Expected behavior**

.github/ISSUE_TEMPLATE/performance-issue.md

@@ -17,7 +17,7 @@ What exactly works slower than expected?
 * Which interface to use, if matters
 * Non-default settings, if any
 * `CREATE TABLE` statements for all tables involved
-* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
+* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
 * Queries to run that lead to slow performance
 
 **Expected performance**

.github/ISSUE_TEMPLATE/unexpected-behaviour.md

@@ -17,7 +17,7 @@ A clear and concise description of what works not as it is supposed to.
 * Which interface to use, if matters
 * Non-default settings, if any
 * `CREATE TABLE` statements for all tables involved
-* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
+* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
 * Queries to run that lead to unexpected result
 
 **Expected behavior**

.github/ISSUE_TEMPLATE/usability-issue.md

@@ -17,7 +17,7 @@ A clear and concise description of what works not as it is supposed to.
 * Which interface to use, if matters
 * Non-default settings, if any
 * `CREATE TABLE` statements for all tables involved
-* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
+* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
 * Queries to run that lead to unexpected result
 
 **Expected behavior**

.gitignore

@@ -16,9 +16,9 @@
 /docs/publish
 /docs/edit
 /docs/website
-/docs/venv/
-/docs/tools/venv/
-/docs/tools/translate/venv/
+/docs/venv
+/docs/tools/venv
+/docs/tools/translate/venv
 /docs/tools/translate/output.md
 /docs/en/single.md
 /docs/ru/single.md
@@ -73,100 +73,100 @@ contrib/libpoco/Poco/
 contrib/libpoco/bin/
 contrib/libpoco/cmake_uninstall.cmake
 contrib/libre2/re2_st/
-dbms/src/Client/clickhouse-benchmark
-dbms/src/Client/clickhouse-client
-dbms/src/Client/tests/test-connect
-dbms/src/Common/tests/arena_with_free_lists
-dbms/src/Common/tests/auto_array
-dbms/src/Common/tests/compact_array
-dbms/src/Common/tests/hash_table
-dbms/src/Common/tests/hashes_test
-dbms/src/Common/tests/int_hashes_perf
-dbms/src/Common/tests/lru_cache
-dbms/src/Common/tests/parallel_aggregation
-dbms/src/Common/tests/parallel_aggregation2
-dbms/src/Common/tests/radix_sort
-dbms/src/Common/tests/shell_command_test
-dbms/src/Common/tests/simple_cache
-dbms/src/Common/tests/sip_hash
-dbms/src/Common/tests/sip_hash_perf
-dbms/src/Common/tests/small_table
-dbms/src/Core/tests/exception
-dbms/src/Core/tests/field
-dbms/src/Core/tests/rvo_test
-dbms/src/Core/tests/string_pool
-dbms/src/DataStreams/tests/aggregating_stream
-dbms/src/DataStreams/tests/block_tab_separated_streams
-dbms/src/DataStreams/tests/collapsing_sorted_stream
-dbms/src/DataStreams/tests/expression_stream
-dbms/src/DataStreams/tests/filter_stream
-dbms/src/DataStreams/tests/filter_stream_hitlog
-dbms/src/DataStreams/tests/fork_streams
-dbms/src/DataStreams/tests/glue_streams
-dbms/src/DataStreams/tests/json_streams
-dbms/src/DataStreams/tests/native_streams
-dbms/src/DataStreams/tests/sorting_stream
-dbms/src/DataStreams/tests/tab_separated_streams
-dbms/src/DataStreams/tests/union_stream
-dbms/src/DataStreams/tests/union_stream2
-dbms/src/DataTypes/tests/data_type_string
-dbms/src/DataTypes/tests/data_types_number_fixed
-dbms/src/Functions/tests/functions_arithmetic
-dbms/src/Functions/tests/logical_functions_performance
-dbms/src/Functions/tests/number_traits
-dbms/src/IO/tests/async_write
-dbms/src/IO/tests/cached_compressed_read_buffer
-dbms/src/IO/tests/compressed_buffer
-dbms/src/IO/tests/hashing_read_buffer
-dbms/src/IO/tests/hashing_write_buffer
-dbms/src/IO/tests/io_and_exceptions
-dbms/src/IO/tests/io_operators
-dbms/src/IO/tests/mempbrk
-dbms/src/IO/tests/o_direct_and_dirty_pages
-dbms/src/IO/tests/parse_int_perf
-dbms/src/IO/tests/parse_int_perf2
-dbms/src/IO/tests/read_buffer
-dbms/src/IO/tests/read_buffer_aio
-dbms/src/IO/tests/read_buffer_perf
-dbms/src/IO/tests/read_escaped_string
-dbms/src/IO/tests/read_float_perf
-dbms/src/IO/tests/read_write_int
-dbms/src/IO/tests/valid_utf8
-dbms/src/IO/tests/valid_utf8_perf
-dbms/src/IO/tests/var_uint
-dbms/src/IO/tests/write_buffer
-dbms/src/IO/tests/write_buffer_aio
-dbms/src/IO/tests/write_buffer_perf
-dbms/src/Interpreters/tests/address_patterns
-dbms/src/Interpreters/tests/aggregate
-dbms/src/Interpreters/tests/compiler_test
-dbms/src/Interpreters/tests/create_query
-dbms/src/Interpreters/tests/expression
-dbms/src/Interpreters/tests/expression_analyzer
-dbms/src/Interpreters/tests/hash_map
-dbms/src/Interpreters/tests/hash_map2
-dbms/src/Interpreters/tests/hash_map3
-dbms/src/Interpreters/tests/hash_map_string
-dbms/src/Interpreters/tests/hash_map_string_2
-dbms/src/Interpreters/tests/hash_map_string_3
-dbms/src/Interpreters/tests/hash_map_string_small
-dbms/src/Interpreters/tests/in_join_subqueries_preprocessor
-dbms/src/Interpreters/tests/logical_expressions_optimizer
-dbms/src/Interpreters/tests/select_query
-dbms/src/Interpreters/tests/two_level_hash_map
-dbms/src/Interpreters/tests/users
-dbms/src/Parsers/tests/create_parser
-dbms/src/Parsers/tests/select_parser
-dbms/src/Server/clickhouse-server
-dbms/src/Server/clickhouse-server.init
-dbms/src/Storages/tests/hit_log
-dbms/src/Storages/tests/merge_tree
-dbms/src/Storages/tests/part_checker
-dbms/src/Storages/tests/part_name
-dbms/src/Storages/tests/pk_condition
-dbms/src/Storages/tests/seek_speed_test
-dbms/src/Storages/tests/storage_log
-dbms/src/Storages/tests/system_numbers
+src/Client/clickhouse-benchmark
+src/Client/clickhouse-client
+src/Client/tests/test-connect
+src/Common/tests/arena_with_free_lists
+src/Common/tests/auto_array
+src/Common/tests/compact_array
+src/Common/tests/hash_table
+src/Common/tests/hashes_test
+src/Common/tests/int_hashes_perf
+src/Common/tests/lru_cache
+src/Common/tests/parallel_aggregation
+src/Common/tests/parallel_aggregation2
+src/Common/tests/radix_sort
+src/Common/tests/shell_command_test
+src/Common/tests/simple_cache
+src/Common/tests/sip_hash
+src/Common/tests/sip_hash_perf
+src/Common/tests/small_table
+src/Core/tests/exception
+src/Core/tests/field
+src/Core/tests/rvo_test
+src/Core/tests/string_pool
+src/DataStreams/tests/aggregating_stream
+src/DataStreams/tests/block_tab_separated_streams
+src/DataStreams/tests/collapsing_sorted_stream
+src/DataStreams/tests/expression_stream
+src/DataStreams/tests/filter_stream
+src/DataStreams/tests/filter_stream_hitlog
+src/DataStreams/tests/fork_streams
+src/DataStreams/tests/glue_streams
+src/DataStreams/tests/json_streams
+src/DataStreams/tests/native_streams
+src/DataStreams/tests/sorting_stream
+src/DataStreams/tests/tab_separated_streams
+src/DataStreams/tests/union_stream
+src/DataStreams/tests/union_stream2
+src/DataTypes/tests/data_type_string
+src/DataTypes/tests/data_types_number_fixed
+src/Functions/tests/functions_arithmetic
+src/Functions/tests/logical_functions_performance
+src/Functions/tests/number_traits
+src/IO/tests/async_write
+src/IO/tests/cached_compressed_read_buffer
+src/IO/tests/compressed_buffer
+src/IO/tests/hashing_read_buffer
+src/IO/tests/hashing_write_buffer
+src/IO/tests/io_and_exceptions
+src/IO/tests/io_operators
+src/IO/tests/mempbrk
+src/IO/tests/o_direct_and_dirty_pages
+src/IO/tests/parse_int_perf
+src/IO/tests/parse_int_perf2
+src/IO/tests/read_buffer
+src/IO/tests/read_buffer_aio
+src/IO/tests/read_buffer_perf
+src/IO/tests/read_escaped_string
+src/IO/tests/read_float_perf
+src/IO/tests/read_write_int
+src/IO/tests/valid_utf8
+src/IO/tests/valid_utf8_perf
+src/IO/tests/var_uint
+src/IO/tests/write_buffer
+src/IO/tests/write_buffer_aio
+src/IO/tests/write_buffer_perf
+src/Interpreters/tests/address_patterns
+src/Interpreters/tests/aggregate
+src/Interpreters/tests/compiler_test
+src/Interpreters/tests/create_query
+src/Interpreters/tests/expression
+src/Interpreters/tests/expression_analyzer
+src/Interpreters/tests/hash_map
+src/Interpreters/tests/hash_map2
+src/Interpreters/tests/hash_map3
+src/Interpreters/tests/hash_map_string
+src/Interpreters/tests/hash_map_string_2
+src/Interpreters/tests/hash_map_string_3
+src/Interpreters/tests/hash_map_string_small
+src/Interpreters/tests/in_join_subqueries_preprocessor
+src/Interpreters/tests/logical_expressions_optimizer
+src/Interpreters/tests/select_query
+src/Interpreters/tests/two_level_hash_map
+src/Interpreters/tests/users
+src/Parsers/tests/create_parser
+src/Parsers/tests/select_parser
+src/Server/clickhouse-server
+src/Server/clickhouse-server.init
+src/Storages/tests/hit_log
+src/Storages/tests/merge_tree
+src/Storages/tests/part_checker
+src/Storages/tests/part_name
+src/Storages/tests/pk_condition
+src/Storages/tests/seek_speed_test
+src/Storages/tests/storage_log
+src/Storages/tests/system_numbers
 libs/libcommon/src/revision.h
 libs/libcommon/src/tests/date_lut2
 libs/libcommon/src/tests/date_lut3
@@ -184,15 +184,15 @@ libs/libzkutil/src/tests/zkutil_zookeeper_holder
 utils/zookeeper-create-entry-to-download-part/zookeeper-create-entry-to-download-part
 utils/zookeeper-dump-tree/zookeeper-dump-tree
 utils/zookeeper-remove-by-list/zookeeper-remove-by-list
-dbms/src/Storages/tests/remove_symlink_directory
+src/Storages/tests/remove_symlink_directory
 libs/libcommon/src/tests/json_test
 utils/compressor/zstd_test
 utils/wikistat-loader/wikistat-loader
-dbms/src/Common/tests/pod_array
+src/Common/tests/pod_array
 
-dbms/src/Server/data/*
-dbms/src/Server/metadata/*
-dbms/src/Server/status
+src/Server/data/*
+src/Server/metadata/*
+src/Server/status
 config-9001.xml
 
 *-preprocessed.xml
@@ -242,7 +242,7 @@ website/package-lock.json
 */.DS_Store
 
 # Ignore files for locally disabled tests
-/dbms/tests/queries/**/*.disabled
+/src/queries/**/*.disabled
 
 # cquery cache
 /.cquery-cache

.gitlab-ci.yml

@@ -31,11 +31,11 @@ build:
     - docker pull $CI_REGISTRY/yandex/clickhouse-builder
     - docker run --rm --volumes-from "${HOSTNAME}-build" --workdir "${CI_PROJECT_DIR}" --env CI_PROJECT_DIR=${CI_PROJECT_DIR} $CI_REGISTRY/yandex/clickhouse-builder /build_gitlab_ci.sh
   # You can upload your binary to nexus
-    - curl -v --keepalive-time 60 --keepalive --user "$NEXUS_USER:$NEXUS_PASSWORD" -XPUT "http://$NEXUS_HOST/repository/binaries/$CI_PROJECT_NAME" --upload-file ./dbms/src/Server/clickhouse
+    - curl -v --keepalive-time 60 --keepalive --user "$NEXUS_USER:$NEXUS_PASSWORD" -XPUT "http://$NEXUS_HOST/repository/binaries/$CI_PROJECT_NAME" --upload-file ./src/Server/clickhouse
   # Or download artifacts from gitlab
   artifacts:
     paths:
-      - ./dbms/src/Server/clickhouse
+      - ./src/Server/clickhouse
     expire_in: 1 day
   tags:
-    - docker
+    - docker

.gitmodules

@@ -13,7 +13,7 @@
 	url = https://github.com/edenhill/librdkafka.git
 [submodule "contrib/cctz"]
 	path = contrib/cctz
-	url = https://github.com/google/cctz.git
+	url = https://github.com/ClickHouse-Extras/cctz.git
 [submodule "contrib/zlib-ng"]
 	path = contrib/zlib-ng
 	url = https://github.com/ClickHouse-Extras/zlib-ng.git
@@ -151,3 +151,6 @@
 [submodule "website/images/feathericons"]
 	path = website/images/feathericons
 	url = https://github.com/feathericons/feather
+[submodule "contrib/msgpack-c"]
+	path = contrib/msgpack-c
+	url = https://github.com/msgpack/msgpack-c

CHANGELOG.md

@@ -1,5 +1,80 @@
 ## ClickHouse release v20.3
 
+### ClickHouse release v20.3.7.46, 2020-04-17
+
+#### Bug Fix
+
+* Fix `Logical error: CROSS JOIN has expressions` error for queries with comma and names joins mix. [#10311](https://github.com/ClickHouse/ClickHouse/pull/10311) ([Artem Zuikov](https://github.com/4ertus2)).
+* Fix queries with `max_bytes_before_external_group_by`. [#10302](https://github.com/ClickHouse/ClickHouse/pull/10302) ([Artem Zuikov](https://github.com/4ertus2)).
+* Fix move-to-prewhere optimization in presense of arrayJoin functions (in certain cases). This fixes [#10092](https://github.com/ClickHouse/ClickHouse/issues/10092). [#10195](https://github.com/ClickHouse/ClickHouse/pull/10195) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Add the ability to relax the restriction on non-deterministic functions usage in mutations with `allow_nondeterministic_mutations` setting. [#10186](https://github.com/ClickHouse/ClickHouse/pull/10186) ([filimonov](https://github.com/filimonov)).
+
+### ClickHouse release v20.3.6.40, 2020-04-16
+
+#### New Feature
+
+* Added function `isConstant`. This function checks whether its argument is constant expression and returns 1 or 0. It is intended for development, debugging and demonstration purposes. [#10198](https://github.com/ClickHouse/ClickHouse/pull/10198) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+
+#### Bug Fix
+
+* Fix error `Pipeline stuck` with `max_rows_to_group_by` and `group_by_overflow_mode = 'break'`. [#10279](https://github.com/ClickHouse/ClickHouse/pull/10279) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix rare possible exception `Cannot drain connections: cancel first`. [#10239](https://github.com/ClickHouse/ClickHouse/pull/10239) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fixed bug where ClickHouse would throw "Unknown function lambda." error message when user tries to run ALTER UPDATE/DELETE on tables with ENGINE = Replicated*. Check for nondeterministic functions now handles lambda expressions correctly. [#10237](https://github.com/ClickHouse/ClickHouse/pull/10237) ([Alexander Kazakov](https://github.com/Akazz)).
+* Fixed "generateRandom" function for Date type. This fixes [#9973](https://github.com/ClickHouse/ClickHouse/issues/9973). Fix an edge case when dates with year 2106 are inserted to MergeTree tables with old-style partitioning but partitions are named with year 1970. [#10218](https://github.com/ClickHouse/ClickHouse/pull/10218) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Convert types if the table definition of a View does not correspond to the SELECT query. This fixes [#10180](https://github.com/ClickHouse/ClickHouse/issues/10180) and [#10022](https://github.com/ClickHouse/ClickHouse/issues/10022). [#10217](https://github.com/ClickHouse/ClickHouse/pull/10217) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix `parseDateTimeBestEffort` for strings in RFC-2822 when day of week is Tuesday or Thursday. This fixes [#10082](https://github.com/ClickHouse/ClickHouse/issues/10082). [#10214](https://github.com/ClickHouse/ClickHouse/pull/10214) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix column names of constants inside JOIN that may clash with names of constants outside of JOIN. [#10207](https://github.com/ClickHouse/ClickHouse/pull/10207) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix possible inifinite query execution when the query actually should stop on LIMIT, while reading from infinite source like `system.numbers` or `system.zeros`. [#10206](https://github.com/ClickHouse/ClickHouse/pull/10206) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix using the current database for access checking when the database isn't specified. [#10192](https://github.com/ClickHouse/ClickHouse/pull/10192) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Convert blocks if structure does not match on INSERT into Distributed(). [#10135](https://github.com/ClickHouse/ClickHouse/pull/10135) ([Azat Khuzhin](https://github.com/azat)).
+* Fix possible incorrect result for extremes in processors pipeline. [#10131](https://github.com/ClickHouse/ClickHouse/pull/10131) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix some kinds of alters with compact parts. [#10130](https://github.com/ClickHouse/ClickHouse/pull/10130) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix incorrect `index_granularity_bytes` check while creating new replica. Fixes [#10098](https://github.com/ClickHouse/ClickHouse/issues/10098). [#10121](https://github.com/ClickHouse/ClickHouse/pull/10121) ([alesapin](https://github.com/alesapin)).
+* Fix SIGSEGV on INSERT into Distributed table when its structure differs from the underlying tables. [#10105](https://github.com/ClickHouse/ClickHouse/pull/10105) ([Azat Khuzhin](https://github.com/azat)).
+* Fix possible rows loss for queries with `JOIN` and `UNION ALL`. Fixes [#9826](https://github.com/ClickHouse/ClickHouse/issues/9826), [#10113](https://github.com/ClickHouse/ClickHouse/issues/10113). [#10099](https://github.com/ClickHouse/ClickHouse/pull/10099) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fixed replicated tables startup when updating from an old ClickHouse version where `/table/replicas/replica_name/metadata` node doesn't exist. Fixes [#10037](https://github.com/ClickHouse/ClickHouse/issues/10037). [#10095](https://github.com/ClickHouse/ClickHouse/pull/10095) ([alesapin](https://github.com/alesapin)).
+* Add some arguments check and support identifier arguments for MySQL Database Engine. [#10077](https://github.com/ClickHouse/ClickHouse/pull/10077) ([Winter Zhang](https://github.com/zhang2014)).
+* Fix bug in clickhouse dictionary source from localhost clickhouse server. The bug may lead to memory corruption if types in dictionary and source are not compatible. [#10071](https://github.com/ClickHouse/ClickHouse/pull/10071) ([alesapin](https://github.com/alesapin)).
+* Fix bug in `CHECK TABLE` query when table contain skip indices. [#10068](https://github.com/ClickHouse/ClickHouse/pull/10068) ([alesapin](https://github.com/alesapin)).
+* Fix error `Cannot clone block with columns because block has 0 columns ... While executing GroupingAggregatedTransform`. It happened when setting `distributed_aggregation_memory_efficient` was enabled, and distributed query read aggregating data with different level from different shards (mixed single and two level aggregation). [#10063](https://github.com/ClickHouse/ClickHouse/pull/10063) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix a segmentation fault that could occur in GROUP BY over string keys containing trailing zero bytes ([#8636](https://github.com/ClickHouse/ClickHouse/issues/8636), [#8925](https://github.com/ClickHouse/ClickHouse/issues/8925)). [#10025](https://github.com/ClickHouse/ClickHouse/pull/10025) ([Alexander Kuzmenkov](https://github.com/akuzm)).
+* Fix parallel distributed INSERT SELECT for remote table. This PR fixes the solution provided in [#9759](https://github.com/ClickHouse/ClickHouse/pull/9759). [#9999](https://github.com/ClickHouse/ClickHouse/pull/9999) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix the number of threads used for remote query execution (performance regression, since 20.3). This happened when query from `Distributed` table was executed simultaneously on local and remote shards. Fixes [#9965](https://github.com/ClickHouse/ClickHouse/issues/9965). [#9971](https://github.com/ClickHouse/ClickHouse/pull/9971) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix bug in which the necessary tables weren't retrieved at one of the processing stages of queries to some databases. Fixes [#9699](https://github.com/ClickHouse/ClickHouse/issues/9699). [#9949](https://github.com/ClickHouse/ClickHouse/pull/9949) ([achulkov2](https://github.com/achulkov2)).
+* Fix 'Not found column in block' error when `JOIN` appears with `TOTALS`. Fixes [#9839](https://github.com/ClickHouse/ClickHouse/issues/9839). [#9939](https://github.com/ClickHouse/ClickHouse/pull/9939) ([Artem Zuikov](https://github.com/4ertus2)).
+* Fix a bug with `ON CLUSTER` DDL queries freezing on server startup. [#9927](https://github.com/ClickHouse/ClickHouse/pull/9927) ([Gagan Arneja](https://github.com/garneja)).
+* Fix parsing multiple hosts set in the CREATE USER command, e.g. `CREATE USER user6 HOST NAME REGEXP 'lo.?*host', NAME REGEXP 'lo*host'`. [#9924](https://github.com/ClickHouse/ClickHouse/pull/9924) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix `TRUNCATE` for Join table engine ([#9917](https://github.com/ClickHouse/ClickHouse/issues/9917)). [#9920](https://github.com/ClickHouse/ClickHouse/pull/9920) ([Amos Bird](https://github.com/amosbird)).
+* Fix "scalar doesn't exist" error in ALTERs ([#9878](https://github.com/ClickHouse/ClickHouse/issues/9878)). [#9904](https://github.com/ClickHouse/ClickHouse/pull/9904) ([Amos Bird](https://github.com/amosbird)).
+* Fix race condition between drop and optimize in `ReplicatedMergeTree`. [#9901](https://github.com/ClickHouse/ClickHouse/pull/9901) ([alesapin](https://github.com/alesapin)).
+* Fix error with qualified names in `distributed_product_mode='local'`. Fixes [#4756](https://github.com/ClickHouse/ClickHouse/issues/4756). [#9891](https://github.com/ClickHouse/ClickHouse/pull/9891) ([Artem Zuikov](https://github.com/4ertus2)).
+* Fix calculating grants for introspection functions from the setting 'allow_introspection_functions'. [#9840](https://github.com/ClickHouse/ClickHouse/pull/9840) ([Vitaly Baranov](https://github.com/vitlibar)).
+
+#### Build/Testing/Packaging Improvement
+
+* Fix integration test `test_settings_constraints`. [#9962](https://github.com/ClickHouse/ClickHouse/pull/9962) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Removed dependency on `clock_getres`. [#9833](https://github.com/ClickHouse/ClickHouse/pull/9833) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+
+
+### ClickHouse release v20.3.5.21, 2020-03-27
+
+#### Bug Fix
+
+* Fix 'Different expressions with the same alias' error when query has PREWHERE and WHERE on distributed table and `SET distributed_product_mode = 'local'`. [#9871](https://github.com/ClickHouse/ClickHouse/pull/9871) ([Artem Zuikov](https://github.com/4ertus2)).
+* Fix mutations excessive memory consumption for tables with a composite primary key. This fixes [#9850](https://github.com/ClickHouse/ClickHouse/issues/9850). [#9860](https://github.com/ClickHouse/ClickHouse/pull/9860) ([alesapin](https://github.com/alesapin)).
+* For INSERT queries shard now clamps the settings got from the initiator to the shard's constaints instead of throwing an exception. This fix allows to send INSERT queries to a shard with another constraints. This change improves fix [#9447](https://github.com/ClickHouse/ClickHouse/issues/9447). [#9852](https://github.com/ClickHouse/ClickHouse/pull/9852) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix 'COMMA to CROSS JOIN rewriter is not enabled or cannot rewrite query' error in case of subqueries with COMMA JOIN out of tables lists (i.e. in WHERE). Fixes [#9782](https://github.com/ClickHouse/ClickHouse/issues/9782). [#9830](https://github.com/ClickHouse/ClickHouse/pull/9830) ([Artem Zuikov](https://github.com/4ertus2)).
+* Fix possible exception `Got 0 in totals chunk, expected 1` on client. It happened for queries with `JOIN` in case if right joined table had zero rows. Example: `select * from system.one t1 join system.one t2 on t1.dummy = t2.dummy limit 0 FORMAT TabSeparated;`. Fixes [#9777](https://github.com/ClickHouse/ClickHouse/issues/9777). [#9823](https://github.com/ClickHouse/ClickHouse/pull/9823) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix SIGSEGV with optimize_skip_unused_shards when type cannot be converted. [#9804](https://github.com/ClickHouse/ClickHouse/pull/9804) ([Azat Khuzhin](https://github.com/azat)).
+* Fix broken `ALTER TABLE DELETE COLUMN` query for compact parts. [#9779](https://github.com/ClickHouse/ClickHouse/pull/9779) ([alesapin](https://github.com/alesapin)).
+* Fix max_distributed_connections (w/ and w/o Processors). [#9673](https://github.com/ClickHouse/ClickHouse/pull/9673) ([Azat Khuzhin](https://github.com/azat)).
+* Fixed a few cases when timezone of the function argument wasn't used properly. [#9574](https://github.com/ClickHouse/ClickHouse/pull/9574) ([Vasily Nemkov](https://github.com/Enmk)).
+
+#### Improvement
+
+* Remove order by stage from mutations because we read from a single ordered part in a single thread. Also add check that the order of rows in mutation is ordered in sorting key order and this order is not violated. [#9886](https://github.com/ClickHouse/ClickHouse/pull/9886) ([alesapin](https://github.com/alesapin)).
+
+
 ### ClickHouse release v20.3.4.10, 2020-03-20
 
 #### Bug Fix
@@ -234,7 +309,7 @@
 * Updated checking for hung queries in clickhouse-test script [#8858](https://github.com/ClickHouse/ClickHouse/pull/8858) ([Alexander Kazakov](https://github.com/Akazz))
 * Removed some useless files from repository. [#8843](https://github.com/ClickHouse/ClickHouse/pull/8843) ([alexey-milovidov](https://github.com/alexey-milovidov))
 * Changed type of math perftests from `once` to `loop`. [#8783](https://github.com/ClickHouse/ClickHouse/pull/8783) ([Nikolai Kochetov](https://github.com/KochetovNicolai))
-* Add docker image which allows to build interactive code browser HTML report for our codebase. [#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([alesapin](https://github.com/alesapin)) See [Woboq Code Browser](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/src/index.html)
+* Add docker image which allows to build interactive code browser HTML report for our codebase. [#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([alesapin](https://github.com/alesapin)) See [Woboq Code Browser](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/index.html)
 * Suppress some test failures under MSan. [#8780](https://github.com/ClickHouse/ClickHouse/pull/8780) ([Alexander Kuzmenkov](https://github.com/akuzm))
 * Speedup "exception while insert" test. This test often time out in debug-with-coverage build. [#8711](https://github.com/ClickHouse/ClickHouse/pull/8711) ([alexey-milovidov](https://github.com/alexey-milovidov))
 * Updated `libcxx` and `libcxxabi` to master. In preparation to [#9304](https://github.com/ClickHouse/ClickHouse/issues/9304) [#9308](https://github.com/ClickHouse/ClickHouse/pull/9308) ([alexey-milovidov](https://github.com/alexey-milovidov))
@@ -255,6 +330,55 @@
 
 ## ClickHouse release v20.1
 
+### ClickHouse release v20.1.10.70, 2020-04-17
+
+#### Bug Fix
+
+* Fix rare possible exception `Cannot drain connections: cancel first`. [#10239](https://github.com/ClickHouse/ClickHouse/pull/10239) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fixed bug where ClickHouse would throw `'Unknown function lambda.'` error message when user tries to run `ALTER UPDATE/DELETE` on tables with `ENGINE = Replicated*`. Check for nondeterministic functions now handles lambda expressions correctly. [#10237](https://github.com/ClickHouse/ClickHouse/pull/10237) ([Alexander Kazakov](https://github.com/Akazz)).
+* Fix `parseDateTimeBestEffort` for strings in RFC-2822 when day of week is Tuesday or Thursday. This fixes [#10082](https://github.com/ClickHouse/ClickHouse/issues/10082). [#10214](https://github.com/ClickHouse/ClickHouse/pull/10214) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix column names of constants inside `JOIN` that may clash with names of constants outside of `JOIN`. [#10207](https://github.com/ClickHouse/ClickHouse/pull/10207) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix possible inifinite query execution when the query actually should stop on LIMIT, while reading from infinite source like `system.numbers` or `system.zeros`. [#10206](https://github.com/ClickHouse/ClickHouse/pull/10206) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix move-to-prewhere optimization in presense of `arrayJoin` functions (in certain cases). This fixes [#10092](https://github.com/ClickHouse/ClickHouse/issues/10092). [#10195](https://github.com/ClickHouse/ClickHouse/pull/10195) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Add the ability to relax the restriction on non-deterministic functions usage in mutations with `allow_nondeterministic_mutations` setting. [#10186](https://github.com/ClickHouse/ClickHouse/pull/10186) ([filimonov](https://github.com/filimonov)).
+* Convert blocks if structure does not match on `INSERT` into table with `Distributed` engine. [#10135](https://github.com/ClickHouse/ClickHouse/pull/10135) ([Azat Khuzhin](https://github.com/azat)).
+* Fix `SIGSEGV` on `INSERT` into `Distributed` table when its structure differs from the underlying tables. [#10105](https://github.com/ClickHouse/ClickHouse/pull/10105) ([Azat Khuzhin](https://github.com/azat)).
+* Fix possible rows loss for queries with `JOIN` and `UNION ALL`. Fixes [#9826](https://github.com/ClickHouse/ClickHouse/issues/9826), [#10113](https://github.com/ClickHouse/ClickHouse/issues/10113). [#10099](https://github.com/ClickHouse/ClickHouse/pull/10099) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Add arguments check and support identifier arguments for MySQL Database Engine. [#10077](https://github.com/ClickHouse/ClickHouse/pull/10077) ([Winter Zhang](https://github.com/zhang2014)).
+* Fix bug in clickhouse dictionary source from localhost clickhouse server. The bug may lead to memory corruption if types in dictionary and source are not compatible. [#10071](https://github.com/ClickHouse/ClickHouse/pull/10071) ([alesapin](https://github.com/alesapin)).
+* Fix error `Cannot clone block with columns because block has 0 columns ... While executing GroupingAggregatedTransform`. It happened when setting `distributed_aggregation_memory_efficient` was enabled, and distributed query read aggregating data with different level from different shards (mixed single and two level aggregation). [#10063](https://github.com/ClickHouse/ClickHouse/pull/10063) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix a segmentation fault that could occur in `GROUP BY` over string keys containing trailing zero bytes ([#8636](https://github.com/ClickHouse/ClickHouse/issues/8636), [#8925](https://github.com/ClickHouse/ClickHouse/issues/8925)). [#10025](https://github.com/ClickHouse/ClickHouse/pull/10025) ([Alexander Kuzmenkov](https://github.com/akuzm)).
+* Fix bug in which the necessary tables weren't retrieved at one of the processing stages of queries to some databases. Fixes [#9699](https://github.com/ClickHouse/ClickHouse/issues/9699). [#9949](https://github.com/ClickHouse/ClickHouse/pull/9949) ([achulkov2](https://github.com/achulkov2)).
+* Fix `'Not found column in block'` error when `JOIN` appears with `TOTALS`. Fixes [#9839](https://github.com/ClickHouse/ClickHouse/issues/9839). [#9939](https://github.com/ClickHouse/ClickHouse/pull/9939) ([Artem Zuikov](https://github.com/4ertus2)).
+* Fix a bug with `ON CLUSTER` DDL queries freezing on server startup. [#9927](https://github.com/ClickHouse/ClickHouse/pull/9927) ([Gagan Arneja](https://github.com/garneja)).
+* Fix `TRUNCATE` for Join table engine ([#9917](https://github.com/ClickHouse/ClickHouse/issues/9917)). [#9920](https://github.com/ClickHouse/ClickHouse/pull/9920) ([Amos Bird](https://github.com/amosbird)).
+* Fix `'scalar doesn't exist'` error in ALTER queries ([#9878](https://github.com/ClickHouse/ClickHouse/issues/9878)). [#9904](https://github.com/ClickHouse/ClickHouse/pull/9904) ([Amos Bird](https://github.com/amosbird)).
+* Fix race condition between drop and optimize in `ReplicatedMergeTree`. [#9901](https://github.com/ClickHouse/ClickHouse/pull/9901) ([alesapin](https://github.com/alesapin)).
+* Fixed `DeleteOnDestroy` logic in `ATTACH PART` which could lead to automatic removal of attached part and added few tests. [#9410](https://github.com/ClickHouse/ClickHouse/pull/9410) ([Vladimir Chebotarev](https://github.com/excitoon)).
+
+#### Build/Testing/Packaging Improvement
+
+* Fix unit test `collapsing_sorted_stream`. [#9367](https://github.com/ClickHouse/ClickHouse/pull/9367) ([Deleted user](https://github.com/ghost)).
+
+### ClickHouse release v20.1.9.54, 2020-03-28
+
+#### Bug Fix
+
+* Fix `'Different expressions with the same alias'` error when query has `PREWHERE` and `WHERE` on distributed table and `SET distributed_product_mode = 'local'`. [#9871](https://github.com/ClickHouse/ClickHouse/pull/9871) ([Artem Zuikov](https://github.com/4ertus2)).
+* Fix mutations excessive memory consumption for tables with a composite primary key. This fixes [#9850](https://github.com/ClickHouse/ClickHouse/issues/9850). [#9860](https://github.com/ClickHouse/ClickHouse/pull/9860) ([alesapin](https://github.com/alesapin)).
+* For INSERT queries shard now clamps the settings got from the initiator to the shard's constaints instead of throwing an exception. This fix allows to send `INSERT` queries to a shard with another constraints. This change improves fix [#9447](https://github.com/ClickHouse/ClickHouse/issues/9447). [#9852](https://github.com/ClickHouse/ClickHouse/pull/9852) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix possible exception `Got 0 in totals chunk, expected 1` on client. It happened for queries with `JOIN` in case if right joined table had zero rows. Example: `select * from system.one t1 join system.one t2 on t1.dummy = t2.dummy limit 0 FORMAT TabSeparated;`. Fixes [#9777](https://github.com/ClickHouse/ClickHouse/issues/9777). [#9823](https://github.com/ClickHouse/ClickHouse/pull/9823) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix `SIGSEGV` with `optimize_skip_unused_shards` when type cannot be converted. [#9804](https://github.com/ClickHouse/ClickHouse/pull/9804) ([Azat Khuzhin](https://github.com/azat)).
+* Fixed a few cases when timezone of the function argument wasn't used properly. [#9574](https://github.com/ClickHouse/ClickHouse/pull/9574) ([Vasily Nemkov](https://github.com/Enmk)).
+
+#### Improvement
+
+* Remove `ORDER BY` stage from mutations because we read from a single ordered part in a single thread. Also add check that the order of rows in mutation is ordered in sorting key order and this order is not violated. [#9886](https://github.com/ClickHouse/ClickHouse/pull/9886) ([alesapin](https://github.com/alesapin)).
+
+#### Build/Testing/Packaging Improvement
+
+* Clean up duplicated linker flags. Make sure the linker won't look up an unexpected symbol. [#9433](https://github.com/ClickHouse/ClickHouse/pull/9433) ([Amos Bird](https://github.com/amosbird)).
+
 ### ClickHouse release v20.1.8.41, 2020-03-20
 
 #### Bug Fix
@@ -341,7 +465,7 @@
 [#9513](https://github.com/ClickHouse/ClickHouse/pull/9513) [(filimonov)](https://github.com/filimonov)
 
 #### New Feature
-* Add `deduplicate_blocks_in_dependent_materialized_views` option to control the behaviour of idempotent inserts into tables with materialized views. This new feature was added to the bugfix release by a special request from Altinity. 
+* Add `deduplicate_blocks_in_dependent_materialized_views` option to control the behaviour of idempotent inserts into tables with materialized views. This new feature was added to the bugfix release by a special request from Altinity.
 [#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) [(urykhy)](https://github.com/urykhy)
 
 ### ClickHouse release v20.1.2.4, 2020-01-22
@@ -641,4 +765,4 @@
 #### Security Fix
 * Fixed the possibility of reading directories structure in tables with `File` table engine. This fixes [#8536](https://github.com/ClickHouse/ClickHouse/issues/8536). [#8537](https://github.com/ClickHouse/ClickHouse/pull/8537) ([alexey-milovidov](https://github.com/alexey-milovidov))
 
-## [Changelog for 2019](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/changelog/2019.md)
+## [Changelog for 2019](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/whats_new/changelog/2019.md)

CMakeLists.txt

@@ -228,7 +228,7 @@ else ()
     set(NOT_UNBUNDLED 1)
 endif ()
 
-if (UNBUNDLED OR NOT (OS_LINUX OR OS_DARWIN) OR ARCH_32)
+if (UNBUNDLED OR NOT (OS_LINUX OR OS_DARWIN))
     # Using system libs can cause a lot of warnings in includes (on macro expansion).
     option (WERROR "Enable -Werror compiler option" OFF)
 else ()
@@ -251,6 +251,8 @@ if (OS_LINUX)
     include(cmake/linux/default_libs.cmake)
 elseif (OS_DARWIN)
     include(cmake/darwin/default_libs.cmake)
+elseif (OS_FREEBSD)
+    include(cmake/freebsd/default_libs.cmake)
 endif ()
 
 ######################################
@@ -316,7 +318,6 @@ include (cmake/find/poco.cmake)
 include (cmake/find/lz4.cmake)
 include (cmake/find/xxhash.cmake)
 include (cmake/find/sparsehash.cmake)
-include (cmake/find/execinfo.cmake)
 include (cmake/find/re2.cmake)
 include (cmake/find/libgsasl.cmake)
 include (cmake/find/rdkafka.cmake)
@@ -342,6 +343,7 @@ include (cmake/find/rapidjson.cmake)
 include (cmake/find/fastops.cmake)
 include (cmake/find/orc.cmake)
 include (cmake/find/avro.cmake)
+include (cmake/find/msgpack.cmake)
 
 find_contrib_lib(cityhash)
 find_contrib_lib(farmhash)
@@ -353,17 +355,11 @@ if (ENABLE_TESTS)
 endif ()
 
 # Need to process before "contrib" dir:
-include (cmake/find/jemalloc.cmake)
 include (cmake/find/mysqlclient.cmake)
 
 # When testing for memory leaks with Valgrind, don't link tcmalloc or jemalloc.
 
-if (USE_JEMALLOC)
-    message (STATUS "Link jemalloc: ${JEMALLOC_LIBRARIES}")
-    set (MALLOC_LIBRARIES ${JEMALLOC_LIBRARIES})
-elseif (SANITIZE)
-    message (STATUS "Will use ${SANITIZE} sanitizer.")
-elseif (OS_LINUX)
+if (OS_LINUX AND NOT ENABLE_JEMALLOC)
     message (WARNING "Non default allocator is disabled. This is not recommended for production Linux builds.")
 endif ()
 
@@ -386,8 +382,13 @@ macro (add_executable target)
     endif()
 endmacro()
 
+set(ConfigIncludePath ${CMAKE_CURRENT_BINARY_DIR}/includes/configs CACHE INTERNAL "Path to generated configuration files.")
+include_directories(${ConfigIncludePath})
+
 add_subdirectory (base)
+add_subdirectory (programs)
+add_subdirectory (src)
+add_subdirectory (tests)
 add_subdirectory (utils)
-add_subdirectory (dbms)
 
 include (cmake/print_include_directories.cmake)

README.md

@@ -8,14 +8,14 @@ ClickHouse is an open-source column-oriented database management system that all
 * [Tutorial](https://clickhouse.tech/docs/en/getting_started/tutorial/) shows how to set up and query small ClickHouse cluster.
 * [Documentation](https://clickhouse.tech/docs/en/) provides more in-depth information.
 * [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format.
-* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/enQtOTUzMjM4ODQwNTc5LWJmMjE3Yjc2YmI1ZDBlZmI4ZTc3OWY3ZTIwYTljYzY4MzBlODM3YzBjZTc1YmYyODRlZTJkYTgzYzBiNTA2Yjk) and [Telegram](https://telegram.me/clickhouse_en) allow to chat with ClickHouse users in real-time.
+* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-d2zxkf9e-XyxDa_ucfPxzuH4SJIm~Ng) and [Telegram](https://telegram.me/clickhouse_en) allow to chat with ClickHouse users in real-time.
 * [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events.
 * [Contacts](https://clickhouse.tech/#contacts) can help to get your questions answered if there are any.
-* You can also [fill this form](https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/) to meet Yandex ClickHouse team in person.
+* You can also [fill this form](https://clickhouse.tech/#meet) to meet Yandex ClickHouse team in person.
 
 ## Upcoming Events
 
+* [ClickHouse Online Meetup West (in English)](https://www.eventbrite.com/e/clickhouse-online-meetup-registration-102886791162) on April 24, 2020.
+* [ClickHouse Online Meetup East (in English)](https://www.eventbrite.com/e/clickhouse-online-meetup-east-registration-102989325846) on April 28, 2020.
 * [ClickHouse Workshop in Novosibirsk](https://2020.codefest.ru/lecture/1628) on TBD date.
-* [Talks on Saint HighLoad++ in St. Petersburg](https://www.highload.ru/spb/2020/abstracts/6647) on TBD date.
 * [Yandex C++ Open-Source Sprints in Moscow](https://events.yandex.ru/events/otkrytyj-kod-v-yandek-28-03-2020) on TBD date.
-* [ClickHouse in Avito (online in Russian)](https://avitotech.timepad.ru/event/1290051/) on April 9, 2020.

base/CMakeLists.txt

@@ -3,8 +3,10 @@ if (USE_CLANG_TIDY)
 endif ()
 
 add_subdirectory (common)
-add_subdirectory (loggers)
 add_subdirectory (daemon)
+add_subdirectory (loggers)
+add_subdirectory (pcg-random)
+add_subdirectory (widechar_width)
 
 if (USE_MYSQL)
     add_subdirectory (mysqlxx)

base/common/DateLUTImpl.cpp

@@ -133,7 +133,10 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_)
     }
 
     /// Fill lookup table for years and months.
-    for (size_t day = 0; day < DATE_LUT_SIZE && lut[day].year <= DATE_LUT_MAX_YEAR; ++day)
+    size_t year_months_lut_index = 0;
+    size_t first_day_of_last_month = 0;
+
+    for (size_t day = 0; day < DATE_LUT_SIZE; ++day)
     {
         const Values & values = lut[day];
 
@@ -141,7 +144,16 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_)
         {
             if (values.month == 1)
                 years_lut[values.year - DATE_LUT_MIN_YEAR] = day;
-            years_months_lut[(values.year - DATE_LUT_MIN_YEAR) * 12 + values.month - 1] = day;
+
+            year_months_lut_index = (values.year - DATE_LUT_MIN_YEAR) * 12 + values.month - 1;
+            years_months_lut[year_months_lut_index] = day;
+            first_day_of_last_month = day;
         }
     }
+
+    /// Fill the rest of lookup table with the same last month (2106-02-01).
+    for (; year_months_lut_index < DATE_LUT_YEARS * 12; ++year_months_lut_index)
+    {
+        years_months_lut[year_months_lut_index] = first_day_of_last_month;
+    }
 }

base/common/DateLUTImpl.h

@@ -12,7 +12,7 @@
 /// Table size is bigger than DATE_LUT_MAX_DAY_NUM to fill all indices within UInt16 range: this allows to remove extra check.
 #define DATE_LUT_SIZE 0x10000
 #define DATE_LUT_MIN_YEAR 1970
-#define DATE_LUT_MAX_YEAR 2105 /// Last supported year
+#define DATE_LUT_MAX_YEAR 2106 /// Last supported year (incomplete)
 #define DATE_LUT_YEARS (1 + DATE_LUT_MAX_YEAR - DATE_LUT_MIN_YEAR) /// Number of years in lookup table
 
 #if defined(__PPC__)

base/common/config_common.h.in

@@ -2,5 +2,4 @@
 
 // .h autogenerated by cmake !
 
-#cmakedefine01 USE_JEMALLOC
 #cmakedefine01 UNBUNDLED

base/common/time.h

@@ -4,4 +4,6 @@
 
 #if defined (OS_DARWIN)
 #    define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC
+#elif defined (OS_FREEBSD)
+#    define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC_FAST
 #endif

base/common/types.h

@@ -11,6 +11,10 @@ using Int16 = int16_t;
 using Int32 = int32_t;
 using Int64 = int64_t;
 
+#if __cplusplus <= 201703L
+using char8_t = unsigned char;
+#endif
+
 using UInt8 = char8_t;
 using UInt16 = uint16_t;
 using UInt32 = uint32_t;

base/common/ya.make

@@ -1,12 +1,47 @@
 LIBRARY()
 
+ADDINCL(
+    GLOBAL clickhouse/base
+    contrib/libs/cctz/include
+)
+
+CFLAGS (GLOBAL -DARCADIA_BUILD)
+
+IF (OS_DARWIN)
+    CFLAGS (GLOBAL -DOS_DARWIN)
+ELSEIF (OS_FREEBSD)
+    CFLAGS (GLOBAL -DOS_FREEBSD)
+ELSEIF (OS_LINUX)
+    CFLAGS (GLOBAL -DOS_LINUX)
+ENDIF ()
+
 PEERDIR(
+    contrib/libs/cctz/src
+    contrib/libs/cxxsupp/libcxx-filesystem
+    contrib/libs/poco/Net
     contrib/libs/poco/Util
+    contrib/restricted/boost
+    contrib/restricted/cityhash-1.0.2
 )
 
 SRCS(
     argsToConfig.cpp
     coverage.cpp
+    DateLUT.cpp
+    DateLUTImpl.cpp
+    demangle.cpp
+    getFQDNOrHostName.cpp
+    getMemoryAmount.cpp
+    getThreadId.cpp
+    JSON.cpp
+    LineReader.cpp
+    mremap.cpp
+    phdr_cache.cpp
+    preciseExp10.c
+    setTerminalEcho.cpp
+    shift10.cpp
+    sleep.cpp
+    terminalColors.cpp
 )
 
 END()

base/daemon/BaseDaemon.cpp

@@ -12,7 +12,6 @@
 #include <unistd.h>
 
 #include <typeinfo>
-#include <sys/time.h>
 #include <sys/resource.h>
 #include <iostream>
 #include <fstream>
@@ -51,11 +50,13 @@
 #include <Common/getMultipleKeysFromConfig.h>
 #include <Common/ClickHouseRevision.h>
 #include <Common/Config/ConfigProcessor.h>
-#include <Common/config_version.h>
 
-#ifdef __APPLE__
-// ucontext is not available without _XOPEN_SOURCE
-#define _XOPEN_SOURCE 700
+#if !defined(ARCADIA_BUILD)
+#    include <Common/config_version.h>
+#endif
+
+#if defined(OS_DARWIN)
+#    define _XOPEN_SOURCE 700  // ucontext is not available without _XOPEN_SOURCE
 #endif
 #include <ucontext.h>
 
@@ -411,7 +412,7 @@ std::string BaseDaemon::getDefaultCorePath() const
 
 void BaseDaemon::closeFDs()
 {
-#if defined(__FreeBSD__) || (defined(__APPLE__) && defined(__MACH__))
+#if defined(OS_FREEBSD) || defined(OS_DARWIN)
     Poco::File proc_path{"/dev/fd"};
 #else
     Poco::File proc_path{"/proc/self/fd"};
@@ -431,7 +432,7 @@ void BaseDaemon::closeFDs()
     else
     {
         int max_fd = -1;
-#ifdef _SC_OPEN_MAX
+#if defined(_SC_OPEN_MAX)
         max_fd = sysconf(_SC_OPEN_MAX);
         if (max_fd == -1)
 #endif
@@ -449,7 +450,7 @@ namespace
 /// the maximum is 1000, and chromium uses 300 for its tab processes. Ignore
 /// whatever errors that occur, because it's just a debugging aid and we don't
 /// care if it breaks.
-#if defined(__linux__) && !defined(NDEBUG)
+#if defined(OS_LINUX) && !defined(NDEBUG)
 void debugIncreaseOOMScore()
 {
     const std::string new_score = "555";

base/daemon/ya.make

@@ -0,0 +1,14 @@
+LIBRARY()
+
+NO_COMPILER_WARNINGS()
+
+PEERDIR(
+    clickhouse/src/Common
+)
+
+SRCS(
+    BaseDaemon.cpp
+    GraphiteWriter.cpp
+)
+
+END()

base/loggers/OwnPatternFormatter.cpp

@@ -75,7 +75,11 @@ void OwnPatternFormatter::formatExtended(const DB::ExtendedLogMessage & msg_ext,
     if (color)
         writeCString(resetColor(), wb);
     writeCString("> ", wb);
+    if (color)
+        writeString(setColor(std::hash<std::string>()(msg.getSource())), wb);
     DB::writeString(msg.getSource(), wb);
+    if (color)
+        writeCString(resetColor(), wb);
     writeCString(": ", wb);
     DB::writeString(msg.getText(), wb);
 }

base/loggers/ya.make

@@ -0,0 +1,15 @@
+LIBRARY()
+
+PEERDIR(
+    clickhouse/src/Common
+)
+
+SRCS(
+    ExtendedLogChannel.cpp
+    Loggers.cpp
+    OwnFormattingChannel.cpp
+    OwnPatternFormatter.cpp
+    OwnSplitChannel.cpp
+)
+
+END()

base/pcg-random/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_library(pcg_random INTERFACE)
+target_include_directories(pcg_random INTERFACE .)

base/pcg-random/pcg_extras.hpp

@@ -292,7 +292,7 @@ inline itype rotl(itype value, bitcount_t rot)
 {
     constexpr bitcount_t bits = sizeof(itype) * 8;
     constexpr bitcount_t mask = bits - 1;
-#if PCG_USE_ZEROCHECK_ROTATE_IDIOM
+#if defined(PCG_USE_ZEROCHECK_ROTATE_IDIOM)
     return rot ? (value << rot) | (value >> (bits - rot)) : value;
 #else
     return (value << rot) | (value >> ((- rot) & mask));
@@ -304,7 +304,7 @@ inline itype rotr(itype value, bitcount_t rot)
 {
     constexpr bitcount_t bits = sizeof(itype) * 8;
     constexpr bitcount_t mask = bits - 1;
-#if PCG_USE_ZEROCHECK_ROTATE_IDIOM
+#if defined(PCG_USE_ZEROCHECK_ROTATE_IDIOM)
     return rot ? (value >> rot) | (value << (bits - rot)) : value;
 #else
     return (value >> rot) | (value << ((- rot) & mask));
@@ -318,7 +318,7 @@ inline itype rotr(itype value, bitcount_t rot)
  *
  * These overloads will be preferred over the general template code above.
  */
-#if PCG_USE_INLINE_ASM && __GNUC__ && (__x86_64__  || __i386__)
+#if defined(PCG_USE_INLINE_ASM) && __GNUC__ && (__x86_64__  || __i386__)
 
 inline uint8_t rotr(uint8_t value, bitcount_t rot)
 {
@@ -600,7 +600,7 @@ std::ostream& operator<<(std::ostream& out, printable_typename<T>) {
 #ifdef __GNUC__
     int status;
     char* pretty_name =
-        abi::__cxa_demangle(implementation_typename, NULL, NULL, &status);
+        abi::__cxa_demangle(implementation_typename, nullptr, nullptr, &status);
     if (status == 0)
         out << pretty_name;
     free(static_cast<void*>(pretty_name));

base/pcg-random/ya.make

@@ -0,0 +1,5 @@
+LIBRARY()
+
+ADDINCL (GLOBAL clickhouse/base/pcg-random)
+
+END()

base/widechar_width/ya.make

@@ -0,0 +1,9 @@
+LIBRARY()
+
+ADDINCL(GLOBAL clickhouse/base/widechar_width)
+
+SRCS(
+    widechar_width.cpp
+)
+
+END()

base/ya.make

@@ -1,3 +1,7 @@
 RECURSE(
     common
+    daemon
+    loggers
+    pcg-random
+    widechar_width
 )

cmake/Modules/FindJeMalloc.cmake

@@ -1,46 +0,0 @@
-# https://github.com/bro/cmake/blob/master/FindJeMalloc.cmake
-#
-# - Try to find jemalloc headers and libraries.
-#
-# Usage of this module as follows:
-#
-#     find_package(JeMalloc)
-#
-# Variables used by this module, they can change the default behaviour and need
-# to be set before calling find_package:
-#
-#  JEMALLOC_ROOT_DIR Set this variable to the root installation of
-#                    jemalloc if the module has problems finding
-#                    the proper installation path.
-#
-# Variables defined by this module:
-#
-#  JEMALLOC_FOUND             System has jemalloc libs/headers
-#  JEMALLOC_LIBRARIES         The jemalloc library/libraries
-#  JEMALLOC_INCLUDE_DIR       The location of jemalloc headers
-
-find_path(JEMALLOC_ROOT_DIR
-    NAMES include/jemalloc/jemalloc.h
-)
-
-find_library(JEMALLOC_LIBRARIES
-    NAMES jemalloc
-    HINTS ${JEMALLOC_ROOT_DIR}/lib
-)
-
-find_path(JEMALLOC_INCLUDE_DIR
-    NAMES jemalloc/jemalloc.h
-    HINTS ${JEMALLOC_ROOT_DIR}/include
-)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(JeMalloc DEFAULT_MSG
-    JEMALLOC_LIBRARIES
-    JEMALLOC_INCLUDE_DIR
-)
-
-mark_as_advanced(
-    JEMALLOC_ROOT_DIR
-    JEMALLOC_LIBRARIES
-    JEMALLOC_INCLUDE_DIR
-)

cmake/analysis.cmake

@@ -10,7 +10,7 @@ if (ENABLE_CLANG_TIDY)
     if (CLANG_TIDY_PATH)
         message(STATUS "Using clang-tidy: ${CLANG_TIDY_PATH}. The checks will be run during build process. See the .clang-tidy file at the root directory to configure the checks.")
         set (USE_CLANG_TIDY 1)
-        # The variable CMAKE_CXX_CLANG_TIDY will be set inside dbms and base directories with non third-party code.
+        # The variable CMAKE_CXX_CLANG_TIDY will be set inside src and base directories with non third-party code.
         # set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}")
     else ()
         message(STATUS "clang-tidy is not found. This is normal - the tool is used only for static code analysis and not essential for build.")

cmake/arch.cmake

@@ -11,7 +11,6 @@ if (CMAKE_LIBRARY_ARCHITECTURE MATCHES "i386")
     set (ARCH_I386 1)
 endif ()
 if ((ARCH_ARM AND NOT ARCH_AARCH64) OR ARCH_I386)
-    set (ARCH_32 1)
     message (FATAL_ERROR "32bit platforms are not supported")
 endif ()
 

cmake/find/execinfo.cmake

@@ -1,8 +0,0 @@
-if (OS_FREEBSD)
-    find_library (EXECINFO_LIBRARY execinfo)
-    find_library (ELF_LIBRARY elf)
-    set (EXECINFO_LIBRARIES ${EXECINFO_LIBRARY} ${ELF_LIBRARY})
-    message (STATUS "Using execinfo: ${EXECINFO_LIBRARIES}")
-else ()
-    set (EXECINFO_LIBRARIES "")
-endif ()

cmake/find/jemalloc.cmake

@@ -1,42 +0,0 @@
-if (OS_LINUX AND NOT SANITIZE AND NOT ARCH_32 AND NOT ARCH_PPC64LE)
-    set(ENABLE_JEMALLOC_DEFAULT ${ENABLE_LIBRARIES})
-else ()
-    set(ENABLE_JEMALLOC_DEFAULT 0)
-endif ()
-
-option (ENABLE_JEMALLOC "Set to TRUE to use jemalloc" ${ENABLE_JEMALLOC_DEFAULT})
-if (OS_LINUX)
-    option (USE_INTERNAL_JEMALLOC_LIBRARY "Set to FALSE to use system jemalloc library instead of bundled" ${NOT_UNBUNDLED})
-else()
-    option (USE_INTERNAL_JEMALLOC_LIBRARY "Set to FALSE to use system jemalloc library instead of bundled" OFF)
-endif()
-
-if (ENABLE_JEMALLOC)
-    if (USE_INTERNAL_JEMALLOC_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/jemalloc/src/jemalloc.c")
-       message (WARNING "submodule contrib/jemalloc is missing. to fix try run: \n git submodule update --init --recursive")
-       set (USE_INTERNAL_JEMALLOC_LIBRARY 0)
-       set (MISSING_INTERNAL_JEMALLOC_LIBRARY 1)
-    endif ()
-
-    if (NOT USE_INTERNAL_JEMALLOC_LIBRARY)
-        find_package (JeMalloc)
-    endif ()
-
-    if ((NOT JEMALLOC_LIBRARIES OR NOT JEMALLOC_INCLUDE_DIR) AND NOT MISSING_INTERNAL_JEMALLOC_LIBRARY )
-        set (JEMALLOC_LIBRARIES "jemalloc")
-        set (JEMALLOC_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/jemalloc-cmake/include" "${ClickHouse_SOURCE_DIR}/contrib/jemalloc-cmake/include_linux_x86_64")
-        set (USE_INTERNAL_JEMALLOC_LIBRARY 1)
-    endif ()
-
-    if (JEMALLOC_LIBRARIES)
-        set (USE_JEMALLOC 1)
-    elseif (NOT MISSING_INTERNAL_JEMALLOC_LIBRARY)
-        message (FATAL_ERROR "ENABLE_JEMALLOC is set to true, but library was not found")
-    endif ()
-
-    if (SANITIZE)
-        message (FATAL_ERROR "ENABLE_JEMALLOC is set to true, but it cannot be used with sanitizers")
-    endif ()
-
-    message (STATUS "Using jemalloc=${USE_JEMALLOC}: ${JEMALLOC_INCLUDE_DIR} : ${JEMALLOC_LIBRARIES}")
-endif ()