Merge branch 'master' into database_atomic

.github/ISSUE_TEMPLATE/backward-compatibility.md

@@ -17,7 +17,7 @@ A clear and concise description of what works not as it is supposed to.
 * Which interface to use, if matters
 * Non-default settings, if any
 * `CREATE TABLE` statements for all tables involved
-* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
+* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
 * Queries to run that lead to unexpected result
 
 **Error message and/or stacktrace**

.github/ISSUE_TEMPLATE/bug-report.md

@@ -17,7 +17,7 @@ A clear and concise description of what works not as it is supposed to.
 * Which interface to use, if matters
 * Non-default settings, if any
 * `CREATE TABLE` statements for all tables involved
-* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
+* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
 * Queries to run that lead to unexpected result
 
 **Expected behavior**

.github/ISSUE_TEMPLATE/performance-issue.md

@@ -17,7 +17,7 @@ What exactly works slower than expected?
 * Which interface to use, if matters
 * Non-default settings, if any
 * `CREATE TABLE` statements for all tables involved
-* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
+* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
 * Queries to run that lead to slow performance
 
 **Expected performance**

.github/ISSUE_TEMPLATE/unexpected-behaviour.md

@@ -17,7 +17,7 @@ A clear and concise description of what works not as it is supposed to.
 * Which interface to use, if matters
 * Non-default settings, if any
 * `CREATE TABLE` statements for all tables involved
-* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
+* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
 * Queries to run that lead to unexpected result
 
 **Expected behavior**

.github/ISSUE_TEMPLATE/usability-issue.md

@@ -17,7 +17,7 @@ A clear and concise description of what works not as it is supposed to.
 * Which interface to use, if matters
 * Non-default settings, if any
 * `CREATE TABLE` statements for all tables involved
-* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/dbms/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
+* Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary
 * Queries to run that lead to unexpected result
 
 **Expected behavior**

.gitignore

@@ -73,100 +73,100 @@ contrib/libpoco/Poco/
 contrib/libpoco/bin/
 contrib/libpoco/cmake_uninstall.cmake
 contrib/libre2/re2_st/
-dbms/src/Client/clickhouse-benchmark
+dbms/Client/clickhouse-benchmark
-dbms/src/Client/clickhouse-client
+dbms/Client/clickhouse-client
-dbms/src/Client/tests/test-connect
+dbms/Client/tests/test-connect
-dbms/src/Common/tests/arena_with_free_lists
+dbms/Common/tests/arena_with_free_lists
-dbms/src/Common/tests/auto_array
+dbms/Common/tests/auto_array
-dbms/src/Common/tests/compact_array
+dbms/Common/tests/compact_array
-dbms/src/Common/tests/hash_table
+dbms/Common/tests/hash_table
-dbms/src/Common/tests/hashes_test
+dbms/Common/tests/hashes_test
-dbms/src/Common/tests/int_hashes_perf
+dbms/Common/tests/int_hashes_perf
-dbms/src/Common/tests/lru_cache
+dbms/Common/tests/lru_cache
-dbms/src/Common/tests/parallel_aggregation
+dbms/Common/tests/parallel_aggregation
-dbms/src/Common/tests/parallel_aggregation2
+dbms/Common/tests/parallel_aggregation2
-dbms/src/Common/tests/radix_sort
+dbms/Common/tests/radix_sort
-dbms/src/Common/tests/shell_command_test
+dbms/Common/tests/shell_command_test
-dbms/src/Common/tests/simple_cache
+dbms/Common/tests/simple_cache
-dbms/src/Common/tests/sip_hash
+dbms/Common/tests/sip_hash
-dbms/src/Common/tests/sip_hash_perf
+dbms/Common/tests/sip_hash_perf
-dbms/src/Common/tests/small_table
+dbms/Common/tests/small_table
-dbms/src/Core/tests/exception
+dbms/Core/tests/exception
-dbms/src/Core/tests/field
+dbms/Core/tests/field
-dbms/src/Core/tests/rvo_test
+dbms/Core/tests/rvo_test
-dbms/src/Core/tests/string_pool
+dbms/Core/tests/string_pool
-dbms/src/DataStreams/tests/aggregating_stream
+dbms/DataStreams/tests/aggregating_stream
-dbms/src/DataStreams/tests/block_tab_separated_streams
+dbms/DataStreams/tests/block_tab_separated_streams
-dbms/src/DataStreams/tests/collapsing_sorted_stream
+dbms/DataStreams/tests/collapsing_sorted_stream
-dbms/src/DataStreams/tests/expression_stream
+dbms/DataStreams/tests/expression_stream
-dbms/src/DataStreams/tests/filter_stream
+dbms/DataStreams/tests/filter_stream
-dbms/src/DataStreams/tests/filter_stream_hitlog
+dbms/DataStreams/tests/filter_stream_hitlog
-dbms/src/DataStreams/tests/fork_streams
+dbms/DataStreams/tests/fork_streams
-dbms/src/DataStreams/tests/glue_streams
+dbms/DataStreams/tests/glue_streams
-dbms/src/DataStreams/tests/json_streams
+dbms/DataStreams/tests/json_streams
-dbms/src/DataStreams/tests/native_streams
+dbms/DataStreams/tests/native_streams
-dbms/src/DataStreams/tests/sorting_stream
+dbms/DataStreams/tests/sorting_stream
-dbms/src/DataStreams/tests/tab_separated_streams
+dbms/DataStreams/tests/tab_separated_streams
-dbms/src/DataStreams/tests/union_stream
+dbms/DataStreams/tests/union_stream
-dbms/src/DataStreams/tests/union_stream2
+dbms/DataStreams/tests/union_stream2
-dbms/src/DataTypes/tests/data_type_string
+dbms/DataTypes/tests/data_type_string
-dbms/src/DataTypes/tests/data_types_number_fixed
+dbms/DataTypes/tests/data_types_number_fixed
-dbms/src/Functions/tests/functions_arithmetic
+dbms/Functions/tests/functions_arithmetic
-dbms/src/Functions/tests/logical_functions_performance
+dbms/Functions/tests/logical_functions_performance
-dbms/src/Functions/tests/number_traits
+dbms/Functions/tests/number_traits
-dbms/src/IO/tests/async_write
+dbms/IO/tests/async_write
-dbms/src/IO/tests/cached_compressed_read_buffer
+dbms/IO/tests/cached_compressed_read_buffer
-dbms/src/IO/tests/compressed_buffer
+dbms/IO/tests/compressed_buffer
-dbms/src/IO/tests/hashing_read_buffer
+dbms/IO/tests/hashing_read_buffer
-dbms/src/IO/tests/hashing_write_buffer
+dbms/IO/tests/hashing_write_buffer
-dbms/src/IO/tests/io_and_exceptions
+dbms/IO/tests/io_and_exceptions
-dbms/src/IO/tests/io_operators
+dbms/IO/tests/io_operators
-dbms/src/IO/tests/mempbrk
+dbms/IO/tests/mempbrk
-dbms/src/IO/tests/o_direct_and_dirty_pages
+dbms/IO/tests/o_direct_and_dirty_pages
-dbms/src/IO/tests/parse_int_perf
+dbms/IO/tests/parse_int_perf
-dbms/src/IO/tests/parse_int_perf2
+dbms/IO/tests/parse_int_perf2
-dbms/src/IO/tests/read_buffer
+dbms/IO/tests/read_buffer
-dbms/src/IO/tests/read_buffer_aio
+dbms/IO/tests/read_buffer_aio
-dbms/src/IO/tests/read_buffer_perf
+dbms/IO/tests/read_buffer_perf
-dbms/src/IO/tests/read_escaped_string
+dbms/IO/tests/read_escaped_string
-dbms/src/IO/tests/read_float_perf
+dbms/IO/tests/read_float_perf
-dbms/src/IO/tests/read_write_int
+dbms/IO/tests/read_write_int
-dbms/src/IO/tests/valid_utf8
+dbms/IO/tests/valid_utf8
-dbms/src/IO/tests/valid_utf8_perf
+dbms/IO/tests/valid_utf8_perf
-dbms/src/IO/tests/var_uint
+dbms/IO/tests/var_uint
-dbms/src/IO/tests/write_buffer
+dbms/IO/tests/write_buffer
-dbms/src/IO/tests/write_buffer_aio
+dbms/IO/tests/write_buffer_aio
-dbms/src/IO/tests/write_buffer_perf
+dbms/IO/tests/write_buffer_perf
-dbms/src/Interpreters/tests/address_patterns
+dbms/Interpreters/tests/address_patterns
-dbms/src/Interpreters/tests/aggregate
+dbms/Interpreters/tests/aggregate
-dbms/src/Interpreters/tests/compiler_test
+dbms/Interpreters/tests/compiler_test
-dbms/src/Interpreters/tests/create_query
+dbms/Interpreters/tests/create_query
-dbms/src/Interpreters/tests/expression
+dbms/Interpreters/tests/expression
-dbms/src/Interpreters/tests/expression_analyzer
+dbms/Interpreters/tests/expression_analyzer
-dbms/src/Interpreters/tests/hash_map
+dbms/Interpreters/tests/hash_map
-dbms/src/Interpreters/tests/hash_map2
+dbms/Interpreters/tests/hash_map2
-dbms/src/Interpreters/tests/hash_map3
+dbms/Interpreters/tests/hash_map3
-dbms/src/Interpreters/tests/hash_map_string
+dbms/Interpreters/tests/hash_map_string
-dbms/src/Interpreters/tests/hash_map_string_2
+dbms/Interpreters/tests/hash_map_string_2
-dbms/src/Interpreters/tests/hash_map_string_3
+dbms/Interpreters/tests/hash_map_string_3
-dbms/src/Interpreters/tests/hash_map_string_small
+dbms/Interpreters/tests/hash_map_string_small
-dbms/src/Interpreters/tests/in_join_subqueries_preprocessor
+dbms/Interpreters/tests/in_join_subqueries_preprocessor
-dbms/src/Interpreters/tests/logical_expressions_optimizer
+dbms/Interpreters/tests/logical_expressions_optimizer
-dbms/src/Interpreters/tests/select_query
+dbms/Interpreters/tests/select_query
-dbms/src/Interpreters/tests/two_level_hash_map
+dbms/Interpreters/tests/two_level_hash_map
-dbms/src/Interpreters/tests/users
+dbms/Interpreters/tests/users
-dbms/src/Parsers/tests/create_parser
+dbms/Parsers/tests/create_parser
-dbms/src/Parsers/tests/select_parser
+dbms/Parsers/tests/select_parser
-dbms/src/Server/clickhouse-server
+dbms/Server/clickhouse-server
-dbms/src/Server/clickhouse-server.init
+dbms/Server/clickhouse-server.init
-dbms/src/Storages/tests/hit_log
+dbms/Storages/tests/hit_log
-dbms/src/Storages/tests/merge_tree
+dbms/Storages/tests/merge_tree
-dbms/src/Storages/tests/part_checker
+dbms/Storages/tests/part_checker
-dbms/src/Storages/tests/part_name
+dbms/Storages/tests/part_name
-dbms/src/Storages/tests/pk_condition
+dbms/Storages/tests/pk_condition
-dbms/src/Storages/tests/seek_speed_test
+dbms/Storages/tests/seek_speed_test
-dbms/src/Storages/tests/storage_log
+dbms/Storages/tests/storage_log
-dbms/src/Storages/tests/system_numbers
+dbms/Storages/tests/system_numbers
 libs/libcommon/src/revision.h
 libs/libcommon/src/tests/date_lut2
 libs/libcommon/src/tests/date_lut3
@@ -184,15 +184,15 @@ libs/libzkutil/src/tests/zkutil_zookeeper_holder
 utils/zookeeper-create-entry-to-download-part/zookeeper-create-entry-to-download-part
 utils/zookeeper-dump-tree/zookeeper-dump-tree
 utils/zookeeper-remove-by-list/zookeeper-remove-by-list
-dbms/src/Storages/tests/remove_symlink_directory
+dbms/Storages/tests/remove_symlink_directory
 libs/libcommon/src/tests/json_test
 utils/compressor/zstd_test
 utils/wikistat-loader/wikistat-loader
-dbms/src/Common/tests/pod_array
+dbms/Common/tests/pod_array
 
-dbms/src/Server/data/*
+dbms/Server/data/*
-dbms/src/Server/metadata/*
+dbms/Server/metadata/*
-dbms/src/Server/status
+dbms/Server/status
 config-9001.xml
 
 *-preprocessed.xml
@@ -242,7 +242,7 @@ website/package-lock.json
 */.DS_Store
 
 # Ignore files for locally disabled tests
-/dbms/tests/queries/**/*.disabled
+/dbms/queries/**/*.disabled
 
 # cquery cache
 /.cquery-cache

.gitlab-ci.yml

@@ -31,11 +31,11 @@ build:
     - docker pull $CI_REGISTRY/yandex/clickhouse-builder
     - docker run --rm --volumes-from "${HOSTNAME}-build" --workdir "${CI_PROJECT_DIR}" --env CI_PROJECT_DIR=${CI_PROJECT_DIR} $CI_REGISTRY/yandex/clickhouse-builder /build_gitlab_ci.sh
   # You can upload your binary to nexus
-    - curl -v --keepalive-time 60 --keepalive --user "$NEXUS_USER:$NEXUS_PASSWORD" -XPUT "http://$NEXUS_HOST/repository/binaries/$CI_PROJECT_NAME" --upload-file ./dbms/src/Server/clickhouse
+    - curl -v --keepalive-time 60 --keepalive --user "$NEXUS_USER:$NEXUS_PASSWORD" -XPUT "http://$NEXUS_HOST/repository/binaries/$CI_PROJECT_NAME" --upload-file ./dbms/Server/clickhouse
   # Or download artifacts from gitlab
   artifacts:
     paths:
-      - ./dbms/src/Server/clickhouse
+      - ./dbms/Server/clickhouse
     expire_in: 1 day
   tags:
-    - docker
+    - docker

CHANGELOG.md

@@ -234,7 +234,7 @@
 * Updated checking for hung queries in clickhouse-test script [#8858](https://github.com/ClickHouse/ClickHouse/pull/8858) ([Alexander Kazakov](https://github.com/Akazz))
 * Removed some useless files from repository. [#8843](https://github.com/ClickHouse/ClickHouse/pull/8843) ([alexey-milovidov](https://github.com/alexey-milovidov))
 * Changed type of math perftests from `once` to `loop`. [#8783](https://github.com/ClickHouse/ClickHouse/pull/8783) ([Nikolai Kochetov](https://github.com/KochetovNicolai))
-* Add docker image which allows to build interactive code browser HTML report for our codebase. [#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([alesapin](https://github.com/alesapin)) See [Woboq Code Browser](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/src/index.html)
+* Add docker image which allows to build interactive code browser HTML report for our codebase. [#8781](https://github.com/ClickHouse/ClickHouse/pull/8781) ([alesapin](https://github.com/alesapin)) See [Woboq Code Browser](https://clickhouse-test-reports.s3.yandex.net/codebrowser/html_report///ClickHouse/dbms/index.html)
 * Suppress some test failures under MSan. [#8780](https://github.com/ClickHouse/ClickHouse/pull/8780) ([Alexander Kuzmenkov](https://github.com/akuzm))
 * Speedup "exception while insert" test. This test often time out in debug-with-coverage build. [#8711](https://github.com/ClickHouse/ClickHouse/pull/8711) ([alexey-milovidov](https://github.com/alexey-milovidov))
 * Updated `libcxx` and `libcxxabi` to master. In preparation to [#9304](https://github.com/ClickHouse/ClickHouse/issues/9304) [#9308](https://github.com/ClickHouse/ClickHouse/pull/9308) ([alexey-milovidov](https://github.com/alexey-milovidov))
@@ -341,7 +341,7 @@
 [#9513](https://github.com/ClickHouse/ClickHouse/pull/9513) [(filimonov)](https://github.com/filimonov)
 
 #### New Feature
-* Add `deduplicate_blocks_in_dependent_materialized_views` option to control the behaviour of idempotent inserts into tables with materialized views. This new feature was added to the bugfix release by a special request from Altinity. 
+* Add `deduplicate_blocks_in_dependent_materialized_views` option to control the behaviour of idempotent inserts into tables with materialized views. This new feature was added to the bugfix release by a special request from Altinity.
 [#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) [(urykhy)](https://github.com/urykhy)
 
 ### ClickHouse release v20.1.2.4, 2020-01-22

CMakeLists.txt

@@ -380,8 +380,13 @@ macro (add_executable target)
     endif()
 endmacro()
 
+set(ConfigIncludePath ${CMAKE_CURRENT_BINARY_DIR}/includes/configs CACHE INTERNAL "Path to generated configuration files.")
+include_directories(${ConfigIncludePath})
+
 add_subdirectory (base)
-add_subdirectory (utils)
 add_subdirectory (dbms)
+add_subdirectory (programs)
+add_subdirectory (tests)
+add_subdirectory (utils)
 
 include (cmake/print_include_directories.cmake)

cmake/lib_name.cmake

@@ -1,5 +1,5 @@
 set(DIVIDE_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libdivide)
-set(DBMS_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/dbms/src ${ClickHouse_BINARY_DIR}/dbms/src)
+set(DBMS_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/dbms ${ClickHouse_BINARY_DIR}/dbms)
 set(DOUBLE_CONVERSION_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/double-conversion)
 set(METROHASH_CONTRIB_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src)
 set(PCG_RANDOM_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libpcg-random/include)

cmake/sanitize.cmake

@@ -23,7 +23,7 @@ if (SANITIZE)
         # RelWithDebInfo, and downgrade optimizations to -O1 but not to -Og, to
         # keep the binary size down.
         # TODO: try compiling with -Og and with ld.gold.
-        set (MSAN_FLAGS "-fsanitize=memory -fsanitize-memory-track-origins -fno-optimize-sibling-calls -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/dbms/tests/msan_suppressions.txt")
+        set (MSAN_FLAGS "-fsanitize=memory -fsanitize-memory-track-origins -fno-optimize-sibling-calls -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/msan_suppressions.txt")
 
         set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${MSAN_FLAGS}")
         set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${MSAN_FLAGS}")

contrib/libcpuid/include/libcpuid/cpuid_main.c

@@ -38,7 +38,7 @@
 
 /* Implementation: */
 
-static int _libcpiud_errno = ERR_OK;
+_Thread_local int _libcpiud_errno = ERR_OK;
 
 int set_error(cpu_error_t err)
 {