diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d3fac8670e8..a2930beb89f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -3,7 +3,7 @@ I hereby agree to the terms of the CLA available at: https://yandex.ru/legal/cla Changelog category (leave one): - New Feature - Improvement -- Bug Fix +- Bug Fix (user-visible misbehaviour in official stable or prestable release) - Performance Improvement - Backward Incompatible Change - Build/Testing/Packaging Improvement diff --git a/CHANGELOG.md b/CHANGELOG.md index 103d8e40fd9..71cdac17825 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ### ClickHouse release v21.8, 2021-08-12 +#### Upgrade Notes +* New version is using `Map` data type for system logs tables (`system.query_log`, `system.query_thread_log`, `system.processes`, `system.opentelemetry_span_log`). These tables will be auto-created with new data types. Virtual columns are created to support old queries. Closes [#18698](https://github.com/ClickHouse/ClickHouse/issues/18698). [#23934](https://github.com/ClickHouse/ClickHouse/pull/23934), [#25773](https://github.com/ClickHouse/ClickHouse/pull/25773) ([hexiaoting](https://github.com/hexiaoting), [sundy-li](https://github.com/sundy-li), [Maksim Kita](https://github.com/kitaisreal)). If you want to *downgrade* from version 21.8 to older versions, you will need to cleanup system tables with logs manually. Look at `/var/lib/clickhouse/data/system/*_log`. + #### New Features * Add support for a part of SQL/JSON standard. [#24148](https://github.com/ClickHouse/ClickHouse/pull/24148) ([l1tsolaiki](https://github.com/l1tsolaiki), [Kseniia Sumarokova](https://github.com/kssenii)). diff --git a/CMakeLists.txt b/CMakeLists.txt index 35c22526816..1aef8c9fc8d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -593,7 +593,23 @@ macro (add_executable target) # disabled for TSAN and gcc since libtsan.a provides overrides too if (TARGET clickhouse_new_delete) # operator::new/delete for executables (MemoryTracker stuff) - target_link_libraries (${target} PRIVATE clickhouse_new_delete ${MALLOC_LIBRARIES}) + target_link_libraries (${target} PRIVATE clickhouse_new_delete) + endif() + + # In case of static jemalloc, because zone_register() is located in zone.c and + # is never used outside (it is declared as constructor) it is omitted + # by the linker, and so jemalloc will not be registered as system + # allocator under osx [1], and clickhouse will SIGSEGV. + # + # [1]: https://github.com/jemalloc/jemalloc/issues/708 + # + # About symbol name: + # - _zone_register not zone_register due to Mach-O binary format, + # - _je_zone_register due to JEMALLOC_PRIVATE_NAMESPACE=je_ under OS X. + # - but jemalloc-cmake does not run private_namespace.sh + # so symbol name should be _zone_register + if (ENABLE_JEMALLOC AND MAKE_STATIC_LIBRARIES AND OS_DARWIN) + set_property(TARGET ${target} APPEND PROPERTY LINK_OPTIONS -u_zone_register) endif() endif() endmacro() diff --git a/base/common/wide_integer_impl.h b/base/common/wide_integer_impl.h index d2ef8b22d65..27efe18eedd 100644 --- a/base/common/wide_integer_impl.h +++ b/base/common/wide_integer_impl.h @@ -152,7 +152,7 @@ namespace wide template struct integer::_impl { - static constexpr size_t _Bits = Bits; + static constexpr size_t _bits = Bits; static constexpr const unsigned byte_count = Bits / 8; static constexpr const unsigned item_count = byte_count / sizeof(base_type); static constexpr const unsigned base_bits = sizeof(base_type) * 8; @@ -614,8 +614,8 @@ public: else { static_assert(IsWideInteger::value); - return std::common_type_t, integer>::_impl::operator_plus( - integer(lhs), rhs); + return std::common_type_t, integer>::_impl::operator_plus( + integer(lhs), rhs); } } @@ -632,8 +632,8 @@ public: else { static_assert(IsWideInteger::value); - return std::common_type_t, integer>::_impl::operator_minus( - integer(lhs), rhs); + return std::common_type_t, integer>::_impl::operator_minus( + integer(lhs), rhs); } } @@ -857,7 +857,7 @@ public: else { static_assert(IsWideInteger::value); - return std::common_type_t, integer>::operator_slash(T(lhs), rhs); + return std::common_type_t, integer>::operator_slash(T(lhs), rhs); } } @@ -877,7 +877,7 @@ public: else { static_assert(IsWideInteger::value); - return std::common_type_t, integer>::operator_percent(T(lhs), rhs); + return std::common_type_t, integer>::operator_percent(T(lhs), rhs); } } diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 3571c64edd6..470e1deb362 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #if !defined(ARCADIA_BUILD) # include "Common/config_version.h" @@ -38,6 +39,13 @@ void setExtras() if (!anonymize) sentry_set_extra("server_name", sentry_value_new_string(getFQDNOrHostName().c_str())); + DB::UUID server_uuid = DB::ServerUUID::get(); + if (server_uuid != DB::UUIDHelpers::Nil) + { + std::string server_uuid_str = DB::toString(server_uuid); + sentry_set_extra("server_uuid", sentry_value_new_string(server_uuid_str.c_str())); + } + sentry_set_tag("version", VERSION_STRING); sentry_set_extra("version_githash", sentry_value_new_string(VERSION_GITHASH)); sentry_set_extra("version_describe", sentry_value_new_string(VERSION_DESCRIBE)); diff --git a/base/glibc-compatibility/musl/getauxval.c b/base/glibc-compatibility/musl/getauxval.c index dad7aa938d7..a429273fa1a 100644 --- a/base/glibc-compatibility/musl/getauxval.c +++ b/base/glibc-compatibility/musl/getauxval.c @@ -1,5 +1,4 @@ #include -#include "atomic.h" #include // __environ #include @@ -18,7 +17,18 @@ static size_t __find_auxv(unsigned long type) return (size_t) -1; } -unsigned long __getauxval(unsigned long type) +__attribute__((constructor)) static void __auxv_init() +{ + size_t i; + for (i = 0; __environ[i]; i++); + __auxv = (unsigned long *) (__environ + i + 1); + + size_t secure_idx = __find_auxv(AT_SECURE); + if (secure_idx != ((size_t) -1)) + __auxv_secure = __auxv[secure_idx]; +} + +unsigned long getauxval(unsigned long type) { if (type == AT_SECURE) return __auxv_secure; @@ -33,38 +43,3 @@ unsigned long __getauxval(unsigned long type) errno = ENOENT; return 0; } - -static void * volatile getauxval_func; - -static unsigned long __auxv_init(unsigned long type) -{ - if (!__environ) - { - // __environ is not initialized yet so we can't initialize __auxv right now. - // That's normally occurred only when getauxval() is called from some sanitizer's internal code. - errno = ENOENT; - return 0; - } - - // Initialize __auxv and __auxv_secure. - size_t i; - for (i = 0; __environ[i]; i++); - __auxv = (unsigned long *) (__environ + i + 1); - - size_t secure_idx = __find_auxv(AT_SECURE); - if (secure_idx != ((size_t) -1)) - __auxv_secure = __auxv[secure_idx]; - - // Now we've initialized __auxv, next time getauxval() will only call __get_auxval(). - a_cas_p(&getauxval_func, (void *)__auxv_init, (void *)__getauxval); - - return __getauxval(type); -} - -// First time getauxval() will call __auxv_init(). -static void * volatile getauxval_func = (void *)__auxv_init; - -unsigned long getauxval(unsigned long type) -{ - return ((unsigned long (*)(unsigned long))getauxval_func)(type); -} diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 2435335f669..03247b4b3ea 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54454) +SET(VERSION_REVISION 54455) SET(VERSION_MAJOR 21) -SET(VERSION_MINOR 9) +SET(VERSION_MINOR 10) SET(VERSION_PATCH 1) -SET(VERSION_GITHASH f063e44131a048ba2d9af8075f03700fd5ec3e69) -SET(VERSION_DESCRIBE v21.9.1.7770-prestable) -SET(VERSION_STRING 21.9.1.7770) +SET(VERSION_GITHASH 09df5018f95edcd0f759d4689ac5d029dd400c2a) +SET(VERSION_DESCRIBE v21.10.1.1-testing) +SET(VERSION_STRING 21.10.1.1) # end of autochange diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 52208bb0278..30dd3baa55b 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -1,9 +1,10 @@ -# Disabled under OSX until https://github.com/ClickHouse/ClickHouse/issues/27568 is fixed if (SANITIZE OR NOT ( - ((OS_LINUX OR OS_FREEBSD) AND (ARCH_AMD64 OR ARCH_ARM OR ARCH_PPC64LE)))) + ((OS_LINUX OR OS_FREEBSD) AND (ARCH_AMD64 OR ARCH_ARM OR ARCH_PPC64LE)) OR + (OS_DARWIN AND (CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo" OR CMAKE_BUILD_TYPE STREQUAL "Debug")) +)) if (ENABLE_JEMALLOC) message (${RECONFIGURE_MESSAGE_LEVEL} - "jemalloc is disabled implicitly: it doesn't work with sanitizers and can only be used with x86_64, aarch64, or ppc64le Linux or FreeBSD builds") + "jemalloc is disabled implicitly: it doesn't work with sanitizers and can only be used with x86_64, aarch64, or ppc64le Linux or FreeBSD builds and RelWithDebInfo macOS builds.") endif () set (ENABLE_JEMALLOC OFF) else () @@ -138,9 +139,5 @@ target_compile_options(jemalloc PRIVATE -Wno-redundant-decls) target_compile_options(jemalloc PRIVATE -D_GNU_SOURCE) set_property(TARGET jemalloc APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS USE_JEMALLOC=1) -if (MAKE_STATIC_LIBRARIES) - # To detect whether we need to register jemalloc for osx as default zone. - set_property(TARGET jemalloc APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS BUNDLED_STATIC_JEMALLOC=1) -endif() message (STATUS "Using jemalloc") diff --git a/contrib/librdkafka b/contrib/librdkafka index 43491d33ca2..b8554f16820 160000 --- a/contrib/librdkafka +++ b/contrib/librdkafka @@ -1 +1 @@ -Subproject commit 43491d33ca2826531d1e3cae70d4bf1e5249e3c9 +Subproject commit b8554f1682062c85ba519eb54ef2f90e02b812cb diff --git a/debian/changelog b/debian/changelog index 38f740ae062..f3e740d20cf 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (21.9.1.1) unstable; urgency=low +clickhouse (21.10.1.1) unstable; urgency=low * Modified source code - -- clickhouse-release Sat, 10 Jul 2021 08:22:49 +0300 + -- clickhouse-release Sat, 17 Jul 2021 08:45:03 +0300 diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index f17fa8ade16..052e008fd56 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=21.9.1.* +ARG version=21.10.1.* RUN apt-get update \ && apt-get install --yes --no-install-recommends \ diff --git a/docker/packager/packager b/docker/packager/packager index 95b7fcd8568..b53220886b1 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -173,6 +173,9 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ cmake_flags.append('-DUSE_GTEST=1') cmake_flags.append('-DENABLE_TESTS=1') cmake_flags.append('-DENABLE_EXAMPLES=1') + cmake_flags.append('-DENABLE_FUZZING=1') + # For fuzzing needs + cmake_flags.append('-DUSE_YAML_CPP=1') # Don't stop on first error to find more clang-tidy errors in one run. result.append('NINJA_FLAGS=-k0') diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 5da9e703f4d..25f01230c5f 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:20.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=21.9.1.* +ARG version=21.10.1.* ARG gosu_ver=1.10 # set non-empty deb_location_url url to create a docker image diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index 5768753cd7c..62cfcf9e896 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=21.9.1.* +ARG version=21.10.1.* RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index e5c9f349ce3..c97e8a6ed2b 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -628,9 +628,6 @@ cat analyze/errors.log >> report/errors.log ||: cat profile-errors.log >> report/errors.log ||: clickhouse-local --query " --- We use decimals specifically to get fixed-point, fixed-width formatting. -set output_format_decimal_trailing_zeros = 1; - create view query_display_names as select * from file('analyze/query-display-names.tsv', TSV, 'test text, query_index int, query_display_name text') @@ -644,6 +641,7 @@ create view partial_query_times as select * from -- Report for partial queries that we could only run on the new server (e.g. -- queries with new functions added in the tested PR). create table partial_queries_report engine File(TSV, 'report/partial-queries-report.tsv') + settings output_format_decimal_trailing_zeros = 1 as select toDecimal64(time_median, 3) time, toDecimal64(time_stddev / time_median, 3) relative_time_stddev, test, query_index, query_display_name @@ -716,8 +714,9 @@ create table queries engine File(TSVWithNamesAndTypes, 'report/queries.tsv') order by test, query_index, metric_name ; -create table changed_perf_report engine File(TSV, 'report/changed-perf.tsv') as - with +create table changed_perf_report engine File(TSV, 'report/changed-perf.tsv') + settings output_format_decimal_trailing_zeros = 1 + as with -- server_time is sometimes reported as zero (if it's less than 1 ms), -- so we have to work around this to not get an error about conversion -- of NaN to decimal. @@ -733,8 +732,9 @@ create table changed_perf_report engine File(TSV, 'report/changed-perf.tsv') as changed_fail, test, query_index, query_display_name from queries where changed_show order by abs(diff) desc; -create table unstable_queries_report engine File(TSV, 'report/unstable-queries.tsv') as - select +create table unstable_queries_report engine File(TSV, 'report/unstable-queries.tsv') + settings output_format_decimal_trailing_zeros = 1 + as select toDecimal64(left, 3), toDecimal64(right, 3), toDecimal64(diff, 3), toDecimal64(stat_threshold, 3), unstable_fail, test, query_index, query_display_name from queries where unstable_show order by stat_threshold desc; @@ -764,8 +764,9 @@ create view total_speedup as from test_speedup ; -create table test_perf_changes_report engine File(TSV, 'report/test-perf-changes.tsv') as - with +create table test_perf_changes_report engine File(TSV, 'report/test-perf-changes.tsv') + settings output_format_decimal_trailing_zeros = 1 + as with (times_speedup >= 1 ? '-' || toString(toDecimal64(times_speedup, 3)) || 'x' : '+' || toString(toDecimal64(1 / times_speedup, 3)) || 'x') @@ -791,8 +792,9 @@ create view total_client_time_per_query as select * from file('analyze/client-times.tsv', TSV, 'test text, query_index int, client float, server float'); -create table slow_on_client_report engine File(TSV, 'report/slow-on-client.tsv') as - select client, server, toDecimal64(client/server, 3) p, +create table slow_on_client_report engine File(TSV, 'report/slow-on-client.tsv') + settings output_format_decimal_trailing_zeros = 1 + as select client, server, toDecimal64(client/server, 3) p, test, query_display_name from total_client_time_per_query left join query_display_names using (test, query_index) where p > toDecimal64(1.02, 3) order by p desc; @@ -877,8 +879,9 @@ create view test_times_view_total as from test_times_view ; -create table test_times_report engine File(TSV, 'report/test-times.tsv') as - select +create table test_times_report engine File(TSV, 'report/test-times.tsv') + settings output_format_decimal_trailing_zeros = 1 + as select test, toDecimal64(real, 3), toDecimal64(total_client_time, 3), @@ -896,8 +899,9 @@ create table test_times_report engine File(TSV, 'report/test-times.tsv') as ; -- report for all queries page, only main metric -create table all_tests_report engine File(TSV, 'report/all-queries.tsv') as - with +create table all_tests_report engine File(TSV, 'report/all-queries.tsv') + settings output_format_decimal_trailing_zeros = 1 + as with -- server_time is sometimes reported as zero (if it's less than 1 ms), -- so we have to work around this to not get an error about conversion -- of NaN to decimal. @@ -978,9 +982,6 @@ for version in {right,left} do rm -rf data clickhouse-local --query " --- We use decimals specifically to get fixed-point, fixed-width formatting. -set output_format_decimal_trailing_zeros = 1; - create view query_profiles as with 0 as left, 1 as right select * from file('analyze/query-profiles.tsv', TSV, @@ -1063,9 +1064,10 @@ create table unstable_run_traces engine File(TSVWithNamesAndTypes, ; create table metric_devation engine File(TSVWithNamesAndTypes, - 'report/metric-deviation.$version.tsv') as + 'report/metric-deviation.$version.tsv') + settings output_format_decimal_trailing_zeros = 1 -- first goes the key used to split the file with grep - select test, query_index, query_display_name, + as select test, query_index, query_display_name, toDecimal64(d, 3) d, q, metric from ( select @@ -1176,9 +1178,6 @@ rm -rf metrics ||: mkdir metrics clickhouse-local --query " --- We use decimals specifically to get fixed-point, fixed-width formatting. -set output_format_decimal_trailing_zeros = 1; - create view right_async_metric_log as select * from file('right-async-metric-log.tsv', TSVWithNamesAndTypes, '$(cat right-async-metric-log.tsv.columns)') @@ -1196,8 +1195,9 @@ create table metrics engine File(TSV, 'metrics/metrics.tsv') as ; -- Show metrics that have changed -create table changes engine File(TSV, 'metrics/changes.tsv') as - select metric, left, right, +create table changes engine File(TSV, 'metrics/changes.tsv') + settings output_format_decimal_trailing_zeros = 1 + as select metric, left, right, toDecimal64(diff, 3), toDecimal64(times_diff, 3) from ( select metric, median(left) as left, median(right) as right, diff --git a/docker/test/performance-comparison/download.sh b/docker/test/performance-comparison/download.sh index bd72547ec1c..49323c28700 100755 --- a/docker/test/performance-comparison/download.sh +++ b/docker/test/performance-comparison/download.sh @@ -13,7 +13,7 @@ left_sha=$2 # right_pr=$3 not used for now right_sha=$4 -datasets=${CHPC_DATASETS:-"hits1 hits10 hits100 values"} +datasets=${CHPC_DATASETS-"hits1 hits10 hits100 values"} declare -A dataset_paths dataset_paths["hits10"]="https://s3.mds.yandex.net/clickhouse-private-datasets/hits_10m_single/partitions/hits_10m_single.tar" diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh index 614debce1c1..1295e5567fb 100755 --- a/docker/test/performance-comparison/entrypoint.sh +++ b/docker/test/performance-comparison/entrypoint.sh @@ -127,6 +127,15 @@ export PATH export REF_PR export REF_SHA +# Try to collect some core dumps. I've seen two patterns in Sandbox: +# 1) |/home/zomb-sandbox/venv/bin/python /home/zomb-sandbox/client/sandbox/bin/coredumper.py %e %p %g %u %s %P %c +# Not sure what this script does (puts them to sandbox resources, logs some messages?), +# and it's not accessible from inside docker anyway. +# 2) something like %e.%p.core.dmp. The dump should end up in the workspace directory. +# At least we remove the ulimit and then try to pack some common file names into output. +ulimit -c unlimited +cat /proc/sys/kernel/core_pattern + # Start the main comparison script. { \ time ../download.sh "$REF_PR" "$REF_SHA" "$PR_TO_TEST" "$SHA_TO_TEST" && \ @@ -144,8 +153,11 @@ done dmesg -T > dmesg.log +ls -lath + 7z a '-x!*/tmp' /output/output.7z ./*.{log,tsv,html,txt,rep,svg,columns} \ {right,left}/{performance,scripts} {{right,left}/db,db0}/preprocessed_configs \ - report analyze benchmark metrics + report analyze benchmark metrics \ + ./*.core.dmp ./*.core cp compare.log /output diff --git a/docs/en/engines/table-engines/mergetree-family/replication.md b/docs/en/engines/table-engines/mergetree-family/replication.md index 4fc30355927..277339f9b47 100644 --- a/docs/en/engines/table-engines/mergetree-family/replication.md +++ b/docs/en/engines/table-engines/mergetree-family/replication.md @@ -137,7 +137,7 @@ CREATE TABLE table_name ) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}', ver) PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) -SAMPLE BY intHash32(UserID) +SAMPLE BY intHash32(UserID); ```
@@ -150,12 +150,12 @@ CREATE TABLE table_name EventDate DateTime, CounterID UInt32, UserID UInt32 -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}', EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID), EventTime), 8192) +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}', EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID), EventTime), 8192); ```
-As the example shows, these parameters can contain substitutions in curly brackets. The substituted values are taken from the «[macros](../../../operations/server-configuration-parameters/settings/#macros) section of the configuration file. +As the example shows, these parameters can contain substitutions in curly brackets. The substituted values are taken from the [macros](../../../operations/server-configuration-parameters/settings.md#macros) section of the configuration file. Example: diff --git a/docs/en/engines/table-engines/special/buffer.md b/docs/en/engines/table-engines/special/buffer.md index cacb310a15c..0e7f0a53da8 100644 --- a/docs/en/engines/table-engines/special/buffer.md +++ b/docs/en/engines/table-engines/special/buffer.md @@ -56,6 +56,9 @@ The same thing happens if the subordinate table does not exist when the buffer i If you need to run ALTER for a subordinate table, and the Buffer table, we recommend first deleting the Buffer table, running ALTER for the subordinate table, then creating the Buffer table again. +!!! attention "Attention" + Running ALTER on the Buffer table in releases made before 28 Sep 2020 will cause a `Block structure mismatch` error (see [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117)), so deleting the Buffer table and then recreating is the only option. It is advisable to check that this error is fixed in your release before trying to run ALTER on the Buffer table. + If the server is restarted abnormally, the data in the buffer is lost. `FINAL` and `SAMPLE` do not work correctly for Buffer tables. These conditions are passed to the destination table, but are not used for processing data in the buffer. If these features are required we recommend only using the Buffer table for writing, while reading from the destination table. diff --git a/docs/en/getting-started/example-datasets/menus.md b/docs/en/getting-started/example-datasets/menus.md index 87e4c75d0d4..8f330f39226 100644 --- a/docs/en/getting-started/example-datasets/menus.md +++ b/docs/en/getting-started/example-datasets/menus.md @@ -105,7 +105,7 @@ We use `Decimal` data type to store prices. Everything else is quite straightfor ## Import Data -Upload data into ClickHouse in parallel: +Upload data into ClickHouse: ``` clickhouse-client --format_csv_allow_single_quotes 0 --input_format_null_as_default 0 --query "INSERT INTO dish FORMAT CSVWithNames" < Dish.csv diff --git a/docs/en/operations/clickhouse-keeper.md b/docs/en/operations/clickhouse-keeper.md index 5fc1baa003c..389cf2361f8 100644 --- a/docs/en/operations/clickhouse-keeper.md +++ b/docs/en/operations/clickhouse-keeper.md @@ -114,5 +114,5 @@ Seamlessly migration from ZooKeeper to `clickhouse-keeper` is impossible you hav clickhouse-keeper-converter --zookeeper-logs-dir /var/lib/zookeeper/version-2 --zookeeper-snapshots-dir /var/lib/zookeeper/version-2 --output-dir /path/to/clickhouse/keeper/snapshots ``` -4. Copy snapshot to `clickhouse-server` nodes with configured `keeper` or start `clickhouse-keeper` instead of ZooKeeper. Snapshot must persist only on leader node, leader will sync it automatically to other nodes. +4. Copy snapshot to `clickhouse-server` nodes with configured `keeper` or start `clickhouse-keeper` instead of ZooKeeper. Snapshot must persist on all nodes, otherwise empty nodes can be faster and one of them can becamse leader. diff --git a/docs/en/operations/configuration-files.md b/docs/en/operations/configuration-files.md index 5c942efc77f..44f9353063c 100644 --- a/docs/en/operations/configuration-files.md +++ b/docs/en/operations/configuration-files.md @@ -18,6 +18,18 @@ Some settings specified in the main configuration file can be overridden in othe - If `replace` is specified, it replaces the entire element with the specified one. - If `remove` is specified, it deletes the element. +You can also declare attributes as coming from environment variables by using `from_env="VARIABLE_NAME"`: + +```xml + + + + + + + +``` + ## Substitution {#substitution} The config can also define “substitutions”. If an element has the `incl` attribute, the corresponding substitution from the file will be used as the value. By default, the path to the file with substitutions is `/etc/metrika.xml`. This can be changed in the [include_from](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-include_from) element in the server config. The substitution values are specified in `/yandex/substitution_name` elements in this file. If a substitution specified in `incl` does not exist, it is recorded in the log. To prevent ClickHouse from logging missing substitutions, specify the `optional="true"` attribute (for example, settings for [macros](../operations/server-configuration-parameters/settings.md)). diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 7179f552460..66a2bde4b4b 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -486,7 +486,7 @@ Parameter substitutions for replicated tables. Can be omitted if replicated tables are not used. -For more information, see the section “[Creating replicated tables](../../engines/table-engines/mergetree-family/replication.md)”. +For more information, see the section [Creating replicated tables](../../engines/table-engines/mergetree-family/replication.md#creating-replicated-tables). **Example** diff --git a/docs/en/operations/system-tables/opentelemetry_span_log.md b/docs/en/operations/system-tables/opentelemetry_span_log.md index e45a989742c..9e36eae7a1b 100644 --- a/docs/en/operations/system-tables/opentelemetry_span_log.md +++ b/docs/en/operations/system-tables/opentelemetry_span_log.md @@ -4,7 +4,7 @@ Contains information about [trace spans](https://opentracing.io/docs/overview/sp Columns: -- `trace_id` ([UUID](../../sql-reference/data-types/uuid.md) — ID of the trace for executed query. +- `trace_id` ([UUID](../../sql-reference/data-types/uuid.md)) — ID of the trace for executed query. - `span_id` ([UInt64](../../sql-reference/data-types/int-uint.md)) — ID of the `trace span`. diff --git a/docs/en/operations/system-tables/zookeeper_log.md b/docs/en/operations/system-tables/zookeeper_log.md new file mode 100644 index 00000000000..f7d86c6689b --- /dev/null +++ b/docs/en/operations/system-tables/zookeeper_log.md @@ -0,0 +1,129 @@ +# system.zookeeper_log {#system-zookeeper_log} + +This table contains information about the parameters of the request to the ZooKeeper server and the response from it. + +For requests, only columns with request parameters are filled in, and the remaining columns are filled with default values (`0` or `NULL`). When the response arrives, the data from the response is added to the other columns. + +Columns with request parameters: + +- `type` ([Enum](../../sql-reference/data-types/enum.md)) — Event type in the ZooKeeper client. Can have one of the following values: + - `Request` — The request has been sent. + - `Response` — The response was received. + - `Finalize` — The connection is lost, no response was received. +- `event_date` ([Date](../../sql-reference/data-types/date.md)) — The date when the event happened. +- `event_time` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — The date and time when the event happened. +- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address of ZooKeeper server that was used to make the request. +- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — The port of ZooKeeper server that was used to make the request. +- `session_id` ([Int64](../../sql-reference/data-types/int-uint.md)) — The session ID that the ZooKeeper server sets for each connection. +- `xid` ([Int32](../../sql-reference/data-types/int-uint.md)) — The ID of the request within the session. This is usually a sequential request number. It is the same for the request row and the paired `response`/`finalize` row. +- `has_watch` ([UInt8](../../sql-reference/data-types/int-uint.md)) — The request whether the [watch](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#ch_zkWatches) has been set. +- `op_num` ([Enum](../../sql-reference/data-types/enum.md)) — The type of request or response. +- `path` ([String](../../sql-reference/data-types/string.md)) — The path to the ZooKeeper node specified in the request, or an empty string if the request not requires specifying a path. +- `data` ([String](../../sql-reference/data-types/string.md)) — The data written to the ZooKeeper node (for the `SET` and `CREATE` requests — what the request wanted to write, for the response to the `GET` request — what was read) or an empty string. +- `is_ephemeral` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Is the ZooKeeper node being created as an [ephemeral](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#Ephemeral+Nodes). +- `is_sequential` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Is the ZooKeeper node being created as an [sequential](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#Sequence+Nodes+--+Unique+Naming). +- `version` ([Nullable(Int32)](../../sql-reference/data-types/nullable.md)) — The version of the ZooKeeper node that the request expects when executing. This is supported for `CHECK`, `SET`, `REMOVE` requests (is relevant `-1` if the request does not check the version or `NULL` for other requests that do not support version checking). +- `requests_size` ([UInt32](../../sql-reference/data-types/int-uint.md)) — The number of requests included in the multi request (this is a special request that consists of several consecutive ordinary requests and executes them atomically). All requests included in multi request will have the same `xid`. +- `request_idx` ([UInt32](../../sql-reference/data-types/int-uint.md)) — The number of the request included in multi request (for multi request — `0`, then in order from `1`). + +Columns with request response parameters: + +- `zxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — ZooKeeper transaction ID. The serial number issued by the ZooKeeper server in response to a successfully executed request (`0` if the request was not executed/returned an error/the client does not know whether the request was executed). +- `error` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — Error code. Can have many values, here are just some of them: + - `ZOK` — The request was executed seccessfully. + - `ZCONNECTIONLOSS` — The connection was lost. + - `ZOPERATIONTIMEOUT` — The request execution timeout has expired. + - `ZSESSIONEXPIRED` — The session has expired. + - `NULL` — The request is completed. +- `watch_type` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — The type of the `watch` event (for responses with `op_num` = `Watch`), for the remaining responses: `NULL`. +- `watch_state` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — The status of the `watch` event (for responses with `op_num` = `Watch`), for the remaining responses: `NULL`. +- `path_created` ([String](../../sql-reference/data-types/string.md)) — The path to the created ZooKeeper node (for responses to the `CREATE` request), may differ from the `path` if the node is created as a `sequential`. +- `stat_czxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — The `zxid` of the change that caused this ZooKeeper node to be created. +- `stat_mzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — The `zxid` of the change that last modified this ZooKeeper node. +- `stat_pzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — The transaction ID of the change that last modified childern of this ZooKeeper node. +- `stat_version` ([Int32](../../sql-reference/data-types/int-uint.md)) — The number of changes to the data of this ZooKeeper node. +- `stat_cversion` ([Int32](../../sql-reference/data-types/int-uint.md)) — The number of changes to the children of this ZooKeeper node. +- `stat_dataLength` ([Int32](../../sql-reference/data-types/int-uint.md)) — The length of the data field of this ZooKeeper node. +- `stat_numChildren` ([Int32](../../sql-reference/data-types/int-uint.md)) — The number of children of this ZooKeeper node. +- `children` ([Array(String)](../../sql-reference/data-types/array.md)) — The list of child ZooKeeper nodes (for responses to `LIST` request). + +**Example** + +Query: + +``` sql +SELECT * FROM system.zookeeper_log WHERE (session_id = '106662742089334927') AND (xid = '10858') FORMAT Vertical; +``` + +Result: + +``` text +Row 1: +────── +type: Request +event_date: 2021-08-09 +event_time: 2021-08-09 21:38:30.291792 +address: :: +port: 2181 +session_id: 106662742089334927 +xid: 10858 +has_watch: 1 +op_num: List +path: /clickhouse/task_queue/ddl +data: +is_ephemeral: 0 +is_sequential: 0 +version: ᴺᵁᴸᴸ +requests_size: 0 +request_idx: 0 +zxid: 0 +error: ᴺᵁᴸᴸ +watch_type: ᴺᵁᴸᴸ +watch_state: ᴺᵁᴸᴸ +path_created: +stat_czxid: 0 +stat_mzxid: 0 +stat_pzxid: 0 +stat_version: 0 +stat_cversion: 0 +stat_dataLength: 0 +stat_numChildren: 0 +children: [] + +Row 2: +────── +type: Response +event_date: 2021-08-09 +event_time: 2021-08-09 21:38:30.292086 +address: :: +port: 2181 +session_id: 106662742089334927 +xid: 10858 +has_watch: 1 +op_num: List +path: /clickhouse/task_queue/ddl +data: +is_ephemeral: 0 +is_sequential: 0 +version: ᴺᵁᴸᴸ +requests_size: 0 +request_idx: 0 +zxid: 16926267 +error: ZOK +watch_type: ᴺᵁᴸᴸ +watch_state: ᴺᵁᴸᴸ +path_created: +stat_czxid: 16925469 +stat_mzxid: 16925469 +stat_pzxid: 16926179 +stat_version: 0 +stat_cversion: 7 +stat_dataLength: 0 +stat_numChildren: 7 +children: ['query-0000000006','query-0000000005','query-0000000004','query-0000000003','query-0000000002','query-0000000001','query-0000000000'] +``` + +**See Also** + +- [ZooKeeper](../../operations/tips.md#zookeeper) +- [ZooKeeper guide](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html) diff --git a/docs/en/sql-reference/functions/geo/h3.md b/docs/en/sql-reference/functions/geo/h3.md index e178603dc45..ad7b4657af7 100644 --- a/docs/en/sql-reference/functions/geo/h3.md +++ b/docs/en/sql-reference/functions/geo/h3.md @@ -197,7 +197,7 @@ Result: ## h3ToGeo {#h3togeo} -Returns `(lon, lat)` that corresponds to the provided H3 index. +Returns the geographical coordinates of longitude and latitude corresponding to the provided [H3](#h3index) index. **Syntax** @@ -207,20 +207,18 @@ h3ToGeo(h3Index) **Arguments** -- `h3Index` — H3 Index. Type: [UInt64](../../../sql-reference/data-types/int-uint.md). +- `h3Index` — H3 Index. [UInt64](../../../sql-reference/data-types/int-uint.md). **Returned values** -- `lon` — Longitude. Type: [Float64](../../../sql-reference/data-types/float.md). -- `lat` — Latitude. Type: [Float64](../../../sql-reference/data-types/float.md). - +- A tuple consisting of two values: `tuple(lon,lat)`. `lon` — Longitude. [Float64](../../../sql-reference/data-types/float.md). `lat` — Latitude. [Float64](../../../sql-reference/data-types/float.md). **Example** Query: ``` sql -SELECT h3ToGeo(644325524701193974) coordinates; +SELECT h3ToGeo(644325524701193974) AS coordinates; ``` Result: @@ -230,6 +228,7 @@ Result: │ (37.79506616830252,55.71290243145668) │ └───────────────────────────────────────┘ ``` + ## h3kRing {#h3kring} Lists all the [H3](#h3index) hexagons in the raduis of `k` from the given hexagon in random order. diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index efd28def688..ad6edaea312 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -1339,3 +1339,149 @@ Result: │ 2,"good" │ └───────────────────────────────────────────┘ ``` + +## snowflakeToDateTime {#snowflakeToDateTime} + +Extract time from snowflake id as DateTime format. + +**Syntax** + +``` sql +snowflakeToDateTime(value [, time_zone]) +``` + +**Parameters** + +- `value` — `snowflake id`, Int64 value. +- `time_zone` — [Timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../../sql-reference/data-types/string.md). + +**Returned value** + +- value converted to the `DateTime` data type. + +**Example** + +Query: + +``` sql +SELECT snowflakeToDateTime(CAST('1426860702823350272', 'Int64'), 'UTC'); +``` + +Result: + +``` text + +┌─snowflakeToDateTime(CAST('1426860702823350272', 'Int64'), 'UTC')─┐ +│ 2021-08-15 10:57:56 │ +└──────────────────────────────────────────────────────────────────┘ +``` + +## snowflakeToDateTime64 {#snowflakeToDateTime64} + +Extract time from snowflake id as DateTime64 format. + +**Syntax** + +``` sql +snowflakeToDateTime64(value [, time_zone]) +``` + +**Parameters** + +- `value` — `snowflake id`, Int64 value. +- `time_zone` — [Timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../../sql-reference/data-types/string.md). + +**Returned value** + +- value converted to the `DateTime64` data type. + +**Example** + +Query: + +``` sql +SELECT snowflakeToDateTime64(CAST('1426860802823350272', 'Int64'), 'UTC'); +``` + +Result: + +``` text + +┌─snowflakeToDateTime64(CAST('1426860802823350272', 'Int64'), 'UTC')─┐ +│ 2021-08-15 10:58:19.841 │ +└────────────────────────────────────────────────────────────────────┘ +``` + +## dateTimeToSnowflake {#dateTimeToSnowflake} + +Convert DateTime to the first snowflake id at the giving time. + +**Syntax** + +``` sql +dateTimeToSnowflake(value) +``` + +**Parameters** + +- `value` — Date and time. [DateTime](../../sql-reference/data-types/datetime.md). + + +**Returned value** + +- `value` converted to the `Int64` data type as the first snowflake id at that time. + +**Example** + +Query: + +``` sql +WITH toDateTime('2021-08-15 18:57:56', 'Asia/Shanghai') AS dt +SELECT dateTimeToSnowflake(dt); +``` + +Result: + +``` text + +┌─dateTimeToSnowflake(dt)─┐ +│ 1426860702823350272 │ +└─────────────────────────┘ +``` + + +## dateTime64ToSnowflake {#dateTime64ToSnowflake} + +Convert DateTime64 to the first snowflake id at the giving time. + +**Syntax** + +``` sql +dateTime64ToSnowflake(value) +``` + +**Parameters** + +- `value` — Date and time. [DateTime64](../../sql-reference/data-types/datetime64.md). + + +**Returned value** + +- `value` converted to the `Int64` data type as the first snowflake id at that time. + +**Example** + +Query: + +``` sql +WITH toDateTime64('2021-08-15 18:57:56.492', 3, 'Asia/Shanghai') AS dt64 +SELECT dateTime64ToSnowflake(dt64); +``` + +Result: + +``` text +┌─dateTime64ToSnowflake(dt64)─┐ +│ 1426860704886947840 │ +└─────────────────────────────┘ +``` \ No newline at end of file diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index d1526c10203..cf2a99a4c5f 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -311,12 +311,12 @@ One may execute query after: - Individual replica path `/replicas/replica_name/` loss. Replica attaches locally found parts and sends info about them to Zookeeper. -Parts present on replica before metadata loss are not re-fetched from other replicas if not being outdated -(so replica restoration does not mean re-downloading all data over the network). +Parts present on a replica before metadata loss are not re-fetched from other ones if not being outdated (so replica restoration does not mean re-downloading all data over the network). -Caveat: parts in all states are moved to `detached/` folder. Parts active before data loss (Committed) are attached. +!!! warning "Warning" + Parts in all states are moved to `detached/` folder. Parts active before data loss (committed) are attached. -#### Syntax +**Syntax** ```sql SYSTEM RESTORE REPLICA [db.]replicated_merge_tree_family_table_name [ON CLUSTER cluster_name] @@ -328,11 +328,11 @@ Alternative syntax: SYSTEM RESTORE REPLICA [ON CLUSTER cluster_name] [db.]replicated_merge_tree_family_table_name ``` -#### Example +**Example** + +Creating a table on multiple servers. After the replica's metadata in ZooKeeper is lost, the table will attach as read-only as metadata is missing. The last query needs to execute on every replica. ```sql --- Creating table on multiple servers - CREATE TABLE test(n UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/', '{replica}') ORDER BY n PARTITION BY n % 10; @@ -341,8 +341,14 @@ INSERT INTO test SELECT * FROM numbers(1000); -- zookeeper_delete_path("/clickhouse/tables/test", recursive=True) <- root loss. -SYSTEM RESTART REPLICA test; -- Table will attach as readonly as metadata is missing. -SYSTEM RESTORE REPLICA test; -- Need to execute on every replica, another way: RESTORE REPLICA test ON CLUSTER cluster +SYSTEM RESTART REPLICA test; +SYSTEM RESTORE REPLICA test; +``` + +Another way: + +```sql +SYSTEM RESTORE REPLICA test ON CLUSTER cluster; ``` ### RESTART REPLICAS {#query_language-system-restart-replicas} diff --git a/docs/en/sql-reference/table-functions/cluster.md b/docs/en/sql-reference/table-functions/cluster.md index 2856e66db9b..a02c2a10fb7 100644 --- a/docs/en/sql-reference/table-functions/cluster.md +++ b/docs/en/sql-reference/table-functions/cluster.md @@ -6,12 +6,13 @@ toc_title: cluster # cluster, clusterAllReplicas {#cluster-clusterallreplicas} Allows to access all shards in an existing cluster which configured in `remote_servers` section without creating a [Distributed](../../engines/table-engines/special/distributed.md) table. One replica of each shard is queried. -`clusterAllReplicas` - same as `cluster` but all replicas are queried. Each replica in a cluster is used as separate shard/connection. + +`clusterAllReplicas` function — same as `cluster`, but all replicas are queried. Each replica in a cluster is used as a separate shard/connection. !!! note "Note" - All available clusters are listed in the `system.clusters` table. + All available clusters are listed in the [system.clusters](../../operations/system-tables/clusters.md) table. -Signatures: +**Syntax** ``` sql cluster('cluster_name', db.table[, sharding_key]) @@ -19,10 +20,27 @@ cluster('cluster_name', db, table[, sharding_key]) clusterAllReplicas('cluster_name', db.table[, sharding_key]) clusterAllReplicas('cluster_name', db, table[, sharding_key]) ``` +**Arguments** -`cluster_name` – Name of a cluster that is used to build a set of addresses and connection parameters to remote and local servers. +- `cluster_name` – Name of a cluster that is used to build a set of addresses and connection parameters to remote and local servers. +- `db.table` or `db`, `table` - Name of a database and a table. +- `sharding_key` - A sharding key. Optional. Needs to be specified if the cluster has more than one shard. -`sharding_key` - When insert into cluster function with more than one shard, sharding_key need to be provided. +**Returned value** + +The dataset from clusters. + +**Using Macros** + +`cluster_name` can contain macros — substitution in curly brackets. The substituted value is taken from the [macros](../../operations/server-configuration-parameters/settings.md#macros) section of the server configuration file. + +Example: + +```sql +SELECT * FROM cluster('{cluster}', default.example_table); +``` + +**Usage and Recommendations** Using the `cluster` and `clusterAllReplicas` table functions are less efficient than creating a `Distributed` table because in this case, the server connection is re-established for every request. When processing a large number of queries, please always create the `Distributed` table ahead of time, and do not use the `cluster` and `clusterAllReplicas` table functions. diff --git a/docs/ru/engines/table-engines/mergetree-family/replication.md b/docs/ru/engines/table-engines/mergetree-family/replication.md index 6a259ebd3b8..db749ba455e 100644 --- a/docs/ru/engines/table-engines/mergetree-family/replication.md +++ b/docs/ru/engines/table-engines/mergetree-family/replication.md @@ -102,7 +102,7 @@ CREATE TABLE table_name ) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}', ver) PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) -SAMPLE BY intHash32(UserID) +SAMPLE BY intHash32(UserID); ```
@@ -115,12 +115,12 @@ CREATE TABLE table_name EventDate DateTime, CounterID UInt32, UserID UInt32 -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}', EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID), EventTime), 8192) +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}', EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID), EventTime), 8192); ```
-Как видно в примере, эти параметры могут содержать подстановки в фигурных скобках. Подставляемые значения достаются из конфигурационного файла, из секции «[macros](../../../operations/server-configuration-parameters/settings/#macros)». +Как видно в примере, эти параметры могут содержать подстановки в фигурных скобках. Эти подстановки заменяются на соответствующие значения из конфигурационного файла, из секции [macros](../../../operations/server-configuration-parameters/settings.md#macros). Пример: diff --git a/docs/ru/engines/table-engines/special/buffer.md b/docs/ru/engines/table-engines/special/buffer.md index ba865b72b78..0c1ae591ae3 100644 --- a/docs/ru/engines/table-engines/special/buffer.md +++ b/docs/ru/engines/table-engines/special/buffer.md @@ -48,7 +48,10 @@ CREATE TABLE merge.hits_buffer AS merge.hits ENGINE = Buffer(merge, hits, 16, 10 Если у одного из столбцов таблицы Buffer и подчинённой таблицы не совпадает тип, то в лог сервера будет записано сообщение об ошибке и буфер будет очищен. То же самое происходит, если подчинённая таблица не существует в момент сброса буфера. -Если есть необходимость выполнить ALTER для подчинённой таблицы и для таблицы Buffer, то рекомендуется удалить таблицу Buffer, затем выполнить ALTER подчинённой таблицы, а затем создать таблицу Buffer заново. +Если есть необходимость выполнить ALTER для подчинённой таблицы и для таблицы Buffer, то рекомендуется удалить таблицу Buffer, затем выполнить ALTER подчинённой таблицы, а после создать таблицу Buffer заново. + +!!! attention "Внимание" + В релизах до 28 сентября 2020 года выполнение ALTER на таблице Buffer ломает структуру блоков и вызывает ошибку (см. [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117)), поэтому удаление буфера и его пересоздание — единственный вариант миграции для данного движка. Перед выполнением ALTER на таблице Buffer убедитесь, что в вашей версии эта ошибка устранена. При нештатном перезапуске сервера, данные, находящиеся в буфере, будут потеряны. diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index a8ae3f7eb3e..98c5748ba41 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -465,9 +465,9 @@ ClickHouse проверяет условия для `min_part_size` и `min_part Подстановки параметров реплицируемых таблиц. -Можно не указывать, если реплицируемых таблицы не используются. +Можно не указывать, если реплицируемые таблицы не используются. -Подробнее смотрите в разделе «[Создание реплицируемых таблиц](../../engines/table-engines/mergetree-family/replication.md)». +Подробнее смотрите в разделе [Создание реплицируемых таблиц](../../engines/table-engines/mergetree-family/replication.md#creating-replicated-tables). **Пример** diff --git a/docs/ru/operations/system-tables/opentelemetry_span_log.md b/docs/ru/operations/system-tables/opentelemetry_span_log.md index c421a602300..5c96f22b6c2 100644 --- a/docs/ru/operations/system-tables/opentelemetry_span_log.md +++ b/docs/ru/operations/system-tables/opentelemetry_span_log.md @@ -4,7 +4,7 @@ Столбцы: -- `trace_id` ([UUID](../../sql-reference/data-types/uuid.md) — идентификатор трассировки для выполненного запроса. +- `trace_id` ([UUID](../../sql-reference/data-types/uuid.md)) — идентификатор трассировки для выполненного запроса. - `span_id` ([UInt64](../../sql-reference/data-types/int-uint.md)) — идентификатор `trace span`. diff --git a/docs/ru/operations/system-tables/zookeeper_log.md b/docs/ru/operations/system-tables/zookeeper_log.md new file mode 100644 index 00000000000..16f02cb0489 --- /dev/null +++ b/docs/ru/operations/system-tables/zookeeper_log.md @@ -0,0 +1,129 @@ +# system.zookeeper_log {#system-zookeeper_log} + +Эта таблица содержит информацию о параметрах запроса к серверу ZooKeeper и ответа от него. + +Для запросов заполняются только столбцы с параметрами запроса, а остальные столбцы заполняются значениями по умолчанию (`0` или `NULL`). Когда поступает ответ, данные добавляются в столбцы с параметрами ответа на запрос. + +Столбцы с параметрами запроса: + +- `type` ([Enum](../../sql-reference/data-types/enum.md)) — тип события в клиенте ZooKeeper. Может иметь одно из следующих значений: + - `Request` — запрос отправлен. + - `Response` — ответ получен. + - `Finalize` — соединение разорвано, ответ не получен. +- `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата, когда произошло событие. +- `event_time` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — дата и время, когда произошло событие. +- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP адрес сервера ZooKeeper, с которого был сделан запрос. +- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — порт сервера ZooKeeper, с которого был сделан запрос. +- `session_id` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор сессии, который сервер ZooKeeper создает для каждого соединения. +- `xid` ([Int32](../../sql-reference/data-types/int-uint.md)) — идентификатор запроса внутри сессии. Обычно это последовательный номер запроса, одинаковый у строки запроса и у парной строки `response`/`finalize`. +- `has_watch` ([UInt8](../../sql-reference/data-types/int-uint.md)) — установлен ли запрос [watch](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#ch_zkWatches). +- `op_num` ([Enum](../../sql-reference/data-types/enum.md)) — тип запроса или ответа на запрос. +- `path` ([String](../../sql-reference/data-types/string.md)) — путь к узлу ZooKeeper, указанный в запросе. Пустая строка, если запрос не требует указания пути. +- `data` ([String](../../sql-reference/data-types/string.md)) — данные, записанные на узле ZooKeeper (для запросов `SET` и `CREATE` — что запрос хотел записать, для ответа на запрос `GET` — что было прочитано), или пустая строка. +- `is_ephemeral` ([UInt8](../../sql-reference/data-types/int-uint.md)) — создается ли узел ZooKeeper как [ephemeral](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#Ephemeral+Nodes). +- `is_sequential` ([UInt8](../../sql-reference/data-types/int-uint.md)) — создается ли узел ZooKeeper как [sequential](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#Sequence+Nodes+--+Unique+Naming). +- `version` ([Nullable(Int32)](../../sql-reference/data-types/nullable.md)) — версия узла ZooKeeper, которую запрос ожидает увидеть при выполнении. Поддерживается для запросов `CHECK`, `SET`, `REMOVE` (`-1` — запрос не проверяет версию, `NULL` — для других запросов, которые не поддерживают проверку версии). +- `requests_size` ([UInt32](../../sql-reference/data-types/int-uint.md)) — количество запросов, включенных в мультизапрос (это специальный запрос, который состоит из нескольких последовательных обычных запросов, выполняющихся атомарно). Все запросы, включенные в мультизапрос, имеют одинаковый `xid`. +- `request_idx` ([UInt32](../../sql-reference/data-types/int-uint.md)) — номер запроса, включенного в мультизапрос (`0` — для мультизапроса, далее по порядку с `1`). + +Столбцы с параметрами ответа на запрос: + +- `zxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор транзакции в ZooKeeper. Последовательный номер, выданный сервером ZooKeeper в ответе на успешно выполненный запрос (`0` — запрос не был выполнен, возвращена ошибка или клиент ZooKeeper не знает, был ли выполнен запрос). +- `error` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — код ошибки. Может иметь много значений, здесь приведены только некоторые из них: + - `ZOK` — запрос успешно выполнен. + - `ZCONNECTIONLOSS` — соединение разорвано. + - `ZOPERATIONTIMEOUT` — истекло время ожидания выполнения запроса. + - `ZSESSIONEXPIRED` — истекло время сессии. + - `NULL` — выполнен запрос. +- `watch_type` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — тип события `watch` (для ответов на запрос при `op_num` = `Watch`), для остальных ответов: `NULL`. +- `watch_state` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — статус события `watch` (для ответов на запрос при `op_num` = `Watch`), для остальных ответов: `NULL`. +- `path_created` ([String](../../sql-reference/data-types/string.md)) — путь к созданному узлу ZooKeeper (для ответов на запрос `CREATE`). Может отличаться от `path`, если узел создается как `sequential`. +- `stat_czxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор транзакции, в результате которой был создан узел ZooKeeper. +- `stat_mzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор транзакции, которая последней модифицировала узел ZooKeeper. +- `stat_pzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор транзакции, которая последней модифицировала дочерние узлы ZooKeeper. +- `stat_version` ([Int32](../../sql-reference/data-types/int-uint.md)) — количество изменений в данных узла ZooKeeper. +- `stat_cversion` ([Int32](../../sql-reference/data-types/int-uint.md)) — количество изменений в дочерних узлах ZooKeeper. +- `stat_dataLength` ([Int32](../../sql-reference/data-types/int-uint.md)) — длина поля данных узла ZooKeeper. +- `stat_numChildren` ([Int32](../../sql-reference/data-types/int-uint.md)) — количество дочерних узлов ZooKeeper. +- `children` ([Array(String)](../../sql-reference/data-types/array.md)) — список дочерних узлов ZooKeeper (для ответов на запрос `LIST`). + +**Пример** + +Запрос: + +``` sql +SELECT * FROM system.zookeeper_log WHERE (session_id = '106662742089334927') AND (xid = '10858') FORMAT Vertical; +``` + +Результат: + +``` text +Row 1: +────── +type: Request +event_date: 2021-08-09 +event_time: 2021-08-09 21:38:30.291792 +address: :: +port: 2181 +session_id: 106662742089334927 +xid: 10858 +has_watch: 1 +op_num: List +path: /clickhouse/task_queue/ddl +data: +is_ephemeral: 0 +is_sequential: 0 +version: ᴺᵁᴸᴸ +requests_size: 0 +request_idx: 0 +zxid: 0 +error: ᴺᵁᴸᴸ +watch_type: ᴺᵁᴸᴸ +watch_state: ᴺᵁᴸᴸ +path_created: +stat_czxid: 0 +stat_mzxid: 0 +stat_pzxid: 0 +stat_version: 0 +stat_cversion: 0 +stat_dataLength: 0 +stat_numChildren: 0 +children: [] + +Row 2: +────── +type: Response +event_date: 2021-08-09 +event_time: 2021-08-09 21:38:30.292086 +address: :: +port: 2181 +session_id: 106662742089334927 +xid: 10858 +has_watch: 1 +op_num: List +path: /clickhouse/task_queue/ddl +data: +is_ephemeral: 0 +is_sequential: 0 +version: ᴺᵁᴸᴸ +requests_size: 0 +request_idx: 0 +zxid: 16926267 +error: ZOK +watch_type: ᴺᵁᴸᴸ +watch_state: ᴺᵁᴸᴸ +path_created: +stat_czxid: 16925469 +stat_mzxid: 16925469 +stat_pzxid: 16926179 +stat_version: 0 +stat_cversion: 7 +stat_dataLength: 0 +stat_numChildren: 7 +children: ['query-0000000006','query-0000000005','query-0000000004','query-0000000003','query-0000000002','query-0000000001','query-0000000000'] +``` + +**См. также** + +- [ZooKeeper](../../operations/tips.md#zookeeper) +- [Руководство по ZooKeeper](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html) diff --git a/docs/ru/sql-reference/functions/geo/h3.md b/docs/ru/sql-reference/functions/geo/h3.md index 088814c4d7d..bc47ca72a39 100644 --- a/docs/ru/sql-reference/functions/geo/h3.md +++ b/docs/ru/sql-reference/functions/geo/h3.md @@ -193,6 +193,40 @@ SELECT geoToH3(37.79506683, 55.71290588, 15) as h3Index; └────────────────────┘ ``` +## h3ToGeo {#h3togeo} + +Возвращает географические координаты долготы и широты, соответствующие указанному [H3](#h3index)-индексу. + +**Синтаксис** + +``` sql +h3ToGeo(h3Index) +``` + +**Аргументы** + +- `h3Index` — [H3](#h3index)-индекс. [UInt64](../../../sql-reference/data-types/int-uint.md). + +**Возвращаемые значения** + +- кортеж из двух значений: `tuple(lon,lat)`, где `lon` — долгота [Float64](../../../sql-reference/data-types/float.md), `lat` — широта [Float64](../../../sql-reference/data-types/float.md). + +**Пример** + +Запрос: + +``` sql +SELECT h3ToGeo(644325524701193974) coordinates; +``` + +Результат: + +``` text +┌─coordinates───────────────────────────┐ +│ (37.79506616830252,55.71290243145668) │ +└───────────────────────────────────────┘ +``` + ## h3kRing {#h3kring} Возвращает [H3](#h3index)-индексы шестигранников в радиусе `k` от данного в произвольном порядке. diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index 634343d112f..e123f506d46 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -36,6 +36,7 @@ toc_title: SYSTEM - [START REPLICATION QUEUES](#query_language-system-start-replication-queues) - [SYNC REPLICA](#query_language-system-sync-replica) - [RESTART REPLICA](#query_language-system-restart-replica) +- [RESTORE REPLICA](#query_language-system-restore-replica) - [RESTART REPLICAS](#query_language-system-restart-replicas) ## RELOAD EMBEDDED DICTIONARIES] {#query_language-system-reload-emdedded-dictionaries} @@ -287,13 +288,66 @@ SYSTEM SYNC REPLICA [db.]replicated_merge_tree_family_table_name ### RESTART REPLICA {#query_language-system-restart-replica} -Реинициализация состояния Zookeeper-сессий для таблицы семейства `ReplicatedMergeTree`. Сравнивает текущее состояние с тем, что хранится в Zookeeper, как источник правды, и добавляет задачи в очередь репликации в Zookeeper, если необходимо. -Инициализация очереди репликации на основе данных ZooKeeper происходит так же, как при attach table. На короткое время таблица станет недоступной для любых операций. +Реинициализирует состояние сессий Zookeeper для таблицы семейства `ReplicatedMergeTree`. Сравнивает текущее состояние с состоянием в Zookeeper (как с эталоном) и при необходимости добавляет задачи в очередь репликации в Zookeeper. +Инициализация очереди репликации на основе данных ZooKeeper происходит так же, как при `ATTACH TABLE`. Некоторое время таблица будет недоступна для любых операций. ``` sql SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name ``` +### RESTORE REPLICA {#query_language-system-restore-replica} + +Восстанавливает реплику, если метаданные в Zookeeper потеряны, но сами данные возможно существуют. + +Работает только с таблицами семейства `ReplicatedMergeTree` и только если таблица находится в readonly-режиме. + +Запрос можно выполнить если: + + - потерян корневой путь ZooKeeper `/`; + - потерян путь реплик `/replicas`; + - потерян путь конкретной реплики `/replicas/replica_name/`. + +К реплике прикрепляются локально найденные куски, информация о них отправляется в Zookeeper. +Если присутствующие в реплике до потери метаданных данные не устарели, они не скачиваются повторно с других реплик. Поэтому восстановление реплики не означает повторную загрузку всех данных по сети. + +!!! warning "Предупреждение" + Потерянные данные в любых состояниях перемещаются в папку `detached/`. Куски, активные до потери данных (находившиеся в состоянии Committed), прикрепляются. + +**Синтаксис** + +```sql +SYSTEM RESTORE REPLICA [db.]replicated_merge_tree_family_table_name [ON CLUSTER cluster_name] +``` + +Альтернативный синтаксис: + +```sql +SYSTEM RESTORE REPLICA [ON CLUSTER cluster_name] [db.]replicated_merge_tree_family_table_name +``` + +**Пример** + +Создание таблицы на нескольких серверах. После потери корневого пути реплики таблица будет прикреплена только для чтения, так как метаданные отсутствуют. Последний запрос необходимо выполнить на каждой реплике. + +```sql +CREATE TABLE test(n UInt32) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/', '{replica}') +ORDER BY n PARTITION BY n % 10; + +INSERT INTO test SELECT * FROM numbers(1000); + +-- zookeeper_delete_path("/clickhouse/tables/test", recursive=True) <- root loss. + +SYSTEM RESTART REPLICA test; +SYSTEM RESTORE REPLICA test; +``` + +Альтернативный способ: + +```sql +SYSTEM RESTORE REPLICA test ON CLUSTER cluster; +``` + ### RESTART REPLICAS {#query_language-system-restart-replicas} Реинициализация состояния ZooKeeper-сессий для всех `ReplicatedMergeTree` таблиц. Сравнивает текущее состояние реплики с тем, что хранится в ZooKeeper, как c источником правды, и добавляет задачи в очередь репликации в ZooKeeper, если необходимо. diff --git a/docs/ru/sql-reference/table-functions/cluster.md b/docs/ru/sql-reference/table-functions/cluster.md index 1a087971afe..a9cff862293 100644 --- a/docs/ru/sql-reference/table-functions/cluster.md +++ b/docs/ru/sql-reference/table-functions/cluster.md @@ -5,22 +5,44 @@ toc_title: cluster # cluster, clusterAllReplicas {#cluster-clusterallreplicas} -Позволяет обратиться ко всем серверам существующего кластера, который присутствует в таблице `system.clusters` и сконфигурирован в секцци `remote_servers` без создания таблицы типа `Distributed`. -`clusterAllReplicas` - работает также как `cluster` но каждая реплика в кластере будет использована как отдельный шард/отдельное соединение. +Позволяет обратиться ко всем шардам существующего кластера, который сконфигурирован в секции `remote_servers` без создания таблицы типа [Distributed](../../engines/table-engines/special/distributed.md). В запросе используется одна реплика каждого шарда. +Функция `clusterAllReplicas` работает также как `cluster`, но каждая реплика в кластере используется как отдельный шард/отдельное соединение. -Сигнатуры: +!!! note "Примечание" + Все доступные кластеры перечислены в таблице [system.clusters](../../operations/system-tables/clusters.md). + +**Синтаксис** ``` sql -cluster('cluster_name', db.table) -cluster('cluster_name', db, table) -clusterAllReplicas('cluster_name', db.table) -clusterAllReplicas('cluster_name', db, table) +cluster('cluster_name', db.table[, sharding_key]) +cluster('cluster_name', db, table[, sharding_key]) +clusterAllReplicas('cluster_name', db.table[, sharding_key]) +clusterAllReplicas('cluster_name', db, table[, sharding_key]) +``` +**Аргументы** + +- `cluster_name` – имя кластера, который обозначает подмножество адресов и параметров подключения к удаленным и локальным серверам, входящим в кластер. +- `db.table` или `db`, `table` - имя базы данных и таблицы. +- `sharding_key` - ключ шардирования. Необязательный аргумент. Указывается, если данные добавляются более чем в один шард кластера. + +**Возвращаемое значение** + +Набор данных из кластеров. + +**Использование макросов** + +`cluster_name` может содержать макрос — подстановку в фигурных скобках. Эта подстановка заменяется на соответствующее значение из секции [macros](../../operations/server-configuration-parameters/settings.md#macros) конфигурационного файла . + +Пример: + +```sql +SELECT * FROM cluster('{cluster}', default.example_table); ``` -`cluster_name` – имя кластера, который обязан присутствовать в таблице `system.clusters` и обозначает подмножество адресов и параметров подключения к удаленным и локальным серверам, входящим в кластер. +**Использование и рекомендации** -Использование табличных функций `cluster` и `clusterAllReplicas` менее оптимальное чем создание таблицы типа `Distributed`, поскольку в этом случае соединение с сервером переустанавливается на каждый запрос. При обработке большого количества запросов, всегда создавайте `Distributed` таблицу заранее и не используйте табличные функции `cluster` и `clusterAllReplicas`. +Использование табличных функций `cluster` и `clusterAllReplicas` менее оптимально, чем создание таблицы типа `Distributed`, поскольку в этом случае при каждом новом запросе устанавливается новое соединение с сервером. При обработке большого количества запросов всегда создавайте `Distributed` таблицу заранее и не используйте табличные функции `cluster` и `clusterAllReplicas`. Табличные функции `cluster` and `clusterAllReplicas` могут быть полезны в следующих случаях: @@ -30,7 +52,7 @@ clusterAllReplicas('cluster_name', db, table) Настройки соединения `user`, `password`, `host`, `post`, `compression`, `secure` берутся из секции `` файлов конфигурации. См. подробности в разделе [Distributed](../../engines/table-engines/special/distributed.md) -**See Also** +**См. также** - [skip_unavailable_shards](../../operations/settings/settings.md#settings-skip_unavailable_shards) - [load_balancing](../../operations/settings/settings.md#settings-load_balancing) diff --git a/docs/zh/operations/troubleshooting.md b/docs/zh/operations/troubleshooting.md index 56b18aa1307..8d1defd6366 100644 --- a/docs/zh/operations/troubleshooting.md +++ b/docs/zh/operations/troubleshooting.md @@ -26,7 +26,7 @@ toc_title: "常见问题" ### 服务器未运行 {#server-is-not-running} -**检查服务器是否运行nnig** +**检查服务器是否正在运行** 命令: diff --git a/docs/zh/sql-reference/functions/ym-dict-functions.md b/docs/zh/sql-reference/functions/ym-dict-functions.md index 429105084dd..2f2b2f80d25 100644 --- a/docs/zh/sql-reference/functions/ym-dict-functions.md +++ b/docs/zh/sql-reference/functions/ym-dict-functions.md @@ -1,8 +1,8 @@ -# 功能与Yandex的工作。梅特里卡词典 {#functions-for-working-with-yandex-metrica-dictionaries} +# 使用 Yandex.Metrica 字典函数 {#functions-for-working-with-yandex-metrica-dictionaries} -为了使下面的功能正常工作，服务器配置必须指定获取所有Yandex的路径和地址。梅特里卡字典. 字典在任何这些函数的第一次调用时加载。 如果无法加载引用列表，则会引发异常。 +为了使下面的功能正常工作，服务器配置必须指定获取所有 Yandex.Metrica 字典的路径和地址。Yandex.Metrica 字典在任何这些函数的第一次调用时加载。 如果无法加载引用列表，则会引发异常。 -For information about creating reference lists, see the section «Dictionaries». +有关创建引用列表的信息，请参阅 «字典» 部分. ## 多个地理基 {#multiple-geobases} @@ -17,18 +17,18 @@ ClickHouse支持同时使用多个备选地理基（区域层次结构），以 所有字典都在运行时重新加载（每隔一定数量的秒重新加载一次，如builtin_dictionaries_reload_interval config参数中定义，或默认情况下每小时一次）。 但是，可用字典列表在服务器启动时定义一次。 -All functions for working with regions have an optional argument at the end – the dictionary key. It is referred to as the geobase. +所有处理区域的函数都在末尾有一个可选参数—字典键。它被称为地基。 示例: - regionToCountry(RegionID) – Uses the default dictionary: /opt/geo/regions_hierarchy.txt - regionToCountry(RegionID, '') – Uses the default dictionary: /opt/geo/regions_hierarchy.txt - regionToCountry(RegionID, 'ua') – Uses the dictionary for the 'ua' key: /opt/geo/regions_hierarchy_ua.txt + regionToCountry(RegionID) – 使用默认路径: /opt/geo/regions_hierarchy.txt + regionToCountry(RegionID, '') – 使用默认路径: /opt/geo/regions_hierarchy.txt + regionToCountry(RegionID, 'ua') – 使用字典中的'ua' 键: /opt/geo/regions_hierarchy_ua.txt -### ﾂ环板(ｮﾂ嘉ｯﾂ偲青regionｼﾂ氾ｶﾂ鉄ﾂ工ﾂ渉\]) {#regiontocityid-geobase} +### regionToCity(id[, geobase]) {#regiontocityid-geobase} -Accepts a UInt32 number – the region ID from the Yandex geobase. If this region is a city or part of a city, it returns the region ID for the appropriate city. Otherwise, returns 0. +从 Yandex geobase 接收一个 UInt32 数字类型的区域ID 。如果该区域是一个城市或城市的一部分，它将返回相应城市的区域ID。否则,返回0。 -### 虏茅驴麓卤戮碌禄路戮鲁拢\]) {#regiontoareaid-geobase} +### regionToArea(id[, geobase]) {#regiontoareaid-geobase} 将区域转换为区域（地理数据库中的类型5）。 在所有其他方式，这个功能是一样的 ‘regionToCity’. @@ -84,36 +84,58 @@ LIMIT 15 │ Federation of Bosnia and Herzegovina │ └──────────────────────────────────────────────────────────┘ -### 虏茅驴麓卤戮碌禄路戮鲁拢(陆毛隆隆(803)888-8325\]) {#regiontocountryid-geobase} +### regionToCountry(id[, geobase]) {#regiontocountryid-geobase} 将区域转换为国家。 在所有其他方式，这个功能是一样的 ‘regionToCity’. 示例: `regionToCountry(toUInt32(213)) = 225` 转换莫斯科（213）到俄罗斯（225）。 -### 掳胫((禄脢鹿脷露胫鲁隆鹿((酶-11-16""\[脪陆,ase\]) {#regiontocontinentid-geobase} +### regionToContinent(id[, geobase]) {#regiontocontinentid-geobase} 将区域转换为大陆。 在所有其他方式，这个功能是一样的 ‘regionToCity’. 示例: `regionToContinent(toUInt32(213)) = 10001` 将莫斯科（213）转换为欧亚大陆（10001）。 -### ﾂ环板(ｮﾂ嘉ｯﾂ偲青regionｬﾂ静ｬﾂ青ｻﾂ催ｬﾂ渉\]) {#regiontopopulationid-geobase} +### regionToTopContinent (#regiontotopcontinent) {#regiontotopcontinent-regiontotopcontinent} + +查找该区域层次结构中最高的大陆。 + +**语法** + +``` sql +regionToTopContinent(id[, geobase]) +``` + +**参数** + +- `id` — Yandex geobase 的区域 ID. [UInt32](../../sql-reference/data-types/int-uint.md). +- `geobase` — 字典的建. 参阅 [Multiple Geobases](#multiple-geobases). [String](../../sql-reference/data-types/string.md). 可选. + +**返回值** + +- 顶级大陆的标识符(当您在区域层次结构中攀爬时，是后者)。 +- 0，如果没有。 + +类型: `UInt32`. + +### regionToPopulation(id\[, geobase\]) {#regiontopopulationid-geobase} 获取区域的人口。 -The population can be recorded in files with the geobase. See the section «External dictionaries». +人口可以记录在文件与地球基。请参阅«外部词典»部分。 如果没有为该区域记录人口，则返回0。 在Yandex地理数据库中，可能会为子区域记录人口，但不会为父区域记录人口。 ### regionIn(lhs,rhs\[,地理数据库\]) {#regioninlhs-rhs-geobase} 检查是否 ‘lhs’ 属于一个区域 ‘rhs’ 区域。 如果属于UInt8，则返回等于1的数字，如果不属于则返回0。 -The relationship is reflexive – any region also belongs to itself. +这种关系是反射的——任何地区也属于自己。 -### ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人\]) {#regionhierarchyid-geobase} +### regionHierarchy(id\[, geobase\]) {#regionhierarchyid-geobase} -Accepts a UInt32 number – the region ID from the Yandex geobase. Returns an array of region IDs consisting of the passed region and all parents along the chain. +从 Yandex geobase 接收一个 UInt32 数字类型的区域ID。返回一个区域ID数组，由传递的区域和链上的所有父节点组成。 示例: `regionHierarchy(toUInt32(213)) = [213,1,3,225,10001,10000]`. -### 地区名称(id\[,郎\]) {#regiontonameid-lang} +### regionToName(id\[, lang\]) {#regiontonameid-lang} -Accepts a UInt32 number – the region ID from the Yandex geobase. A string with the name of the language can be passed as a second argument. Supported languages are: ru, en, ua, uk, by, kz, tr. If the second argument is omitted, the language ‘ru’ is used. If the language is not supported, an exception is thrown. Returns a string – the name of the region in the corresponding language. If the region with the specified ID doesn’t exist, an empty string is returned. +从 Yandex geobase 接收一个 UInt32 数字类型的区域ID。带有语言名称的字符串可以作为第二个参数传递。支持的语言有:ru, en, ua, uk, by, kz, tr。如果省略第二个参数，则使用' ru '语言。如果不支持该语言，则抛出异常。返回一个字符串-对应语言的区域名称。如果指定ID的区域不存在，则返回一个空字符串。 `ua` 和 `uk` 都意味着乌克兰。 diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 14442167042..65e245750b3 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -2,6 +2,7 @@ #include "Common/MemoryTracker.h" #include "Columns/ColumnsNumber.h" #include "ConnectionParameters.h" +#include "IO/CompressionMethod.h" #include "QueryFuzzer.h" #include "Suggest.h" #include "TestHint.h" @@ -128,6 +129,7 @@ namespace ErrorCodes extern const int UNRECOGNIZED_ARGUMENTS; extern const int SYNTAX_ERROR; extern const int TOO_DEEP_RECURSION; + extern const int AUTHENTICATION_FAILED; } @@ -772,31 +774,50 @@ private: << connection_parameters.host << ":" << connection_parameters.port << (!connection_parameters.user.empty() ? " as user " + connection_parameters.user : "") << "." << std::endl; - connection = std::make_unique( - connection_parameters.host, - connection_parameters.port, - connection_parameters.default_database, - connection_parameters.user, - connection_parameters.password, - "", /* cluster */ - "", /* cluster_secret */ - "client", - connection_parameters.compression, - connection_parameters.security); - String server_name; UInt64 server_version_major = 0; UInt64 server_version_minor = 0; UInt64 server_version_patch = 0; - if (max_client_network_bandwidth) + try { - ThrottlerPtr throttler = std::make_shared(max_client_network_bandwidth, 0, ""); - connection->setThrottler(throttler); - } + connection = std::make_unique( + connection_parameters.host, + connection_parameters.port, + connection_parameters.default_database, + connection_parameters.user, + connection_parameters.password, + "", /* cluster */ + "", /* cluster_secret */ + "client", + connection_parameters.compression, + connection_parameters.security); - connection->getServerVersion( - connection_parameters.timeouts, server_name, server_version_major, server_version_minor, server_version_patch, server_revision); + if (max_client_network_bandwidth) + { + ThrottlerPtr throttler = std::make_shared(max_client_network_bandwidth, 0, ""); + connection->setThrottler(throttler); + } + + connection->getServerVersion( + connection_parameters.timeouts, server_name, server_version_major, server_version_minor, server_version_patch, server_revision); + } + catch (const Exception & e) + { + /// It is typical when users install ClickHouse, type some password and instantly forget it. + if ((connection_parameters.user.empty() || connection_parameters.user == "default") + && e.code() == DB::ErrorCodes::AUTHENTICATION_FAILED) + { + std::cerr << std::endl + << "If you have installed ClickHouse and forgot password you can reset it in the configuration file." << std::endl + << "The password for default user is typically located at /etc/clickhouse-server/users.d/default-password.xml" << std::endl + << "and deleting this file will reset the password." << std::endl + << "See also /etc/clickhouse-server/users.xml on the server where ClickHouse is installed." << std::endl + << std::endl; + } + + throw; + } server_version = toString(server_version_major) + "." + toString(server_version_minor) + "." + toString(server_version_patch); @@ -1823,7 +1844,7 @@ private: void processInsertQuery() { const auto parsed_insert_query = parsed_query->as(); - if (!parsed_insert_query.data && (is_interactive || (!stdin_is_a_tty && std_in.eof()))) + if ((!parsed_insert_query.data && !parsed_insert_query.infile) && (is_interactive || (!stdin_is_a_tty && std_in.eof()))) throw Exception("No data to insert", ErrorCodes::NO_DATA_TO_INSERT); connection->sendQuery( @@ -1894,7 +1915,24 @@ private: if (!parsed_insert_query) return; - if (parsed_insert_query->data) + if (parsed_insert_query->infile) + { + const auto & in_file_node = parsed_insert_query->infile->as(); + const auto in_file = in_file_node.value.safeGet(); + + auto in_buffer = wrapReadBufferWithCompressionMethod(std::make_unique(in_file), chooseCompressionMethod(in_file, "")); + + try + { + sendDataFrom(*in_buffer, sample, columns_description); + } + catch (Exception & e) + { + e.addMessage("data for INSERT was parsed from file"); + throw; + } + } + else if (parsed_insert_query->data) { /// Send data contained in the query. ReadBufferFromMemory data_in(parsed_insert_query->data, parsed_insert_query->end - parsed_insert_query->data); diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index 60695cbfeeb..4d01a523853 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -326,6 +327,8 @@ int Keeper::main(const std::vector & /*args*/) } } + DB::ServerUUID::load(path + "/uuid", log); + const Settings & settings = global_context->getSettingsRef(); GlobalThreadPool::initialize(config().getUInt("max_thread_pool_size", 100)); diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index e256338a538..957bda4d75c 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -374,14 +375,13 @@ void LocalServer::processQueries() if (!parse_res.second) throw Exception("Cannot parse and execute the following part of query: " + String(parse_res.first), ErrorCodes::SYNTAX_ERROR); - /// we can't mutate global global_context (can lead to races, as it was already passed to some background threads) - /// so we can't reuse it safely as a query context and need a copy here - auto context = Context::createCopy(global_context); + /// Authenticate and create a context to execute queries. + Session session{global_context, ClientInfo::Interface::TCP}; + session.authenticate("default", "", Poco::Net::SocketAddress{}); - context->makeSessionContext(); - context->makeQueryContext(); - - context->setUser("default", "", Poco::Net::SocketAddress{}); + /// Use the same context for all queries. + auto context = session.makeQueryContext(); + context->makeSessionContext(); /// initial_create_query requires a session context to be set. context->setCurrentQueryId(""); applyCmdSettings(context); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 5520f920823..7c1527bf4b0 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -79,7 +80,6 @@ #include #include - #if !defined(ARCADIA_BUILD) # include "config_core.h" # include "Common/config_version.h" @@ -146,7 +146,6 @@ static bool jemallocOptionEnabled(const char *name) static bool jemallocOptionEnabled(const char *) { return 0; } #endif - int mainEntryClickHouseServer(int argc, char ** argv) { DB::Server app; @@ -667,13 +666,14 @@ if (ThreadFuzzer::instance().isEffective()) global_context->setRemoteHostFilter(config()); - std::string path = getCanonicalPath(config().getString("path", DBMS_DEFAULT_PATH)); + std::string path_str = getCanonicalPath(config().getString("path", DBMS_DEFAULT_PATH)); + fs::path path = path_str; std::string default_database = config().getString("default_database", "default"); /// Check that the process user id matches the owner of the data. const auto effective_user_id = geteuid(); struct stat statbuf; - if (stat(path.c_str(), &statbuf) == 0 && effective_user_id != statbuf.st_uid) + if (stat(path_str.c_str(), &statbuf) == 0 && effective_user_id != statbuf.st_uid) { const auto effective_user = getUserName(effective_user_id); const auto data_owner = getUserName(statbuf.st_uid); @@ -690,9 +690,11 @@ if (ThreadFuzzer::instance().isEffective()) } } - global_context->setPath(path); + global_context->setPath(path_str); - StatusFile status{path + "status", StatusFile::write_full_info}; + StatusFile status{path / "status", StatusFile::write_full_info}; + + DB::ServerUUID::load(path / "uuid", log); /// Try to increase limit on number of open files. { @@ -726,7 +728,7 @@ if (ThreadFuzzer::instance().isEffective()) /// Storage with temporary data for processing of heavy queries. { - std::string tmp_path = config().getString("tmp_path", path + "tmp/"); + std::string tmp_path = config().getString("tmp_path", path / "tmp/"); std::string tmp_policy = config().getString("tmp_policy", ""); const VolumePtr & volume = global_context->setTemporaryStorage(tmp_path, tmp_policy); for (const DiskPtr & disk : volume->getDisks()) @@ -738,7 +740,7 @@ if (ThreadFuzzer::instance().isEffective()) * Examples: do repair of local data; clone all replicated tables from replica. */ { - auto flags_path = fs::path(path) / "flags/"; + auto flags_path = path / "flags/"; fs::create_directories(flags_path); global_context->setFlagsPath(flags_path); } @@ -747,29 +749,29 @@ if (ThreadFuzzer::instance().isEffective()) */ { - std::string user_files_path = config().getString("user_files_path", fs::path(path) / "user_files/"); + std::string user_files_path = config().getString("user_files_path", path / "user_files/"); global_context->setUserFilesPath(user_files_path); fs::create_directories(user_files_path); } { - std::string dictionaries_lib_path = config().getString("dictionaries_lib_path", fs::path(path) / "dictionaries_lib/"); + std::string dictionaries_lib_path = config().getString("dictionaries_lib_path", path / "dictionaries_lib/"); global_context->setDictionariesLibPath(dictionaries_lib_path); fs::create_directories(dictionaries_lib_path); } /// top_level_domains_lists { - const std::string & top_level_domains_path = config().getString("top_level_domains_path", fs::path(path) / "top_level_domains/"); + const std::string & top_level_domains_path = config().getString("top_level_domains_path", path / "top_level_domains/"); TLDListsHolder::getInstance().parseConfig(fs::path(top_level_domains_path) / "", config()); } { - fs::create_directories(fs::path(path) / "data/"); - fs::create_directories(fs::path(path) / "metadata/"); + fs::create_directories(path / "data/"); + fs::create_directories(path / "metadata/"); /// Directory with metadata of tables, which was marked as dropped by Atomic database - fs::create_directories(fs::path(path) / "metadata_dropped/"); + fs::create_directories(path / "metadata_dropped/"); } if (config().has("interserver_http_port") && config().has("interserver_https_port")) @@ -952,7 +954,7 @@ if (ThreadFuzzer::instance().isEffective()) #endif /// Set path for format schema files - fs::path format_schema_path(config().getString("format_schema_path", fs::path(path) / "format_schemas/")); + fs::path format_schema_path(config().getString("format_schema_path", path / "format_schemas/")); global_context->setFormatSchemaPath(format_schema_path); fs::create_directories(format_schema_path); @@ -1088,7 +1090,7 @@ if (ThreadFuzzer::instance().isEffective()) /// system logs may copy global context. global_context->setCurrentDatabaseNameInGlobalContext(default_database); - LOG_INFO(log, "Loading metadata from {}", path); + LOG_INFO(log, "Loading metadata from {}", path_str); try { @@ -1428,7 +1430,6 @@ if (ThreadFuzzer::instance().isEffective()) /// Must be done after initialization of `servers`, because async_metrics will access `servers` variable from its thread. async_metrics.start(); - global_context->enableNamedSessions(); { String level_str = config().getString("text_log.level", ""); diff --git a/src/Access/ContextAccess.h b/src/Access/ContextAccess.h index 70145b0a3ef..cde69471800 100644 --- a/src/Access/ContextAccess.h +++ b/src/Access/ContextAccess.h @@ -70,6 +70,7 @@ public: /// Returns the current user. The function can return nullptr. UserPtr getUser() const; String getUserName() const; + std::optional getUserID() const { return getParams().user_id; } /// Returns information about current and enabled roles. std::shared_ptr getRolesInfo() const; diff --git a/src/Access/Credentials.h b/src/Access/Credentials.h index 5e9fd1589e0..256ed3853ab 100644 --- a/src/Access/Credentials.h +++ b/src/Access/Credentials.h @@ -26,6 +26,8 @@ protected: String user_name; }; +/// Does not check the password/credentials and that the specified host is allowed. +/// (Used only internally in cluster, if the secret matches) class AlwaysAllowCredentials : public Credentials { diff --git a/src/AggregateFunctions/UniqVariadicHash.h b/src/AggregateFunctions/UniqVariadicHash.h index b3607a63285..94f54a7a059 100644 --- a/src/AggregateFunctions/UniqVariadicHash.h +++ b/src/AggregateFunctions/UniqVariadicHash.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB diff --git a/src/Bridge/IBridgeHelper.cpp b/src/Bridge/IBridgeHelper.cpp index 5c884a2ca3d..984641be3d2 100644 --- a/src/Bridge/IBridgeHelper.cpp +++ b/src/Bridge/IBridgeHelper.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace fs = std::filesystem; diff --git a/src/Client/MultiplexedConnections.cpp b/src/Client/MultiplexedConnections.cpp index fe3879fdd30..a4e1eb09253 100644 --- a/src/Client/MultiplexedConnections.cpp +++ b/src/Client/MultiplexedConnections.cpp @@ -373,7 +373,9 @@ MultiplexedConnections::ReplicaState & MultiplexedConnections::getReplicaForRead except_list, is_draining ? drain_timeout : receive_timeout); - if (n == 0) + /// We treat any error as timeout for simplicity. + /// And we also check if read_list is still empty just in case. + if (n <= 0 || read_list.empty()) { auto err_msg = fmt::format("Timeout exceeded while reading from {}", dumpAddressesUnlocked()); for (ReplicaState & state : replica_states) @@ -389,9 +391,7 @@ MultiplexedConnections::ReplicaState & MultiplexedConnections::getReplicaForRead } } - /// TODO Absolutely wrong code: read_list could be empty; motivation of rand is unclear. - /// This code path is disabled by default. - + /// TODO Motivation of rand is unclear. auto & socket = read_list[thread_local_rng() % read_list.size()]; if (fd_to_replica_state_idx.empty()) { diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 1601fb1ff94..7c7c6a4d9db 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -565,7 +565,7 @@ void ColumnArray::expand(const IColumn::Filter & mask, bool inverted) while (index >= 0) { offsets_data[index] = last_offset; - if (mask[index] ^ inverted) + if (!!mask[index] ^ inverted) { if (from < 0) throw Exception("Too many bytes in mask", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index e818e974493..ce4f11a38d6 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -354,7 +354,7 @@ void ColumnFixedString::expand(const IColumn::Filter & mask, bool inverted) chars.resize_fill(mask.size() * n, 0); while (index >= 0) { - if (mask[index] ^ inverted) + if (!!mask[index] ^ inverted) { if (from < 0) throw Exception("Too many bytes in mask", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 2f5903abfc1..d02fa66baf4 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -178,7 +178,7 @@ void ColumnString::expand(const IColumn::Filter & mask, bool inverted) while (index >= 0) { offsets_data[index] = last_offset; - if (mask[index] ^ inverted) + if (!!mask[index] ^ inverted) { if (from < 0) throw Exception("Too many bytes in mask", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Columns/ColumnTuple.h b/src/Columns/ColumnTuple.h index eb751d59407..54723239a06 100644 --- a/src/Columns/ColumnTuple.h +++ b/src/Columns/ColumnTuple.h @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace DB diff --git a/src/Columns/ColumnUnique.h b/src/Columns/ColumnUnique.h index 8ca4baff7c7..bfa80b5e3b2 100644 --- a/src/Columns/ColumnUnique.h +++ b/src/Columns/ColumnUnique.h @@ -304,7 +304,7 @@ size_t ColumnUnique::uniqueInsert(const Field & x) if (x.getType() == Field::Types::Null) return getNullValueIndex(); - if (isNumeric()) + if (valuesHaveFixedSize()) return uniqueInsertData(&x.reinterpret(), size_of_value_if_fixed); auto & val = x.get(); diff --git a/src/Columns/MaskOperations.cpp b/src/Columns/MaskOperations.cpp index 6852c895d51..759d0af7127 100644 --- a/src/Columns/MaskOperations.cpp +++ b/src/Columns/MaskOperations.cpp @@ -26,7 +26,7 @@ void expandDataByMask(PaddedPODArray & data, const PaddedPODArray & ma data.resize(mask.size()); while (index >= 0) { - if (mask[index] ^ inverted) + if (!!mask[index] ^ inverted) { if (from < 0) throw Exception("Too many bytes in mask", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index 8ef85d82a1d..4cb1df3ff65 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -191,10 +191,11 @@ void ThreadPoolImpl::wait() template ThreadPoolImpl::~ThreadPoolImpl() { + /// Note: should not use logger from here, + /// because it can be an instance of GlobalThreadPool that is a global variable + /// and the destruction order of global variables is unspecified. + finalize(); - /// wait() hadn't been called, log exception at least. - if (first_exception) - DB::tryLogException(first_exception, __PRETTY_FUNCTION__); } template @@ -273,21 +274,11 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ } catch (...) { - ALLOW_ALLOCATIONS_IN_SCOPE; - /// job should be reset before decrementing scheduled_jobs to /// ensure that the Job destroyed before wait() returns. job = {}; { - /// In case thread pool will not be terminated on exception - /// (this is the case for GlobalThreadPool), - /// than first_exception may be overwritten and got lost, - /// and this usually is an error, since this will finish the thread, - /// and for this the caller may not be ready. - if (!shutdown_on_exception) - DB::tryLogException(std::current_exception(), __PRETTY_FUNCTION__); - std::unique_lock lock(mutex); if (!first_exception) first_exception = std::current_exception(); // NOLINT diff --git a/src/Common/examples/CMakeLists.txt b/src/Common/examples/CMakeLists.txt index b154d5965ca..64d28fec5c2 100644 --- a/src/Common/examples/CMakeLists.txt +++ b/src/Common/examples/CMakeLists.txt @@ -80,8 +80,3 @@ target_link_libraries (average PRIVATE clickhouse_common_io) add_executable (shell_command_inout shell_command_inout.cpp) target_link_libraries (shell_command_inout PRIVATE clickhouse_common_io) - -if (ENABLE_FUZZING) - add_executable(YAML_fuzzer YAML_fuzzer.cpp ${SRCS}) - target_link_libraries(YAML_fuzzer PRIVATE clickhouse_parsers ${LIB_FUZZING_ENGINE}) -endif () diff --git a/src/Common/examples/YAML_fuzzer.cpp b/src/Common/examples/YAML_fuzzer.cpp deleted file mode 100644 index f4b570e0e0a..00000000000 --- a/src/Common/examples/YAML_fuzzer.cpp +++ /dev/null @@ -1,39 +0,0 @@ -#include -#include -#include -#include -#include -#include - -extern "C" int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size) -{ - /// How to test: - /// build ClickHouse with YAML_fuzzer.cpp - /// ./YAML_fuzzer YAML_CORPUS - /// where YAML_CORPUS is a directory with different YAML configs for libfuzzer - char file_name[L_tmpnam]; - if (!std::tmpnam(file_name)) - { - std::cerr << "Cannot create temp file!\n"; - return 1; - } - std::string input = std::string(reinterpret_cast(data), size); - DB::YAMLParser parser; - - { - std::ofstream temp_file(file_name); - temp_file << input; - } - - try - { - DB::YAMLParser::parse(std::string(file_name)); - } - catch (...) - { - std::cerr << "YAML_fuzzer failed: " << getCurrentExceptionMessage() << std::endl; - return 1; - } - return 0; -} - diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp index 4855500b776..9c3db0f3e30 100644 --- a/src/Common/filesystemHelpers.cpp +++ b/src/Common/filesystemHelpers.cpp @@ -122,6 +122,24 @@ bool pathStartsWith(const std::filesystem::path & path, const std::filesystem::p return path_starts_with_prefix_path; } +bool symlinkStartsWith(const std::filesystem::path & path, const std::filesystem::path & prefix_path) +{ + /// Differs from pathStartsWith in how `path` is normalized before comparison. + /// Make `path` absolute if it was relative and put it into normalized form: remove + /// `.` and `..` and extra `/`. Path is not canonized because otherwise path will + /// not be a path of a symlink itself. + + auto absolute_path = std::filesystem::absolute(path); + absolute_path = absolute_path.lexically_normal(); /// Normalize path. + auto absolute_prefix_path = std::filesystem::absolute(prefix_path); + absolute_prefix_path = absolute_prefix_path.lexically_normal(); /// Normalize path. + + auto [_, prefix_path_mismatch_it] = std::mismatch(absolute_path.begin(), absolute_path.end(), absolute_prefix_path.begin(), absolute_prefix_path.end()); + + bool path_starts_with_prefix_path = (prefix_path_mismatch_it == absolute_prefix_path.end()); + return path_starts_with_prefix_path; +} + bool pathStartsWith(const String & path, const String & prefix_path) { auto filesystem_path = std::filesystem::path(path); @@ -130,6 +148,13 @@ bool pathStartsWith(const String & path, const String & prefix_path) return pathStartsWith(filesystem_path, filesystem_prefix_path); } +bool symlinkStartsWith(const String & path, const String & prefix_path) +{ + auto filesystem_path = std::filesystem::path(path); + auto filesystem_prefix_path = std::filesystem::path(prefix_path); + + return symlinkStartsWith(filesystem_path, filesystem_prefix_path); +} } diff --git a/src/Common/filesystemHelpers.h b/src/Common/filesystemHelpers.h index b7525a64fae..71ef7844ef7 100644 --- a/src/Common/filesystemHelpers.h +++ b/src/Common/filesystemHelpers.h @@ -35,6 +35,8 @@ bool pathStartsWith(const std::filesystem::path & path, const std::filesystem::p /// Returns true if path starts with prefix path bool pathStartsWith(const String & path, const String & prefix_path); +bool symlinkStartsWith(const String & path, const String & prefix_path); + } namespace FS diff --git a/src/Common/memory.cpp b/src/Common/memory.cpp deleted file mode 100644 index a79d3572071..00000000000 --- a/src/Common/memory.cpp +++ /dev/null @@ -1,25 +0,0 @@ -#if defined(OS_DARWIN) && defined(BUNDLED_STATIC_JEMALLOC) - -extern "C" -{ - extern void zone_register(); -} - -struct InitializeJemallocZoneAllocatorForOSX -{ - InitializeJemallocZoneAllocatorForOSX() - { - /// In case of OSX jemalloc register itself as a default zone allocator. - /// - /// But when you link statically then zone_register() will not be called, - /// and even will be optimized out: - /// - /// It is ok to call it twice (i.e. in case of shared libraries) - /// Since zone_register() is a no-op if the default zone is already replaced with something. - /// - /// https://github.com/jemalloc/jemalloc/issues/708 - zone_register(); - } -} initializeJemallocZoneAllocatorForOSX; - -#endif diff --git a/src/Compression/CMakeLists.txt b/src/Compression/CMakeLists.txt index 390835f17ae..34369d8dbc8 100644 --- a/src/Compression/CMakeLists.txt +++ b/src/Compression/CMakeLists.txt @@ -1,3 +1,18 @@ -if(ENABLE_EXAMPLES) +if (ENABLE_FUZZING) + include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") + add_headers_and_sources(fuzz_compression .) + + # Remove this file, because it has dependencies on DataTypes + list(REMOVE_ITEM ${fuzz_compression_sources} CompressionFactoryAdditions.cpp) + + add_library(fuzz_compression ${fuzz_compression_headers} ${fuzz_compression_sources}) + target_link_libraries(fuzz_compression PUBLIC clickhouse_parsers clickhouse_common_io common lz4) +endif() + +if (ENABLE_EXAMPLES) add_subdirectory(examples) endif() + +if (ENABLE_FUZZING) + add_subdirectory(fuzzers) +endif() diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp index bb2b00a56ef..4dbc72c22f5 100644 --- a/src/Compression/CompressionFactory.cpp +++ b/src/Compression/CompressionFactory.cpp @@ -22,13 +22,10 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int UNKNOWN_CODEC; - extern const int BAD_ARGUMENTS; extern const int UNEXPECTED_AST_STRUCTURE; extern const int DATA_TYPE_CANNOT_HAVE_ARGUMENTS; } -static constexpr auto DEFAULT_CODEC_NAME = "Default"; - CompressionCodecPtr CompressionCodecFactory::getDefaultCodec() const { return default_codec; @@ -49,184 +46,6 @@ CompressionCodecPtr CompressionCodecFactory::get(const String & family_name, std } } -void CompressionCodecFactory::validateCodec( - const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs) const -{ - if (family_name.empty()) - throw Exception("Compression codec name cannot be empty", ErrorCodes::BAD_ARGUMENTS); - - if (level) - { - auto literal = std::make_shared(static_cast(*level)); - validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), literal)), - {}, sanity_check, allow_experimental_codecs); - } - else - { - auto identifier = std::make_shared(Poco::toUpper(family_name)); - validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", identifier), - {}, sanity_check, allow_experimental_codecs); - } -} - -ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( - const ASTPtr & ast, const IDataType * column_type, bool sanity_check, bool allow_experimental_codecs) const -{ - if (const auto * func = ast->as()) - { - ASTPtr codecs_descriptions = std::make_shared(); - - bool is_compression = false; - bool has_none = false; - std::optional generic_compression_codec_pos; - std::set post_processing_codecs; - - bool can_substitute_codec_arguments = true; - for (size_t i = 0, size = func->arguments->children.size(); i < size; ++i) - { - const auto & inner_codec_ast = func->arguments->children[i]; - String codec_family_name; - ASTPtr codec_arguments; - if (const auto * family_name = inner_codec_ast->as()) - { - codec_family_name = family_name->name(); - codec_arguments = {}; - } - else if (const auto * ast_func = inner_codec_ast->as()) - { - codec_family_name = ast_func->name; - codec_arguments = ast_func->arguments; - } - else - throw Exception("Unexpected AST element for compression codec", ErrorCodes::UNEXPECTED_AST_STRUCTURE); - - /// Default codec replaced with current default codec which may depend on different - /// settings (and properties of data) in runtime. - CompressionCodecPtr result_codec; - if (codec_family_name == DEFAULT_CODEC_NAME) - { - if (codec_arguments != nullptr) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "{} codec cannot have any arguments, it's just an alias for codec specified in config.xml", DEFAULT_CODEC_NAME); - - result_codec = default_codec; - codecs_descriptions->children.emplace_back(std::make_shared(DEFAULT_CODEC_NAME)); - } - else - { - if (column_type) - { - CompressionCodecPtr prev_codec; - IDataType::StreamCallbackWithType callback = [&]( - const ISerialization::SubstreamPath & substream_path, const IDataType & substream_type) - { - if (ISerialization::isSpecialCompressionAllowed(substream_path)) - { - result_codec = getImpl(codec_family_name, codec_arguments, &substream_type); - - /// Case for column Tuple, which compressed with codec which depends on data type, like Delta. - /// We cannot substitute parameters for such codecs. - if (prev_codec && prev_codec->getHash() != result_codec->getHash()) - can_substitute_codec_arguments = false; - prev_codec = result_codec; - } - }; - - ISerialization::SubstreamPath stream_path; - column_type->enumerateStreams(column_type->getDefaultSerialization(), callback, stream_path); - - if (!result_codec) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find any substream with data type for type {}. It's a bug", column_type->getName()); - } - else - { - result_codec = getImpl(codec_family_name, codec_arguments, nullptr); - } - - if (!allow_experimental_codecs && result_codec->isExperimental()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Codec {} is experimental and not meant to be used in production." - " You can enable it with the 'allow_experimental_codecs' setting.", - codec_family_name); - - codecs_descriptions->children.emplace_back(result_codec->getCodecDesc()); - } - - is_compression |= result_codec->isCompression(); - has_none |= result_codec->isNone(); - - if (!generic_compression_codec_pos && result_codec->isGenericCompression()) - generic_compression_codec_pos = i; - - if (result_codec->isPostProcessing()) - post_processing_codecs.insert(i); - } - - String codec_description = queryToString(codecs_descriptions); - - if (sanity_check) - { - if (codecs_descriptions->children.size() > 1 && has_none) - throw Exception( - "It does not make sense to have codec NONE along with other compression codecs: " + codec_description - + ". (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", - ErrorCodes::BAD_ARGUMENTS); - - /// Allow to explicitly specify single NONE codec if user don't want any compression. - /// But applying other transformations solely without compression (e.g. Delta) does not make sense. - /// It's okay to apply post-processing codecs solely without anything else. - if (!is_compression && !has_none && post_processing_codecs.size() != codecs_descriptions->children.size()) - throw Exception( - "Compression codec " + codec_description - + " does not compress anything." - " You may want to add generic compression algorithm after other transformations, like: " - + codec_description - + ", LZ4." - " (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", - ErrorCodes::BAD_ARGUMENTS); - - /// It does not make sense to apply any non-post-processing codecs - /// after post-processing one. - if (!post_processing_codecs.empty() && - *post_processing_codecs.begin() != codecs_descriptions->children.size() - post_processing_codecs.size()) - throw Exception("The combination of compression codecs " + codec_description + " is meaningless," - " because it does not make sense to apply any non-post-processing codecs after" - " post-processing ones. (Note: you can enable setting 'allow_suspicious_codecs'" - " to skip this check).", ErrorCodes::BAD_ARGUMENTS); - - /// It does not make sense to apply any transformations after generic compression algorithm - /// So, generic compression can be only one and only at the end. - if (generic_compression_codec_pos && - *generic_compression_codec_pos != codecs_descriptions->children.size() - 1 - post_processing_codecs.size()) - throw Exception("The combination of compression codecs " + codec_description + " is meaningless," - " because it does not make sense to apply any transformations after generic compression algorithm." - " (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", ErrorCodes::BAD_ARGUMENTS); - - } - - /// For columns with nested types like Tuple(UInt32, UInt64) we - /// obviously cannot substitute parameters for codecs which depend on - /// data type, because for the first column Delta(4) is suitable and - /// Delta(8) for the second. So we should leave codec description as is - /// and deduce them in get method for each subtype separately. For all - /// other types it's better to substitute parameters, for better - /// readability and backward compatibility. - if (can_substitute_codec_arguments) - { - std::shared_ptr result = std::make_shared(); - result->name = "CODEC"; - result->arguments = codecs_descriptions; - return result; - } - else - { - return ast; - } - } - - throw Exception("Unknown codec family: " + queryToString(ast), ErrorCodes::UNKNOWN_CODEC); -} - CompressionCodecPtr CompressionCodecFactory::get( const ASTPtr & ast, const IDataType * column_type, CompressionCodecPtr current_default, bool only_generic) const diff --git a/src/Compression/CompressionFactory.h b/src/Compression/CompressionFactory.h index cdbb663935a..f00e5071990 100644 --- a/src/Compression/CompressionFactory.h +++ b/src/Compression/CompressionFactory.h @@ -14,6 +14,8 @@ namespace DB { +static constexpr auto DEFAULT_CODEC_NAME = "Default"; + class ICompressionCodec; using CompressionCodecPtr = std::shared_ptr; diff --git a/src/Compression/CompressionFactoryAdditions.cpp b/src/Compression/CompressionFactoryAdditions.cpp new file mode 100644 index 00000000000..bff294d1ea5 --- /dev/null +++ b/src/Compression/CompressionFactoryAdditions.cpp @@ -0,0 +1,214 @@ +/** + * This file contains a part of CompressionCodecFactory methods definitions and + * is needed only because they have dependencies on DataTypes. + * They are not useful for fuzzers, so we leave them in other translation unit. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int UNEXPECTED_AST_STRUCTURE; + extern const int UNKNOWN_CODEC; + extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; +} + + +void CompressionCodecFactory::validateCodec( + const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs) const +{ + if (family_name.empty()) + throw Exception("Compression codec name cannot be empty", ErrorCodes::BAD_ARGUMENTS); + + if (level) + { + auto literal = std::make_shared(static_cast(*level)); + validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), literal)), + {}, sanity_check, allow_experimental_codecs); + } + else + { + auto identifier = std::make_shared(Poco::toUpper(family_name)); + validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", identifier), + {}, sanity_check, allow_experimental_codecs); + } +} + + +ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( + const ASTPtr & ast, const IDataType * column_type, bool sanity_check, bool allow_experimental_codecs) const +{ + if (const auto * func = ast->as()) + { + ASTPtr codecs_descriptions = std::make_shared(); + + bool is_compression = false; + bool has_none = false; + std::optional generic_compression_codec_pos; + std::set post_processing_codecs; + + bool can_substitute_codec_arguments = true; + for (size_t i = 0, size = func->arguments->children.size(); i < size; ++i) + { + const auto & inner_codec_ast = func->arguments->children[i]; + String codec_family_name; + ASTPtr codec_arguments; + if (const auto * family_name = inner_codec_ast->as()) + { + codec_family_name = family_name->name(); + codec_arguments = {}; + } + else if (const auto * ast_func = inner_codec_ast->as()) + { + codec_family_name = ast_func->name; + codec_arguments = ast_func->arguments; + } + else + throw Exception("Unexpected AST element for compression codec", ErrorCodes::UNEXPECTED_AST_STRUCTURE); + + /// Default codec replaced with current default codec which may depend on different + /// settings (and properties of data) in runtime. + CompressionCodecPtr result_codec; + if (codec_family_name == DEFAULT_CODEC_NAME) + { + if (codec_arguments != nullptr) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "{} codec cannot have any arguments, it's just an alias for codec specified in config.xml", DEFAULT_CODEC_NAME); + + result_codec = default_codec; + codecs_descriptions->children.emplace_back(std::make_shared(DEFAULT_CODEC_NAME)); + } + else + { + if (column_type) + { + CompressionCodecPtr prev_codec; + IDataType::StreamCallbackWithType callback = [&]( + const ISerialization::SubstreamPath & substream_path, const IDataType & substream_type) + { + if (ISerialization::isSpecialCompressionAllowed(substream_path)) + { + result_codec = getImpl(codec_family_name, codec_arguments, &substream_type); + + /// Case for column Tuple, which compressed with codec which depends on data type, like Delta. + /// We cannot substitute parameters for such codecs. + if (prev_codec && prev_codec->getHash() != result_codec->getHash()) + can_substitute_codec_arguments = false; + prev_codec = result_codec; + } + }; + + ISerialization::SubstreamPath stream_path; + column_type->enumerateStreams(column_type->getDefaultSerialization(), callback, stream_path); + + if (!result_codec) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot find any substream with data type for type {}. It's a bug", column_type->getName()); + } + else + { + result_codec = getImpl(codec_family_name, codec_arguments, nullptr); + } + + if (!allow_experimental_codecs && result_codec->isExperimental()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Codec {} is experimental and not meant to be used in production." + " You can enable it with the 'allow_experimental_codecs' setting.", + codec_family_name); + + codecs_descriptions->children.emplace_back(result_codec->getCodecDesc()); + } + + is_compression |= result_codec->isCompression(); + has_none |= result_codec->isNone(); + + if (!generic_compression_codec_pos && result_codec->isGenericCompression()) + generic_compression_codec_pos = i; + + if (result_codec->isPostProcessing()) + post_processing_codecs.insert(i); + } + + String codec_description = queryToString(codecs_descriptions); + + if (sanity_check) + { + if (codecs_descriptions->children.size() > 1 && has_none) + throw Exception( + "It does not make sense to have codec NONE along with other compression codecs: " + codec_description + + ". (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", + ErrorCodes::BAD_ARGUMENTS); + + /// Allow to explicitly specify single NONE codec if user don't want any compression. + /// But applying other transformations solely without compression (e.g. Delta) does not make sense. + /// It's okay to apply post-processing codecs solely without anything else. + if (!is_compression && !has_none && post_processing_codecs.size() != codecs_descriptions->children.size()) + throw Exception( + "Compression codec " + codec_description + + " does not compress anything." + " You may want to add generic compression algorithm after other transformations, like: " + + codec_description + + ", LZ4." + " (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", + ErrorCodes::BAD_ARGUMENTS); + + /// It does not make sense to apply any non-post-processing codecs + /// after post-processing one. + if (!post_processing_codecs.empty() && + *post_processing_codecs.begin() != codecs_descriptions->children.size() - post_processing_codecs.size()) + throw Exception("The combination of compression codecs " + codec_description + " is meaningless," + " because it does not make sense to apply any non-post-processing codecs after" + " post-processing ones. (Note: you can enable setting 'allow_suspicious_codecs'" + " to skip this check).", ErrorCodes::BAD_ARGUMENTS); + + /// It does not make sense to apply any transformations after generic compression algorithm + /// So, generic compression can be only one and only at the end. + if (generic_compression_codec_pos && + *generic_compression_codec_pos != codecs_descriptions->children.size() - 1 - post_processing_codecs.size()) + throw Exception("The combination of compression codecs " + codec_description + " is meaningless," + " because it does not make sense to apply any transformations after generic compression algorithm." + " (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", ErrorCodes::BAD_ARGUMENTS); + + } + + /// For columns with nested types like Tuple(UInt32, UInt64) we + /// obviously cannot substitute parameters for codecs which depend on + /// data type, because for the first column Delta(4) is suitable and + /// Delta(8) for the second. So we should leave codec description as is + /// and deduce them in get method for each subtype separately. For all + /// other types it's better to substitute parameters, for better + /// readability and backward compatibility. + if (can_substitute_codec_arguments) + { + std::shared_ptr result = std::make_shared(); + result->name = "CODEC"; + result->arguments = codecs_descriptions; + return result; + } + else + { + return ast; + } + } + + throw Exception("Unknown codec family: " + queryToString(ast), ErrorCodes::UNKNOWN_CODEC); +} + + +} diff --git a/src/Compression/LZ4_decompress_faster.cpp b/src/Compression/LZ4_decompress_faster.cpp index 6972457f11b..28a285f00f4 100644 --- a/src/Compression/LZ4_decompress_faster.cpp +++ b/src/Compression/LZ4_decompress_faster.cpp @@ -439,11 +439,14 @@ bool NO_INLINE decompressImpl( { s = *ip++; length += s; - } while (unlikely(s == 255)); + } while (unlikely(s == 255 && ip < input_end)); }; /// Get literal length. + if (unlikely(ip >= input_end)) + return false; + const unsigned token = *ip++; length = token >> 4; if (length == 0x0F) @@ -464,18 +467,18 @@ bool NO_INLINE decompressImpl( /// output: xyzHello, w /// ^-op (we will overwrite excessive bytes on next iteration) - { - auto * target = std::min(copy_end, output_end); - wildCopy(op, ip, target); /// Here we can write up to copy_amount - 1 bytes after buffer. + if (unlikely(copy_end > output_end)) + return false; - if (target == output_end) - return true; - } + wildCopy(op, ip, copy_end); /// Here we can write up to copy_amount - 1 bytes after buffer. + + if (copy_end == output_end) + return true; ip += length; op = copy_end; - if (unlikely(ip > input_end)) + if (unlikely(ip + 1 >= input_end)) return false; /// Get match offset. @@ -528,8 +531,9 @@ bool NO_INLINE decompressImpl( copy(op, match); /// copy_amount + copy_amount - 1 - 4 * 2 bytes after buffer. if (length > copy_amount * 2) { - auto * target = std::min(copy_end, output_end); - wildCopy(op + copy_amount, match + copy_amount, target); + if (unlikely(copy_end > output_end)) + return false; + wildCopy(op + copy_amount, match + copy_amount, copy_end); } op = copy_end; diff --git a/src/Compression/examples/CMakeLists.txt b/src/Compression/examples/CMakeLists.txt index f47c4404cf6..3cfc0ccb7dc 100644 --- a/src/Compression/examples/CMakeLists.txt +++ b/src/Compression/examples/CMakeLists.txt @@ -3,8 +3,3 @@ target_link_libraries (compressed_buffer PRIVATE dbms) add_executable (cached_compressed_read_buffer cached_compressed_read_buffer.cpp) target_link_libraries (cached_compressed_read_buffer PRIVATE dbms) - -if (ENABLE_FUZZING) - add_executable (compressed_buffer_fuzzer compressed_buffer_fuzzer.cpp) - target_link_libraries (compressed_buffer_fuzzer PRIVATE dbms ${LIB_FUZZING_ENGINE}) -endif () diff --git a/src/Compression/fuzzers/CMakeLists.txt b/src/Compression/fuzzers/CMakeLists.txt new file mode 100644 index 00000000000..74bf2d2649b --- /dev/null +++ b/src/Compression/fuzzers/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable (compressed_buffer_fuzzer compressed_buffer_fuzzer.cpp) +target_link_libraries (compressed_buffer_fuzzer PRIVATE fuzz_compression clickhouse_common_io ${LIB_FUZZING_ENGINE}) diff --git a/src/Compression/examples/compressed_buffer_fuzzer.cpp b/src/Compression/fuzzers/compressed_buffer_fuzzer.cpp similarity index 100% rename from src/Compression/examples/compressed_buffer_fuzzer.cpp rename to src/Compression/fuzzers/compressed_buffer_fuzzer.cpp diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index 4c3f649a6b6..320754c7d31 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -248,117 +248,117 @@ struct KeeperStorageCreateRequest final : public KeeperStorageRequest Coordination::ZooKeeperCreateResponse & response = dynamic_cast(*response_ptr); Coordination::ZooKeeperCreateRequest & request = dynamic_cast(*zk_request); - if (container.contains(request.path)) + auto parent_path = parentPath(request.path); + auto it = container.find(parent_path); + + if (it == container.end()) + { + response.error = Coordination::Error::ZNONODE; + return { response_ptr, undo }; + } + else if (it->value.stat.ephemeralOwner != 0) + { + response.error = Coordination::Error::ZNOCHILDRENFOREPHEMERALS; + return { response_ptr, undo }; + } + std::string path_created = request.path; + if (request.is_sequential) + { + auto seq_num = it->value.seq_num; + + std::stringstream seq_num_str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + seq_num_str.exceptions(std::ios::failbit); + seq_num_str << std::setw(10) << std::setfill('0') << seq_num; + + path_created += seq_num_str.str(); + } + if (container.contains(path_created)) { response.error = Coordination::Error::ZNODEEXISTS; + return { response_ptr, undo }; } - else + auto child_path = getBaseName(path_created); + if (child_path.empty()) { - auto parent_path = parentPath(request.path); - auto it = container.find(parent_path); - - if (it == container.end()) - { - response.error = Coordination::Error::ZNONODE; - } - else if (it->value.stat.ephemeralOwner != 0) - { - response.error = Coordination::Error::ZNOCHILDRENFOREPHEMERALS; - } - else - { - auto & session_auth_ids = storage.session_and_auth[session_id]; - - KeeperStorage::Node created_node; - - Coordination::ACLs node_acls; - if (!fixupACL(request.acls, session_auth_ids, node_acls, !request.restored_from_zookeeper_log)) - { - response.error = Coordination::Error::ZINVALIDACL; - return {response_ptr, {}}; - } - - uint64_t acl_id = storage.acl_map.convertACLs(node_acls); - storage.acl_map.addUsage(acl_id); - - created_node.acl_id = acl_id; - created_node.stat.czxid = zxid; - created_node.stat.mzxid = zxid; - created_node.stat.pzxid = zxid; - created_node.stat.ctime = std::chrono::system_clock::now().time_since_epoch() / std::chrono::milliseconds(1); - created_node.stat.mtime = created_node.stat.ctime; - created_node.stat.numChildren = 0; - created_node.stat.dataLength = request.data.length(); - created_node.stat.ephemeralOwner = request.is_ephemeral ? session_id : 0; - created_node.data = request.data; - created_node.is_sequental = request.is_sequential; - - std::string path_created = request.path; - - if (request.is_sequential) - { - auto seq_num = it->value.seq_num; - - std::stringstream seq_num_str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - seq_num_str.exceptions(std::ios::failbit); - seq_num_str << std::setw(10) << std::setfill('0') << seq_num; - - path_created += seq_num_str.str(); - } - - int32_t parent_cversion = request.parent_cversion; - auto child_path = getBaseName(path_created); - int64_t prev_parent_zxid; - int32_t prev_parent_cversion; - container.updateValue(parent_path, [child_path, zxid, &prev_parent_zxid, - parent_cversion, &prev_parent_cversion] (KeeperStorage::Node & parent) - { - - parent.children.insert(child_path); - prev_parent_cversion = parent.stat.cversion; - prev_parent_zxid = parent.stat.pzxid; - - /// Increment sequential number even if node is not sequential - ++parent.seq_num; - - if (parent_cversion == -1) - ++parent.stat.cversion; - else if (parent_cversion > parent.stat.cversion) - parent.stat.cversion = parent_cversion; - - if (zxid > parent.stat.pzxid) - parent.stat.pzxid = zxid; - ++parent.stat.numChildren; - }); - - response.path_created = path_created; - container.insert(path_created, std::move(created_node)); - - if (request.is_ephemeral) - ephemerals[session_id].emplace(path_created); - - undo = [&storage, prev_parent_zxid, prev_parent_cversion, session_id, path_created, is_ephemeral = request.is_ephemeral, parent_path, child_path, acl_id] - { - storage.container.erase(path_created); - storage.acl_map.removeUsage(acl_id); - - if (is_ephemeral) - storage.ephemerals[session_id].erase(path_created); - - storage.container.updateValue(parent_path, [child_path, prev_parent_zxid, prev_parent_cversion] (KeeperStorage::Node & undo_parent) - { - --undo_parent.stat.numChildren; - --undo_parent.seq_num; - undo_parent.stat.cversion = prev_parent_cversion; - undo_parent.stat.pzxid = prev_parent_zxid; - undo_parent.children.erase(child_path); - }); - }; - - response.error = Coordination::Error::ZOK; - } + response.error = Coordination::Error::ZBADARGUMENTS; + return { response_ptr, undo }; } + auto & session_auth_ids = storage.session_and_auth[session_id]; + + KeeperStorage::Node created_node; + + Coordination::ACLs node_acls; + if (!fixupACL(request.acls, session_auth_ids, node_acls, !request.restored_from_zookeeper_log)) + { + response.error = Coordination::Error::ZINVALIDACL; + return {response_ptr, {}}; + } + + uint64_t acl_id = storage.acl_map.convertACLs(node_acls); + storage.acl_map.addUsage(acl_id); + + created_node.acl_id = acl_id; + created_node.stat.czxid = zxid; + created_node.stat.mzxid = zxid; + created_node.stat.pzxid = zxid; + created_node.stat.ctime = std::chrono::system_clock::now().time_since_epoch() / std::chrono::milliseconds(1); + created_node.stat.mtime = created_node.stat.ctime; + created_node.stat.numChildren = 0; + created_node.stat.dataLength = request.data.length(); + created_node.stat.ephemeralOwner = request.is_ephemeral ? session_id : 0; + created_node.data = request.data; + created_node.is_sequental = request.is_sequential; + + int32_t parent_cversion = request.parent_cversion; + int64_t prev_parent_zxid; + int32_t prev_parent_cversion; + container.updateValue(parent_path, [child_path, zxid, &prev_parent_zxid, + parent_cversion, &prev_parent_cversion] (KeeperStorage::Node & parent) + { + + parent.children.insert(child_path); + prev_parent_cversion = parent.stat.cversion; + prev_parent_zxid = parent.stat.pzxid; + + /// Increment sequential number even if node is not sequential + ++parent.seq_num; + + if (parent_cversion == -1) + ++parent.stat.cversion; + else if (parent_cversion > parent.stat.cversion) + parent.stat.cversion = parent_cversion; + + if (zxid > parent.stat.pzxid) + parent.stat.pzxid = zxid; + ++parent.stat.numChildren; + }); + + response.path_created = path_created; + container.insert(path_created, std::move(created_node)); + + if (request.is_ephemeral) + ephemerals[session_id].emplace(path_created); + + undo = [&storage, prev_parent_zxid, prev_parent_cversion, session_id, path_created, is_ephemeral = request.is_ephemeral, parent_path, child_path, acl_id] + { + storage.container.erase(path_created); + storage.acl_map.removeUsage(acl_id); + + if (is_ephemeral) + storage.ephemerals[session_id].erase(path_created); + + storage.container.updateValue(parent_path, [child_path, prev_parent_zxid, prev_parent_cversion] (KeeperStorage::Node & undo_parent) + { + --undo_parent.stat.numChildren; + --undo_parent.seq_num; + undo_parent.stat.cversion = prev_parent_cversion; + undo_parent.stat.pzxid = prev_parent_zxid; + undo_parent.children.erase(child_path); + }); + }; + + response.error = Coordination::Error::ZOK; return { response_ptr, undo }; } }; diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index 96667862e41..ddfd62c2efb 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -22,6 +22,85 @@ namespace ErrorCodes extern const int POSITION_OUT_OF_BOUND; extern const int NOT_FOUND_COLUMN_IN_BLOCK; extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; + extern const int AMBIGUOUS_COLUMN_NAME; +} + +template +static ReturnType onError(const std::string & message [[maybe_unused]], int code [[maybe_unused]]) +{ + if constexpr (std::is_same_v) + throw Exception(message, code); + else + return false; +}; + + +template +static ReturnType checkColumnStructure(const ColumnWithTypeAndName & actual, const ColumnWithTypeAndName & expected, + const std::string & context_description, bool allow_remove_constants, int code) +{ + if (actual.name != expected.name) + return onError("Block structure mismatch in " + context_description + " stream: different names of columns:\n" + + actual.dumpStructure() + "\n" + expected.dumpStructure(), code); + + if (!actual.type->equals(*expected.type)) + return onError("Block structure mismatch in " + context_description + " stream: different types:\n" + + actual.dumpStructure() + "\n" + expected.dumpStructure(), code); + + if (!actual.column || !expected.column) + return ReturnType(true); + + const IColumn * actual_column = actual.column.get(); + + /// If we allow to remove constants, and expected column is not const, then unwrap actual constant column. + if (allow_remove_constants && !isColumnConst(*expected.column)) + { + if (const auto * column_const = typeid_cast(actual_column)) + actual_column = &column_const->getDataColumn(); + } + + if (actual_column->getName() != expected.column->getName()) + return onError("Block structure mismatch in " + context_description + " stream: different columns:\n" + + actual.dumpStructure() + "\n" + expected.dumpStructure(), code); + + if (isColumnConst(*actual.column) && isColumnConst(*expected.column)) + { + Field actual_value = assert_cast(*actual.column).getField(); + Field expected_value = assert_cast(*expected.column).getField(); + + if (actual_value != expected_value) + return onError("Block structure mismatch in " + context_description + " stream: different values of constants, actual: " + + applyVisitor(FieldVisitorToString(), actual_value) + ", expected: " + applyVisitor(FieldVisitorToString(), expected_value), + code); + } + + return ReturnType(true); +} + + +template +static ReturnType checkBlockStructure(const Block & lhs, const Block & rhs, const std::string & context_description, bool allow_remove_constants) +{ + size_t columns = rhs.columns(); + if (lhs.columns() != columns) + return onError("Block structure mismatch in " + context_description + " stream: different number of columns:\n" + + lhs.dumpStructure() + "\n" + rhs.dumpStructure(), ErrorCodes::LOGICAL_ERROR); + + for (size_t i = 0; i < columns; ++i) + { + const auto & actual = lhs.getByPosition(i); + const auto & expected = rhs.getByPosition(i); + + if constexpr (std::is_same_v) + { + if (!checkColumnStructure(actual, expected, context_description, allow_remove_constants, ErrorCodes::LOGICAL_ERROR)) + return false; + } + else + checkColumnStructure(actual, expected, context_description, allow_remove_constants, ErrorCodes::LOGICAL_ERROR); + } + + return ReturnType(true); } @@ -57,24 +136,41 @@ void Block::insert(size_t position, ColumnWithTypeAndName elem) throw Exception("Position out of bound in Block::insert(), max position = " + toString(data.size()), ErrorCodes::POSITION_OUT_OF_BOUND); + if (elem.name.empty()) + throw Exception("Column name in Block cannot be empty", ErrorCodes::AMBIGUOUS_COLUMN_NAME); + for (auto & name_pos : index_by_name) if (name_pos.second >= position) ++name_pos.second; - index_by_name.emplace(elem.name, position); + auto [it, inserted] = index_by_name.emplace(elem.name, position); + if (!inserted) + checkColumnStructure(data[it->second], elem, + "(columns with identical name must have identical structure)", true, ErrorCodes::AMBIGUOUS_COLUMN_NAME); + data.emplace(data.begin() + position, std::move(elem)); } void Block::insert(ColumnWithTypeAndName elem) { - index_by_name.emplace(elem.name, data.size()); + if (elem.name.empty()) + throw Exception("Column name in Block cannot be empty", ErrorCodes::AMBIGUOUS_COLUMN_NAME); + + auto [it, inserted] = index_by_name.emplace(elem.name, data.size()); + if (!inserted) + checkColumnStructure(data[it->second], elem, + "(columns with identical name must have identical structure)", true, ErrorCodes::AMBIGUOUS_COLUMN_NAME); + data.emplace_back(std::move(elem)); } void Block::insertUnique(ColumnWithTypeAndName elem) { + if (elem.name.empty()) + throw Exception("Column name in Block cannot be empty", ErrorCodes::AMBIGUOUS_COLUMN_NAME); + if (index_by_name.end() == index_by_name.find(elem.name)) insert(std::move(elem)); } @@ -487,67 +583,6 @@ DataTypes Block::getDataTypes() const } -template -static ReturnType checkBlockStructure(const Block & lhs, const Block & rhs, const std::string & context_description, bool allow_remove_constants) -{ - auto on_error = [](const std::string & message [[maybe_unused]], int code [[maybe_unused]]) - { - if constexpr (std::is_same_v) - throw Exception(message, code); - else - return false; - }; - - size_t columns = rhs.columns(); - if (lhs.columns() != columns) - return on_error("Block structure mismatch in " + context_description + " stream: different number of columns:\n" - + lhs.dumpStructure() + "\n" + rhs.dumpStructure(), ErrorCodes::LOGICAL_ERROR); - - for (size_t i = 0; i < columns; ++i) - { - const auto & expected = rhs.getByPosition(i); - const auto & actual = lhs.getByPosition(i); - - if (actual.name != expected.name) - return on_error("Block structure mismatch in " + context_description + " stream: different names of columns:\n" - + lhs.dumpStructure() + "\n" + rhs.dumpStructure(), ErrorCodes::LOGICAL_ERROR); - - if (!actual.type->equals(*expected.type)) - return on_error("Block structure mismatch in " + context_description + " stream: different types:\n" - + lhs.dumpStructure() + "\n" + rhs.dumpStructure(), ErrorCodes::LOGICAL_ERROR); - - if (!actual.column || !expected.column) - continue; - - const IColumn * actual_column = actual.column.get(); - - /// If we allow to remove constants, and expected column is not const, then unwrap actual constant column. - if (allow_remove_constants && !isColumnConst(*expected.column)) - { - if (const auto * column_const = typeid_cast(actual_column)) - actual_column = &column_const->getDataColumn(); - } - - if (actual_column->getName() != expected.column->getName()) - return on_error("Block structure mismatch in " + context_description + " stream: different columns:\n" - + lhs.dumpStructure() + "\n" + rhs.dumpStructure(), ErrorCodes::LOGICAL_ERROR); - - if (isColumnConst(*actual.column) && isColumnConst(*expected.column)) - { - Field actual_value = assert_cast(*actual.column).getField(); - Field expected_value = assert_cast(*expected.column).getField(); - - if (actual_value != expected_value) - return on_error("Block structure mismatch in " + context_description + " stream: different values of constants, actual: " - + applyVisitor(FieldVisitorToString(), actual_value) + ", expected: " + applyVisitor(FieldVisitorToString(), expected_value), - ErrorCodes::LOGICAL_ERROR); - } - } - - return ReturnType(true); -} - - bool blocksHaveEqualStructure(const Block & lhs, const Block & rhs) { return checkBlockStructure(lhs, rhs, {}, false); diff --git a/src/Core/CMakeLists.txt b/src/Core/CMakeLists.txt index a6176efc7f3..85e2008753d 100644 --- a/src/Core/CMakeLists.txt +++ b/src/Core/CMakeLists.txt @@ -1,3 +1,7 @@ if (ENABLE_EXAMPLES) add_subdirectory(examples) endif () + +if (ENABLE_FUZZING) + add_subdirectory(fuzzers) +endif() diff --git a/src/Core/MySQL/Authentication.cpp b/src/Core/MySQL/Authentication.cpp index 658c86795b1..aeb9a411082 100644 --- a/src/Core/MySQL/Authentication.cpp +++ b/src/Core/MySQL/Authentication.cpp @@ -2,8 +2,7 @@ #include #include #include -#include -#include +#include #include #include @@ -73,7 +72,7 @@ Native41::Native41(const String & password, const String & auth_plugin_data) } void Native41::authenticate( - const String & user_name, std::optional auth_response, ContextMutablePtr context, + const String & user_name, Session & session, std::optional auth_response, std::shared_ptr packet_endpoint, bool, const Poco::Net::SocketAddress & address) { if (!auth_response) @@ -86,7 +85,7 @@ void Native41::authenticate( if (auth_response->empty()) { - context->setUser(user_name, "", address); + session.authenticate(user_name, "", address); return; } @@ -96,9 +95,7 @@ void Native41::authenticate( + " bytes, received: " + std::to_string(auth_response->size()) + " bytes.", ErrorCodes::UNKNOWN_EXCEPTION); - auto user = context->getAccessControlManager().read(user_name); - - Poco::SHA1Engine::Digest double_sha1_value = user->authentication.getPasswordDoubleSHA1(); + Poco::SHA1Engine::Digest double_sha1_value = session.getPasswordDoubleSHA1(user_name); assert(double_sha1_value.size() == Poco::SHA1Engine::DIGEST_SIZE); Poco::SHA1Engine engine; @@ -111,7 +108,7 @@ void Native41::authenticate( { password_sha1[i] = digest[i] ^ static_cast((*auth_response)[i]); } - context->setUser(user_name, password_sha1, address); + session.authenticate(user_name, password_sha1, address); } #if USE_SSL @@ -136,7 +133,7 @@ Sha256Password::Sha256Password(RSA & public_key_, RSA & private_key_, Poco::Logg } void Sha256Password::authenticate( - const String & user_name, std::optional auth_response, ContextMutablePtr context, + const String & user_name, Session & session, std::optional auth_response, std::shared_ptr packet_endpoint, bool is_secure_connection, const Poco::Net::SocketAddress & address) { if (!auth_response) @@ -231,7 +228,7 @@ void Sha256Password::authenticate( password.pop_back(); } - context->setUser(user_name, password, address); + session.authenticate(user_name, password, address); } #endif diff --git a/src/Core/MySQL/Authentication.h b/src/Core/MySQL/Authentication.h index acbda2bdb58..a60e769434e 100644 --- a/src/Core/MySQL/Authentication.h +++ b/src/Core/MySQL/Authentication.h @@ -15,6 +15,7 @@ namespace DB { +class Session; namespace MySQLProtocol { @@ -32,7 +33,7 @@ public: virtual String getAuthPluginData() = 0; virtual void authenticate( - const String & user_name, std::optional auth_response, ContextMutablePtr context, + const String & user_name, Session & session, std::optional auth_response, std::shared_ptr packet_endpoint, bool is_secure_connection, const Poco::Net::SocketAddress & address) = 0; }; @@ -49,7 +50,7 @@ public: String getAuthPluginData() override { return scramble; } void authenticate( - const String & user_name, std::optional auth_response, ContextMutablePtr context, + const String & user_name, Session & session, std::optional auth_response, std::shared_ptr packet_endpoint, bool /* is_secure_connection */, const Poco::Net::SocketAddress & address) override; private: @@ -69,7 +70,7 @@ public: String getAuthPluginData() override { return scramble; } void authenticate( - const String & user_name, std::optional auth_response, ContextMutablePtr context, + const String & user_name, Session & session, std::optional auth_response, std::shared_ptr packet_endpoint, bool is_secure_connection, const Poco::Net::SocketAddress & address) override; private: diff --git a/src/Core/PostgreSQL/insertPostgreSQLValue.cpp b/src/Core/PostgreSQL/insertPostgreSQLValue.cpp index 19560cec9ea..4f31c8dfb52 100644 --- a/src/Core/PostgreSQL/insertPostgreSQLValue.cpp +++ b/src/Core/PostgreSQL/insertPostgreSQLValue.cpp @@ -110,7 +110,7 @@ void insertPostgreSQLValue( readDateTime64Text(time, 6, in, assert_cast(data_type.get())->getTimeZone()); if (time < 0) time = 0; - assert_cast &>(column).insertValue(time); + assert_cast(column).insertValue(time); break; } case ExternalResultDescription::ValueType::vtDecimal32: [[fallthrough]]; diff --git a/src/Core/PostgreSQLProtocol.h b/src/Core/PostgreSQLProtocol.h index 114abc0101f..aef0ed6ab25 100644 --- a/src/Core/PostgreSQLProtocol.h +++ b/src/Core/PostgreSQLProtocol.h @@ -1,13 +1,11 @@ #pragma once -#include -#include #include -#include #include #include #include #include +#include #include #include #include @@ -803,12 +801,13 @@ protected: static void setPassword( const String & user_name, const String & password, - ContextMutablePtr context, + Session & session, Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) { - try { - context->setUser(user_name, password, address); + try + { + session.authenticate(user_name, password, address); } catch (const Exception &) { @@ -822,7 +821,7 @@ protected: public: virtual void authenticate( const String & user_name, - ContextMutablePtr context, + Session & session, Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) = 0; @@ -836,11 +835,11 @@ class NoPasswordAuth : public AuthenticationMethod public: void authenticate( const String & user_name, - ContextMutablePtr context, + Session & session, Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) override { - setPassword(user_name, "", context, mt, address); + return setPassword(user_name, "", session, mt, address); } Authentication::Type getType() const override @@ -854,7 +853,7 @@ class CleartextPasswordAuth : public AuthenticationMethod public: void authenticate( const String & user_name, - ContextMutablePtr context, + Session & session, Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) override { @@ -864,7 +863,7 @@ public: if (type == Messaging::FrontMessageType::PASSWORD_MESSAGE) { std::unique_ptr password = mt.receive(); - setPassword(user_name, password->password, context, mt, address); + return setPassword(user_name, password->password, session, mt, address); } else throw Exception( @@ -897,16 +896,15 @@ public: void authenticate( const String & user_name, - ContextMutablePtr context, + Session & session, Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) { - auto user = context->getAccessControlManager().read(user_name); - Authentication::Type user_auth_type = user->authentication.getType(); + Authentication::Type user_auth_type = session.getAuthenticationType(user_name); if (type_to_method.find(user_auth_type) != type_to_method.end()) { - type_to_method[user_auth_type]->authenticate(user_name, context, mt, address); + type_to_method[user_auth_type]->authenticate(user_name, session, mt, address); mt.send(Messaging::AuthenticationOk(), true); LOG_DEBUG(log, "Authentication for user {} was successful.", user_name); return; diff --git a/src/Core/ServerUUID.cpp b/src/Core/ServerUUID.cpp new file mode 100644 index 00000000000..721c406ff5f --- /dev/null +++ b/src/Core/ServerUUID.cpp @@ -0,0 +1,56 @@ +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int CANNOT_CREATE_FILE; +} + +void ServerUUID::load(const fs::path & server_uuid_file, Poco::Logger * log) +{ + /// Write a uuid file containing a unique uuid if the file doesn't already exist during server start. + + if (fs::exists(server_uuid_file)) + { + try + { + UUID uuid; + ReadBufferFromFile in(server_uuid_file); + readUUIDText(uuid, in); + assertEOF(in); + server_uuid = uuid; + return; + } + catch (...) + { + /// As for now it's ok to just overwrite it, because persistency in not essential. + LOG_ERROR(log, "Cannot read server UUID from file {}: {}. Will overwrite it", + server_uuid_file.string(), getCurrentExceptionMessage(true)); + } + } + + try + { + UUID new_uuid = UUIDHelpers::generateV4(); + auto uuid_str = toString(new_uuid); + WriteBufferFromFile out(server_uuid_file); + out.write(uuid_str.data(), uuid_str.size()); + out.sync(); + out.finalize(); + server_uuid = new_uuid; + } + catch (...) + { + throw Exception(ErrorCodes::CANNOT_CREATE_FILE, "Caught Exception {} while writing the Server UUID file {}", + getCurrentExceptionMessage(false), server_uuid_file.string()); + } +} + +} diff --git a/src/Core/ServerUUID.h b/src/Core/ServerUUID.h new file mode 100644 index 00000000000..36bbf0e6315 --- /dev/null +++ b/src/Core/ServerUUID.h @@ -0,0 +1,26 @@ +#pragma once +#include +#include + +namespace fs = std::filesystem; +namespace Poco +{ + class Logger; +} + +namespace DB +{ + +class ServerUUID +{ + inline static UUID server_uuid = UUIDHelpers::Nil; + +public: + /// Returns persistent UUID of current clickhouse-server or clickhouse-keeper instance. + static UUID get() { return server_uuid; } + + /// Loads server UUID from file or creates new one. Should be called on daemon startup. + static void load(const fs::path & server_uuid_file, Poco::Logger * log); +}; + +} diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 30f3c69ff4a..ea71b4a498d 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -114,6 +114,7 @@ class IColumn; M(UInt64, group_by_two_level_threshold_bytes, 50000000, "From what size of the aggregation state in bytes, a two-level aggregation begins to be used. 0 - the threshold is not set. Two-level aggregation is used when at least one of the thresholds is triggered.", 0) \ M(Bool, distributed_aggregation_memory_efficient, true, "Is the memory-saving mode of distributed aggregation enabled.", 0) \ M(UInt64, aggregation_memory_efficient_merge_threads, 0, "Number of threads to use for merge intermediate aggregation results in memory efficient mode. When bigger, then more memory is consumed. 0 means - same as 'max_threads'.", 0) \ + M(Bool, enable_positional_arguments, false, "Enable positional arguments in ORDER BY, GROUP BY and LIMIT BY", 0) \ \ M(UInt64, max_parallel_replicas, 1, "The maximum number of replicas of each shard used when the query is executed. For consistency (to get different parts of the same partition), this option only works for the specified sampling key. The lag of the replicas is not controlled.", 0) \ M(UInt64, parallel_replicas_count, 0, "", 0) \ @@ -252,6 +253,7 @@ class IColumn; M(Bool, use_index_for_in_with_subqueries, true, "Try using an index if there is a subquery or a table expression on the right side of the IN operator.", 0) \ M(Bool, joined_subquery_requires_alias, true, "Force joined subqueries and table functions to have aliases for correct name qualification.", 0) \ M(Bool, empty_result_for_aggregation_by_empty_set, false, "Return empty result when aggregating without keys on empty set.", 0) \ + M(Bool, empty_result_for_aggregation_by_constant_keys_on_empty_set, true, "Return empty result when aggregating by constant keys on empty set.", 0) \ M(Bool, allow_distributed_ddl, true, "If it is set to true, then a user is allowed to executed distributed DDL queries.", 0) \ M(Bool, allow_suspicious_codecs, false, "If it is set to true, allow to specify meaningless compression codecs.", 0) \ M(Bool, allow_experimental_codecs, false, "If it is set to true, allow to specify experimental compression codecs (but we don't have those yet and this option does nothing).", 0) \ diff --git a/src/Core/examples/CMakeLists.txt b/src/Core/examples/CMakeLists.txt index cd6450633ff..6b07dfbbfa6 100644 --- a/src/Core/examples/CMakeLists.txt +++ b/src/Core/examples/CMakeLists.txt @@ -8,11 +8,6 @@ target_link_libraries (field PRIVATE dbms) add_executable (string_ref_hash string_ref_hash.cpp) target_link_libraries (string_ref_hash PRIVATE clickhouse_common_io) -if (ENABLE_FUZZING) - add_executable (names_and_types_fuzzer names_and_types_fuzzer.cpp) - target_link_libraries (names_and_types_fuzzer PRIVATE dbms ${LIB_FUZZING_ENGINE}) -endif () - add_executable (mysql_protocol mysql_protocol.cpp) target_link_libraries (mysql_protocol PRIVATE dbms) if(USE_SSL) diff --git a/src/Core/fuzzers/CMakeLists.txt b/src/Core/fuzzers/CMakeLists.txt new file mode 100644 index 00000000000..a5416035010 --- /dev/null +++ b/src/Core/fuzzers/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable (names_and_types_fuzzer names_and_types_fuzzer.cpp) +target_link_libraries (names_and_types_fuzzer PRIVATE dbms ${LIB_FUZZING_ENGINE}) diff --git a/src/Core/examples/names_and_types_fuzzer.cpp b/src/Core/fuzzers/names_and_types_fuzzer.cpp similarity index 100% rename from src/Core/examples/names_and_types_fuzzer.cpp rename to src/Core/fuzzers/names_and_types_fuzzer.cpp diff --git a/src/DataTypes/IDataType.cpp b/src/DataTypes/IDataType.cpp index c0679557ec9..4b727a49861 100644 --- a/src/DataTypes/IDataType.cpp +++ b/src/DataTypes/IDataType.cpp @@ -26,23 +26,6 @@ namespace ErrorCodes IDataType::~IDataType() = default; -String IDataType::getName() const -{ - if (custom_name) - { - return custom_name->getName(); - } - else - { - return doGetName(); - } -} - -String IDataType::doGetName() const -{ - return getFamilyName(); -} - void IDataType::updateAvgValueSizeHint(const IColumn & column, double & avg_value_size_hint) { /// Update the average value size hint if amount of read rows isn't too small diff --git a/src/DataTypes/IDataType.h b/src/DataTypes/IDataType.h index a7a9f8ffc04..9d943769c0a 100644 --- a/src/DataTypes/IDataType.h +++ b/src/DataTypes/IDataType.h @@ -62,7 +62,13 @@ public: /// static constexpr bool is_parametric = false; /// Name of data type (examples: UInt64, Array(String)). - String getName() const; + String getName() const + { + if (custom_name) + return custom_name->getName(); + else + return doGetName(); + } /// Name of data type family (example: FixedString, Array). virtual const char * getFamilyName() const = 0; @@ -105,7 +111,7 @@ public: void enumerateStreams(const SerializationPtr & serialization, const StreamCallbackWithType & callback) const { enumerateStreams(serialization, callback, {}); } protected: - virtual String doGetName() const; + virtual String doGetName() const { return getFamilyName(); } virtual SerializationPtr doGetDefaultSerialization() const = 0; DataTypePtr getTypeForSubstream(const ISerialization::SubstreamPath & substream_path) const; diff --git a/src/Dictionaries/CassandraDictionarySource.cpp b/src/Dictionaries/CassandraDictionarySource.cpp index aa8d6107508..d9a4dd0fd22 100644 --- a/src/Dictionaries/CassandraDictionarySource.cpp +++ b/src/Dictionaries/CassandraDictionarySource.cpp @@ -17,7 +17,7 @@ void registerDictionarySourceCassandra(DictionarySourceFactory & factory) [[maybe_unused]] const Poco::Util::AbstractConfiguration & config, [[maybe_unused]] const std::string & config_prefix, [[maybe_unused]] Block & sample_block, - ContextPtr /* context */, + ContextPtr /* global_context */, const std::string & /* default_database */, bool /*created_from_ddl*/) -> DictionarySourcePtr { diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index 0f085a7c1a2..b09a7b14cc4 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -63,19 +64,18 @@ ClickHouseDictionarySource::ClickHouseDictionarySource( const DictionaryStructure & dict_struct_, const Configuration & configuration_, const Block & sample_block_, - ContextPtr context_) + ContextMutablePtr context_, + std::shared_ptr local_session_) : update_time{std::chrono::system_clock::from_time_t(0)} , dict_struct{dict_struct_} , configuration{configuration_} , query_builder{dict_struct, configuration.db, "", configuration.table, configuration.query, configuration.where, IdentifierQuotingStyle::Backticks} , sample_block{sample_block_} - , context(Context::createCopy(context_)) + , local_session(local_session_) + , context(context_) , pool{createPool(configuration)} , load_all_query{query_builder.composeLoadAllQuery()} { - /// Query context is needed because some code in executeQuery function may assume it exists. - /// Current example is Context::getSampleBlockCache from InterpreterSelectWithUnionQuery::getSampleBlock. - context->makeQueryContext(); } ClickHouseDictionarySource::ClickHouseDictionarySource(const ClickHouseDictionarySource & other) @@ -85,11 +85,11 @@ ClickHouseDictionarySource::ClickHouseDictionarySource(const ClickHouseDictionar , invalidate_query_response{other.invalidate_query_response} , query_builder{dict_struct, configuration.db, "", configuration.table, configuration.query, configuration.where, IdentifierQuotingStyle::Backticks} , sample_block{other.sample_block} + , local_session(other.local_session) , context(Context::createCopy(other.context)) , pool{createPool(configuration)} , load_all_query{other.load_all_query} { - context->makeQueryContext(); } std::string ClickHouseDictionarySource::getUpdateFieldAndDate() @@ -222,14 +222,13 @@ void registerDictionarySourceClickHouse(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextPtr global_context, const std::string & default_database [[maybe_unused]], bool /* created_from_ddl */) -> DictionarySourcePtr { bool secure = config.getBool(config_prefix + ".secure", false); - auto context_copy = Context::createCopy(context); - UInt16 default_port = getPortFromContext(context_copy, secure); + UInt16 default_port = getPortFromContext(global_context, secure); std::string settings_config_prefix = config_prefix + ".clickhouse"; std::string host = config.getString(settings_config_prefix + ".host", "localhost"); @@ -252,12 +251,18 @@ void registerDictionarySourceClickHouse(DictionarySourceFactory & factory) .secure = config.getBool(settings_config_prefix + ".secure", false) }; - /// We should set user info even for the case when the dictionary is loaded in-process (without TCP communication). + ContextMutablePtr context; + std::shared_ptr local_session; if (configuration.is_local) { - context_copy->setUser(configuration.user, configuration.password, Poco::Net::SocketAddress("127.0.0.1", 0)); - context_copy = copyContextAndApplySettings(config_prefix, context_copy, config); + /// Start local session in case when the dictionary is loaded in-process (without TCP communication). + local_session = std::make_shared(global_context, ClientInfo::Interface::TCP); + local_session->authenticate(configuration.user, configuration.password, Poco::Net::SocketAddress{"127.0.0.1", 0}); + context = local_session->makeQueryContext(); + context->applySettingsChanges(readSettingsFromDictionaryConfig(config, config_prefix)); } + else + context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix); String dictionary_name = config.getString(".dictionary.name", ""); String dictionary_database = config.getString(".dictionary.database", ""); @@ -265,7 +270,7 @@ void registerDictionarySourceClickHouse(DictionarySourceFactory & factory) if (dictionary_name == configuration.table && dictionary_database == configuration.db) throw Exception(ErrorCodes::BAD_ARGUMENTS, "ClickHouseDictionarySource table cannot be dictionary table"); - return std::make_unique(dict_struct, configuration, sample_block, context_copy); + return std::make_unique(dict_struct, configuration, sample_block, context, local_session); }; factory.registerSource("clickhouse", create_table_source); diff --git a/src/Dictionaries/ClickHouseDictionarySource.h b/src/Dictionaries/ClickHouseDictionarySource.h index 2daa296af3e..58243e43b15 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.h +++ b/src/Dictionaries/ClickHouseDictionarySource.h @@ -39,7 +39,8 @@ public: const DictionaryStructure & dict_struct_, const Configuration & configuration_, const Block & sample_block_, - ContextPtr context); + ContextMutablePtr context_, + std::shared_ptr local_session_); /// copy-constructor is provided in order to support cloneability ClickHouseDictionarySource(const ClickHouseDictionarySource & other); @@ -81,6 +82,7 @@ private: mutable std::string invalidate_query_response; ExternalQueryBuilder query_builder; Block sample_block; + std::shared_ptr local_session; ContextMutablePtr context; ConnectionPoolWithFailoverPtr pool; const std::string load_all_query; diff --git a/src/Dictionaries/DictionaryFactory.cpp b/src/Dictionaries/DictionaryFactory.cpp index 62b28ed7d14..4cab42c9445 100644 --- a/src/Dictionaries/DictionaryFactory.cpp +++ b/src/Dictionaries/DictionaryFactory.cpp @@ -31,7 +31,7 @@ DictionaryPtr DictionaryFactory::create( const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, - ContextPtr context, + ContextPtr global_context, bool created_from_ddl) const { Poco::Util::AbstractConfiguration::Keys keys; @@ -45,12 +45,9 @@ DictionaryPtr DictionaryFactory::create( const DictionaryStructure dict_struct{config, config_prefix}; DictionarySourcePtr source_ptr = DictionarySourceFactory::instance().create( - name, config, config_prefix + ".source", dict_struct, context, config.getString(config_prefix + ".database", ""), created_from_ddl); + name, config, config_prefix + ".source", dict_struct, global_context, config.getString(config_prefix + ".database", ""), created_from_ddl); LOG_TRACE(&Poco::Logger::get("DictionaryFactory"), "Created dictionary source '{}' for dictionary '{}'", source_ptr->toString(), name); - if (context->hasQueryContext() && context->getSettingsRef().log_queries) - context->getQueryContext()->addQueryFactoriesInfo(Context::QueryLogFactories::Dictionary, name); - const auto & layout_type = keys.front(); { @@ -58,7 +55,7 @@ DictionaryPtr DictionaryFactory::create( if (found != registered_layouts.end()) { const auto & layout_creator = found->second.layout_create_function; - return layout_creator(name, dict_struct, config, config_prefix, std::move(source_ptr), context, created_from_ddl); + return layout_creator(name, dict_struct, config, config_prefix, std::move(source_ptr), global_context, created_from_ddl); } } @@ -68,10 +65,10 @@ DictionaryPtr DictionaryFactory::create( layout_type); } -DictionaryPtr DictionaryFactory::create(const std::string & name, const ASTCreateQuery & ast, ContextPtr context) const +DictionaryPtr DictionaryFactory::create(const std::string & name, const ASTCreateQuery & ast, ContextPtr global_context) const { - auto configuration = getDictionaryConfigurationFromAST(ast, context); - return DictionaryFactory::create(name, *configuration, "dictionary", context, true); + auto configuration = getDictionaryConfigurationFromAST(ast, global_context); + return DictionaryFactory::create(name, *configuration, "dictionary", global_context, true); } bool DictionaryFactory::isComplex(const std::string & layout_type) const diff --git a/src/Dictionaries/DictionaryFactory.h b/src/Dictionaries/DictionaryFactory.h index b717009aa8a..b1dad340f4b 100644 --- a/src/Dictionaries/DictionaryFactory.h +++ b/src/Dictionaries/DictionaryFactory.h @@ -36,13 +36,13 @@ public: const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, - ContextPtr context, + ContextPtr global_context, bool created_from_ddl) const; /// Create dictionary from DDL-query DictionaryPtr create(const std::string & name, const ASTCreateQuery & ast, - ContextPtr context) const; + ContextPtr global_context) const; using LayoutCreateFunction = std::function; bool isComplex(const std::string & layout_type) const; diff --git a/src/Dictionaries/DictionarySourceFactory.cpp b/src/Dictionaries/DictionarySourceFactory.cpp index 1992c87d31f..80b60f29e37 100644 --- a/src/Dictionaries/DictionarySourceFactory.cpp +++ b/src/Dictionaries/DictionarySourceFactory.cpp @@ -80,7 +80,7 @@ DictionarySourcePtr DictionarySourceFactory::create( const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, const DictionaryStructure & dict_struct, - ContextPtr context, + ContextPtr global_context, const std::string & default_database, bool check_config) const { @@ -99,7 +99,7 @@ DictionarySourcePtr DictionarySourceFactory::create( { const auto & create_source = found->second; auto sample_block = createSampleBlock(dict_struct); - return create_source(dict_struct, config, config_prefix, sample_block, context, default_database, check_config); + return create_source(dict_struct, config, config_prefix, sample_block, global_context, default_database, check_config); } throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, diff --git a/src/Dictionaries/DictionarySourceFactory.h b/src/Dictionaries/DictionarySourceFactory.h index bb583927ac4..f4c3fa12163 100644 --- a/src/Dictionaries/DictionarySourceFactory.h +++ b/src/Dictionaries/DictionarySourceFactory.h @@ -35,7 +35,7 @@ public: const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextPtr global_context, const std::string & default_database, bool check_config)>; @@ -48,7 +48,7 @@ public: const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, const DictionaryStructure & dict_struct, - ContextPtr context, + ContextPtr global_context, const std::string & default_database, bool check_config) const; diff --git a/src/Dictionaries/DictionarySourceHelpers.cpp b/src/Dictionaries/DictionarySourceHelpers.cpp index 092e7187e8f..cf003dceb8e 100644 --- a/src/Dictionaries/DictionarySourceHelpers.cpp +++ b/src/Dictionaries/DictionarySourceHelpers.cpp @@ -59,30 +59,36 @@ Block blockForKeys( return block; } -ContextMutablePtr copyContextAndApplySettings( - const std::string & config_prefix, - ContextPtr context, - const Poco::Util::AbstractConfiguration & config) + +SettingsChanges readSettingsFromDictionaryConfig(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix) { - auto local_context = Context::createCopy(context); - if (config.has(config_prefix + ".settings")) + if (!config.has(config_prefix + ".settings")) + return {}; + + const auto prefix = config_prefix + ".settings"; + + Poco::Util::AbstractConfiguration::Keys config_keys; + config.keys(prefix, config_keys); + + SettingsChanges changes; + + for (const std::string & key : config_keys) { - const auto prefix = config_prefix + ".settings"; - - Poco::Util::AbstractConfiguration::Keys config_keys; - config.keys(prefix, config_keys); - - SettingsChanges changes; - - for (const std::string & key : config_keys) - { - const auto value = config.getString(prefix + "." + key); - changes.emplace_back(key, value); - } - - local_context->applySettingsChanges(changes); + const auto value = config.getString(prefix + "." + key); + changes.emplace_back(key, value); } - return local_context; + + return changes; +} + + +ContextMutablePtr copyContextAndApplySettingsFromDictionaryConfig( + const ContextPtr & context, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix) +{ + auto context_copy = Context::createCopy(context); + auto changes = readSettingsFromDictionaryConfig(config, config_prefix); + context_copy->applySettingsChanges(changes); + return context_copy; } static Block transformHeader(Block header, Block block_to_add) diff --git a/src/Dictionaries/DictionarySourceHelpers.h b/src/Dictionaries/DictionarySourceHelpers.h index b955b6ffb66..5470321745a 100644 --- a/src/Dictionaries/DictionarySourceHelpers.h +++ b/src/Dictionaries/DictionarySourceHelpers.h @@ -14,6 +14,7 @@ namespace DB { struct DictionaryStructure; +class SettingsChanges; /// For simple key @@ -29,10 +30,8 @@ Block blockForKeys( const std::vector & requested_rows); /// Used for applying settings to copied context in some register[...]Source functions -ContextMutablePtr copyContextAndApplySettings( - const std::string & config_prefix, - ContextPtr context, - const Poco::Util::AbstractConfiguration & config); +SettingsChanges readSettingsFromDictionaryConfig(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix); +ContextMutablePtr copyContextAndApplySettingsFromDictionaryConfig(const ContextPtr & context, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix); /** A stream, adds additional columns to each block that it will read from inner stream. * diff --git a/src/Dictionaries/DictionaryStructure.cpp b/src/Dictionaries/DictionaryStructure.cpp index 2fc3a5aa0d0..97c1b98a823 100644 --- a/src/Dictionaries/DictionaryStructure.cpp +++ b/src/Dictionaries/DictionaryStructure.cpp @@ -161,9 +161,6 @@ DictionaryStructure::DictionaryStructure(const Poco::Util::AbstractConfiguration } } - if (attributes.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Dictionary has no attributes defined"); - if (config.getBool(config_prefix + ".layout.ip_trie.access_to_key_from_attributes", false)) access_to_key_from_attributes = true; } diff --git a/src/Dictionaries/DirectDictionary.cpp b/src/Dictionaries/DirectDictionary.cpp index e12100a556d..10e7414b42f 100644 --- a/src/Dictionaries/DirectDictionary.cpp +++ b/src/Dictionaries/DirectDictionary.cpp @@ -307,7 +307,7 @@ namespace const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr /* context */, + ContextPtr /* global_context */, bool /* created_from_ddl */) { const auto * layout_name = dictionary_key_type == DictionaryKeyType::Simple ? "direct" : "complex_key_direct"; diff --git a/src/Dictionaries/ExecutableDictionarySource.cpp b/src/Dictionaries/ExecutableDictionarySource.cpp index 8802d04ff30..5c6add34f1f 100644 --- a/src/Dictionaries/ExecutableDictionarySource.cpp +++ b/src/Dictionaries/ExecutableDictionarySource.cpp @@ -275,7 +275,7 @@ void registerDictionarySourceExecutable(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextPtr global_context, const std::string & /* default_database */, bool created_from_ddl) -> DictionarySourcePtr { @@ -285,10 +285,10 @@ void registerDictionarySourceExecutable(DictionarySourceFactory & factory) /// Executable dictionaries may execute arbitrary commands. /// It's OK for dictionaries created by administrator from xml-file, but /// maybe dangerous for dictionaries created from DDL-queries. - if (created_from_ddl && context->getApplicationType() != Context::ApplicationType::LOCAL) + if (created_from_ddl && global_context->getApplicationType() != Context::ApplicationType::LOCAL) throw Exception(ErrorCodes::DICTIONARY_ACCESS_DENIED, "Dictionaries with executable dictionary source are not allowed to be created from DDL query"); - auto context_local_copy = copyContextAndApplySettings(config_prefix, context, config); + auto context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix); std::string settings_config_prefix = config_prefix + ".executable"; @@ -301,7 +301,7 @@ void registerDictionarySourceExecutable(DictionarySourceFactory & factory) .implicit_key = config.getBool(settings_config_prefix + ".implicit_key", false) }; - return std::make_unique(dict_struct, configuration, sample_block, context_local_copy); + return std::make_unique(dict_struct, configuration, sample_block, context); }; factory.registerSource("executable", create_table_source); diff --git a/src/Dictionaries/ExecutablePoolDictionarySource.cpp b/src/Dictionaries/ExecutablePoolDictionarySource.cpp index e97b4253407..e18664e477c 100644 --- a/src/Dictionaries/ExecutablePoolDictionarySource.cpp +++ b/src/Dictionaries/ExecutablePoolDictionarySource.cpp @@ -279,7 +279,7 @@ void registerDictionarySourceExecutablePool(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextPtr global_context, const std::string & /* default_database */, bool created_from_ddl) -> DictionarySourcePtr { @@ -289,17 +289,15 @@ void registerDictionarySourceExecutablePool(DictionarySourceFactory & factory) /// Executable dictionaries may execute arbitrary commands. /// It's OK for dictionaries created by administrator from xml-file, but /// maybe dangerous for dictionaries created from DDL-queries. - if (created_from_ddl && context->getApplicationType() != Context::ApplicationType::LOCAL) + if (created_from_ddl && global_context->getApplicationType() != Context::ApplicationType::LOCAL) throw Exception(ErrorCodes::DICTIONARY_ACCESS_DENIED, "Dictionaries with executable pool dictionary source are not allowed to be created from DDL query"); - auto context_local_copy = copyContextAndApplySettings(config_prefix, context, config); + ContextMutablePtr context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix); /** Currently parallel parsing input format cannot read exactly max_block_size rows from input, * so it will be blocked on ReadBufferFromFileDescriptor because this file descriptor represent pipe that does not have eof. */ - auto settings_no_parallel_parsing = context_local_copy->getSettings(); - settings_no_parallel_parsing.input_format_parallel_parsing = false; - context_local_copy->setSettings(settings_no_parallel_parsing); + context->setSetting("input_format_parallel_parsing", Field{false}); String settings_config_prefix = config_prefix + ".executable_pool"; @@ -319,7 +317,7 @@ void registerDictionarySourceExecutablePool(DictionarySourceFactory & factory) .implicit_key = config.getBool(settings_config_prefix + ".implicit_key", false), }; - return std::make_unique(dict_struct, configuration, sample_block, context_local_copy); + return std::make_unique(dict_struct, configuration, sample_block, context); }; factory.registerSource("executable_pool", create_table_source); diff --git a/src/Dictionaries/FileDictionarySource.cpp b/src/Dictionaries/FileDictionarySource.cpp index bea14d88d1e..54ce5e4a448 100644 --- a/src/Dictionaries/FileDictionarySource.cpp +++ b/src/Dictionaries/FileDictionarySource.cpp @@ -77,7 +77,7 @@ void registerDictionarySourceFile(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextPtr global_context, const std::string & /* default_database */, bool created_from_ddl) -> DictionarySourcePtr { @@ -87,9 +87,9 @@ void registerDictionarySourceFile(DictionarySourceFactory & factory) const auto filepath = config.getString(config_prefix + ".file.path"); const auto format = config.getString(config_prefix + ".file.format"); - auto context_local_copy = copyContextAndApplySettings(config_prefix, context, config); + const auto context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix); - return std::make_unique(filepath, format, sample_block, context_local_copy, created_from_ddl); + return std::make_unique(filepath, format, sample_block, context, created_from_ddl); }; factory.registerSource("file", create_table_source); diff --git a/src/Dictionaries/FlatDictionary.cpp b/src/Dictionaries/FlatDictionary.cpp index 26667db1081..5ecf3299ea6 100644 --- a/src/Dictionaries/FlatDictionary.cpp +++ b/src/Dictionaries/FlatDictionary.cpp @@ -557,7 +557,7 @@ void registerDictionaryFlat(DictionaryFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr /* context */, + ContextPtr /* global_context */, bool /* created_from_ddl */) -> DictionaryPtr { if (dict_struct.key) diff --git a/src/Dictionaries/HTTPDictionarySource.cpp b/src/Dictionaries/HTTPDictionarySource.cpp index 4a80ebdf975..b5cf59b4474 100644 --- a/src/Dictionaries/HTTPDictionarySource.cpp +++ b/src/Dictionaries/HTTPDictionarySource.cpp @@ -213,13 +213,13 @@ void registerDictionarySourceHTTP(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextPtr global_context, const std::string & /* default_database */, bool created_from_ddl) -> DictionarySourcePtr { if (dict_struct.has_expressions) throw Exception(ErrorCodes::LOGICAL_ERROR, "Dictionary source of type `http` does not support attribute expressions"); - auto context_local_copy = copyContextAndApplySettings(config_prefix, context, config); + auto context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix); const auto & settings_config_prefix = config_prefix + ".http"; const auto & credentials_prefix = settings_config_prefix + ".credentials"; @@ -258,7 +258,7 @@ void registerDictionarySourceHTTP(DictionarySourceFactory & factory) .header_entries = std::move(header_entries) }; - return std::make_unique(dict_struct, configuration, credentials, sample_block, context_local_copy, created_from_ddl); + return std::make_unique(dict_struct, configuration, credentials, sample_block, context, created_from_ddl); }; factory.registerSource("http", create_table_source); } diff --git a/src/Dictionaries/HashedDictionary.cpp b/src/Dictionaries/HashedDictionary.cpp index d462631fba8..fd5865e24c0 100644 --- a/src/Dictionaries/HashedDictionary.cpp +++ b/src/Dictionaries/HashedDictionary.cpp @@ -756,13 +756,13 @@ void registerDictionaryHashed(DictionaryFactory & factory) using namespace std::placeholders; factory.registerLayout("hashed", - [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::Simple, /* sparse = */ false); }, false); + [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* global_context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::Simple, /* sparse = */ false); }, false); factory.registerLayout("sparse_hashed", - [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::Simple, /* sparse = */ true); }, false); + [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* global_context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::Simple, /* sparse = */ true); }, false); factory.registerLayout("complex_key_hashed", - [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::Complex, /* sparse = */ false); }, true); + [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* global_context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::Complex, /* sparse = */ false); }, true); factory.registerLayout("complex_key_sparse_hashed", - [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::Complex, /* sparse = */ true); }, true); + [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* global_context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::Complex, /* sparse = */ true); }, true); } diff --git a/src/Dictionaries/IPAddressDictionary.cpp b/src/Dictionaries/IPAddressDictionary.cpp index fbe911c1d49..b688362d048 100644 --- a/src/Dictionaries/IPAddressDictionary.cpp +++ b/src/Dictionaries/IPAddressDictionary.cpp @@ -954,7 +954,7 @@ void registerDictionaryTrie(DictionaryFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr /* context */, + ContextPtr /* global_context */, bool /*created_from_ddl*/) -> DictionaryPtr { if (!dict_struct.key || dict_struct.key->size() != 1) diff --git a/src/Dictionaries/LibraryDictionarySource.cpp b/src/Dictionaries/LibraryDictionarySource.cpp index 288abde8788..2a47d4c9172 100644 --- a/src/Dictionaries/LibraryDictionarySource.cpp +++ b/src/Dictionaries/LibraryDictionarySource.cpp @@ -41,10 +41,13 @@ LibraryDictionarySource::LibraryDictionarySource( , sample_block{sample_block_} , context(Context::createCopy(context_)) { - if (fs::path(path).is_relative()) - path = fs::canonical(path); + bool path_checked = false; + if (fs::is_symlink(path)) + path_checked = symlinkStartsWith(path, context->getDictionariesLibPath()); + else + path_checked = pathStartsWith(path, context->getDictionariesLibPath()); - if (created_from_ddl && !pathStartsWith(path, context->getDictionariesLibPath())) + if (created_from_ddl && !path_checked) throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File path {} is not inside {}", path, context->getDictionariesLibPath()); if (!fs::exists(path)) @@ -183,11 +186,11 @@ void registerDictionarySourceLibrary(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextPtr global_context, const std::string & /* default_database */, bool created_from_ddl) -> DictionarySourcePtr { - return std::make_unique(dict_struct, config, config_prefix + ".library", sample_block, context, created_from_ddl); + return std::make_unique(dict_struct, config, config_prefix + ".library", sample_block, global_context, created_from_ddl); }; factory.registerSource("library", create_table_source); diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index 2eebb6970d0..bd53c1e60a7 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -31,11 +31,11 @@ void registerDictionarySourceMysql(DictionarySourceFactory & factory) [[maybe_unused]] const Poco::Util::AbstractConfiguration & config, [[maybe_unused]] const std::string & config_prefix, [[maybe_unused]] Block & sample_block, - [[maybe_unused]] ContextPtr context, + [[maybe_unused]] ContextPtr global_context, const std::string & /* default_database */, bool /* created_from_ddl */) -> DictionarySourcePtr { #if USE_MYSQL - StreamSettings mysql_input_stream_settings(context->getSettingsRef() + StreamSettings mysql_input_stream_settings(global_context->getSettingsRef() , config.getBool(config_prefix + ".mysql.close_connection", false) || config.getBool(config_prefix + ".mysql.share_connection", false) , false , config.getBool(config_prefix + ".mysql.fail_on_connection_loss", false) ? 1 : default_num_tries_on_connection_loss); diff --git a/src/Dictionaries/PolygonDictionaryImplementations.cpp b/src/Dictionaries/PolygonDictionaryImplementations.cpp index 7c3eb421a4a..72869ad57ba 100644 --- a/src/Dictionaries/PolygonDictionaryImplementations.cpp +++ b/src/Dictionaries/PolygonDictionaryImplementations.cpp @@ -167,7 +167,7 @@ DictionaryPtr createLayout(const std::string & , const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr /* context */, + ContextPtr /* global_context */, bool /*created_from_ddl*/) { const String database = config.getString(config_prefix + ".database", ""); diff --git a/src/Dictionaries/PostgreSQLDictionarySource.cpp b/src/Dictionaries/PostgreSQLDictionarySource.cpp index 5a546820959..3fe9e899cd9 100644 --- a/src/Dictionaries/PostgreSQLDictionarySource.cpp +++ b/src/Dictionaries/PostgreSQLDictionarySource.cpp @@ -182,7 +182,7 @@ void registerDictionarySourcePostgreSQL(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextPtr global_context, const std::string & /* default_database */, bool /* created_from_ddl */) -> DictionarySourcePtr { @@ -190,8 +190,8 @@ void registerDictionarySourcePostgreSQL(DictionarySourceFactory & factory) const auto settings_config_prefix = config_prefix + ".postgresql"; auto pool = std::make_shared( config, settings_config_prefix, - context->getSettingsRef().postgresql_connection_pool_size, - context->getSettingsRef().postgresql_connection_pool_wait_timeout); + global_context->getSettingsRef().postgresql_connection_pool_size, + global_context->getSettingsRef().postgresql_connection_pool_wait_timeout); PostgreSQLDictionarySource::Configuration configuration { @@ -211,7 +211,7 @@ void registerDictionarySourcePostgreSQL(DictionarySourceFactory & factory) (void)config; (void)config_prefix; (void)sample_block; - (void)context; + (void)global_context; throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Dictionary source of type `postgresql` is disabled because ClickHouse was built without postgresql support."); #endif diff --git a/src/Dictionaries/RangeHashedDictionary.cpp b/src/Dictionaries/RangeHashedDictionary.cpp index ea0af493bdf..390871661c7 100644 --- a/src/Dictionaries/RangeHashedDictionary.cpp +++ b/src/Dictionaries/RangeHashedDictionary.cpp @@ -688,7 +688,7 @@ void registerDictionaryRangeHashed(DictionaryFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr /* context */, + ContextPtr /* global_context */, bool /*created_from_ddl*/) -> DictionaryPtr { if (dict_struct.key) diff --git a/src/Dictionaries/RedisDictionarySource.cpp b/src/Dictionaries/RedisDictionarySource.cpp index 6561a122e9d..24a14d8cc80 100644 --- a/src/Dictionaries/RedisDictionarySource.cpp +++ b/src/Dictionaries/RedisDictionarySource.cpp @@ -12,7 +12,7 @@ void registerDictionarySourceRedis(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const String & config_prefix, Block & sample_block, - ContextPtr /* context */, + ContextPtr /* global_context */, const std::string & /* default_database */, bool /* created_from_ddl */) -> DictionarySourcePtr { return std::make_unique(dict_struct, config, config_prefix + ".redis", sample_block); diff --git a/src/Dictionaries/XDBCDictionarySource.cpp b/src/Dictionaries/XDBCDictionarySource.cpp index e79e55910b7..9fc7e92634b 100644 --- a/src/Dictionaries/XDBCDictionarySource.cpp +++ b/src/Dictionaries/XDBCDictionarySource.cpp @@ -234,12 +234,12 @@ void registerDictionarySourceXDBC(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextPtr global_context, const std::string & /* default_database */, bool /* check_config */) -> DictionarySourcePtr { #if USE_ODBC BridgeHelperPtr bridge = std::make_shared>( - context, context->getSettings().http_receive_timeout, config.getString(config_prefix + ".odbc.connection_string")); + global_context, global_context->getSettings().http_receive_timeout, config.getString(config_prefix + ".odbc.connection_string")); std::string settings_config_prefix = config_prefix + ".odbc"; @@ -255,13 +255,13 @@ void registerDictionarySourceXDBC(DictionarySourceFactory & factory) .update_lag = config.getUInt64(settings_config_prefix + ".update_lag", 1) }; - return std::make_unique(dict_struct, configuration, sample_block, context, bridge); + return std::make_unique(dict_struct, configuration, sample_block, global_context, bridge); #else (void)dict_struct; (void)config; (void)config_prefix; (void)sample_block; - (void)context; + (void)global_context; throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Dictionary source of type `odbc` is disabled because poco library was built without ODBC support."); #endif @@ -276,7 +276,7 @@ void registerDictionarySourceJDBC(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & /* config */, const std::string & /* config_prefix */, Block & /* sample_block */, - ContextPtr /* context */, + ContextPtr /* global_context */, const std::string & /* default_database */, bool /* created_from_ddl */) -> DictionarySourcePtr { throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, diff --git a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp index ba81b1f1364..c8cb2611651 100644 --- a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp +++ b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp @@ -496,9 +496,6 @@ void checkAST(const ASTCreateQuery & query) if (!query.is_dictionary || query.dictionary == nullptr) throw Exception(ErrorCodes::INCORRECT_DICTIONARY_DEFINITION, "Cannot convert dictionary to configuration from non-dictionary AST."); - if (query.dictionary_attributes_list == nullptr || query.dictionary_attributes_list->children.empty()) - throw Exception(ErrorCodes::INCORRECT_DICTIONARY_DEFINITION, "Cannot create dictionary with empty attributes list"); - if (query.dictionary->layout == nullptr) throw Exception(ErrorCodes::INCORRECT_DICTIONARY_DEFINITION, "Cannot create dictionary with empty layout"); @@ -512,8 +509,6 @@ void checkAST(const ASTCreateQuery & query) if (query.dictionary->source == nullptr) throw Exception(ErrorCodes::INCORRECT_DICTIONARY_DEFINITION, "Cannot create dictionary with empty source"); - - /// Range can be empty } void checkPrimaryKey(const NamesToTypeNames & all_attrs, const Names & key_attrs) diff --git a/src/Dictionaries/registerCacheDictionaries.cpp b/src/Dictionaries/registerCacheDictionaries.cpp index 64c1c55e0ba..69197f992f0 100644 --- a/src/Dictionaries/registerCacheDictionaries.cpp +++ b/src/Dictionaries/registerCacheDictionaries.cpp @@ -154,7 +154,7 @@ DictionaryPtr createCacheDictionaryLayout( const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr context [[maybe_unused]], + ContextPtr global_context [[maybe_unused]], bool created_from_ddl [[maybe_unused]]) { String layout_type; @@ -213,8 +213,8 @@ DictionaryPtr createCacheDictionaryLayout( else { auto storage_configuration = parseSSDCacheStorageConfiguration(config, full_name, layout_type, dictionary_layout_prefix, dict_lifetime); - if (created_from_ddl && !pathStartsWith(storage_configuration.file_path, context->getUserFilesPath())) - throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File path {} is not inside {}", storage_configuration.file_path, context->getUserFilesPath()); + if (created_from_ddl && !pathStartsWith(storage_configuration.file_path, global_context->getUserFilesPath())) + throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File path {} is not inside {}", storage_configuration.file_path, global_context->getUserFilesPath()); storage = std::make_shared>(storage_configuration); } @@ -239,10 +239,10 @@ void registerDictionaryCache(DictionaryFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr context, + ContextPtr global_context, bool created_from_ddl) -> DictionaryPtr { - return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), std::move(context), created_from_ddl); + return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), global_context, created_from_ddl); }; factory.registerLayout("cache", create_simple_cache_layout, false); @@ -252,10 +252,10 @@ void registerDictionaryCache(DictionaryFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr context, + ContextPtr global_context, bool created_from_ddl) -> DictionaryPtr { - return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), std::move(context), created_from_ddl); + return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), global_context, created_from_ddl); }; factory.registerLayout("complex_key_cache", create_complex_key_cache_layout, true); @@ -267,10 +267,10 @@ void registerDictionaryCache(DictionaryFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr context, + ContextPtr global_context, bool created_from_ddl) -> DictionaryPtr { - return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), std::move(context), created_from_ddl); + return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), global_context, created_from_ddl); }; factory.registerLayout("ssd_cache", create_simple_ssd_cache_layout, false); @@ -280,9 +280,9 @@ void registerDictionaryCache(DictionaryFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr context, + ContextPtr global_context, bool created_from_ddl) -> DictionaryPtr { - return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), std::move(context), created_from_ddl); + return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), global_context, created_from_ddl); }; factory.registerLayout("complex_key_ssd_cache", create_complex_key_ssd_cache_layout, true); diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index 2c866d3c31c..4907bf6abda 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -958,7 +958,7 @@ public: bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & arguments) const override { return ((IsOperation::div_int || IsOperation::modulo) && !arguments[1].is_const) - || (IsOperation::div_floating && (isDecimal(arguments[0].type) || isDecimal(arguments[1].type))); + || (IsOperation::div_floating && (isDecimalOrNullableDecimal(arguments[0].type) || isDecimalOrNullableDecimal(arguments[1].type))); } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override diff --git a/src/Functions/FunctionHelpers.cpp b/src/Functions/FunctionHelpers.cpp index eac1a7ad1a1..16dd34d0162 100644 --- a/src/Functions/FunctionHelpers.cpp +++ b/src/Functions/FunctionHelpers.cpp @@ -303,4 +303,14 @@ NullPresence getNullPresense(const ColumnsWithTypeAndName & args) return res; } +bool isDecimalOrNullableDecimal(const DataTypePtr & type) +{ + WhichDataType which(type); + if (which.isDecimal()) + return true; + if (!which.isNullable()) + return false; + return isDecimal(assert_cast(type.get())->getNestedType()); +} + } diff --git a/src/Functions/FunctionHelpers.h b/src/Functions/FunctionHelpers.h index 5abe24f4e50..5fc8a06681a 100644 --- a/src/Functions/FunctionHelpers.h +++ b/src/Functions/FunctionHelpers.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -171,4 +172,5 @@ struct NullPresence NullPresence getNullPresense(const ColumnsWithTypeAndName & args); +bool isDecimalOrNullableDecimal(const DataTypePtr & type); } diff --git a/src/Functions/FunctionSnowflake.h b/src/Functions/FunctionSnowflake.h new file mode 100644 index 00000000000..95d02de3a2b --- /dev/null +++ b/src/Functions/FunctionSnowflake.h @@ -0,0 +1,207 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + +/** According to Twitter's post on Snowflake, we can extract the timestamp for a snowflake ID by right shifting + * the snowflake ID by 22 bits(10 bits machine ID and 12 bits sequence ID) and adding the Twitter epoch time of 1288834974657. + * https://en.wikipedia.org/wiki/Snowflake_ID + * https://blog.twitter.com/engineering/en_us/a/2010/announcing-snowflake + * https://ws-dl.blogspot.com/2019/08/2019-08-03-tweetedat-finding-tweet.html +*/ +static constexpr long snowflake_epoch = 1288834974657L; +static constexpr int time_shift = 22; + +class FunctionDateTimeToSnowflake : public IFunction +{ +private: + const char * name; + +public: + FunctionDateTimeToSnowflake(const char * name_) : name(name_) { } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 1; } + bool isVariadic() const override { return false; } + bool useDefaultImplementationForConstants() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (!isDateTime(arguments[0].type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The only argument for function {} must be DateTime", name); + + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto & src = arguments[0]; + const auto & col = *src.column; + + auto res_column = ColumnInt64::create(input_rows_count); + auto & result_data = res_column->getData(); + + const auto & source_data = typeid_cast(col).getData(); + for (size_t i = 0; i < input_rows_count; ++i) + { + result_data[i] = (Int64(source_data[i]) * 1000 - snowflake_epoch) << time_shift; + } + + return res_column; + } +}; + + +class FunctionSnowflakeToDateTime : public IFunction +{ +private: + const char * name; + +public: + FunctionSnowflakeToDateTime(const char * name_) : name(name_) { } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 0; } + bool isVariadic() const override { return true; } + bool useDefaultImplementationForConstants() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (arguments.size() < 1 || arguments.size() > 2) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} takes one or two arguments", name); + + if (!typeid_cast(arguments[0].type.get())) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The first argument for function {} must be Int64", name); + + std::string timezone; + if (arguments.size() == 2) + timezone = extractTimeZoneNameFromFunctionArguments(arguments, 1, 0); + + return std::make_shared(timezone); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto & src = arguments[0]; + const auto & col = *src.column; + + auto res_column = ColumnUInt32::create(input_rows_count); + auto & result_data = res_column->getData(); + + const auto & source_data = typeid_cast(col).getData(); + + for (size_t i = 0; i < input_rows_count; ++i) + { + result_data[i] = ((source_data[i] >> time_shift) + snowflake_epoch) / 1000; + } + return res_column; + } +}; + + +class FunctionDateTime64ToSnowflake : public IFunction +{ +private: + const char * name; + +public: + FunctionDateTime64ToSnowflake(const char * name_) : name(name_) { } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 1; } + bool isVariadic() const override { return false; } + bool useDefaultImplementationForConstants() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (!isDateTime64(arguments[0].type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The only argument for function {} must be DateTime64", name); + + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto & src = arguments[0]; + const auto & col = *src.column; + + auto res_column = ColumnInt64::create(input_rows_count); + auto & result_data = res_column->getData(); + + const auto & source_data = typeid_cast &>(col).getData(); + for (size_t i = 0; i < input_rows_count; ++i) + { + result_data[i] = (source_data[i] - snowflake_epoch) << time_shift; + } + + return res_column; + } +}; + + +class FunctionSnowflakeToDateTime64 : public IFunction +{ +private: + const char * name; + +public: + FunctionSnowflakeToDateTime64(const char * name_) : name(name_) { } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 0; } + bool isVariadic() const override { return true; } + bool useDefaultImplementationForConstants() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (arguments.size() < 1 || arguments.size() > 2) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} takes one or two arguments", name); + + if (!typeid_cast(arguments[0].type.get())) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The first argument for function {} must be Int64", name); + + std::string timezone; + if (arguments.size() == 2) + timezone = extractTimeZoneNameFromFunctionArguments(arguments, 1, 0); + + return std::make_shared(3, timezone); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto & src = arguments[0]; + const auto & col = *src.column; + + auto res_column = ColumnDecimal::create(input_rows_count, 3); + auto & result_data = res_column->getData(); + + const auto & source_data = typeid_cast(col).getData(); + + for (size_t i = 0; i < input_rows_count; ++i) + { + result_data[i] = (source_data[i] >> time_shift) + snowflake_epoch; + } + return res_column; + } +}; + +} diff --git a/src/Functions/in.cpp b/src/Functions/in.cpp index 6cc21e6f50f..87e5886b247 100644 --- a/src/Functions/in.cpp +++ b/src/Functions/in.cpp @@ -104,7 +104,7 @@ public: throw Exception("Second argument for function '" + getName() + "' must be Set; found " + column_set_ptr->getName(), ErrorCodes::ILLEGAL_COLUMN); - DB::Block columns_of_key_columns; + ColumnsWithTypeAndName columns_of_key_columns; /// First argument may be a tuple or a single column. const ColumnWithTypeAndName & left_arg = arguments[0]; @@ -127,16 +127,16 @@ public: const DataTypes & tuple_types = type_tuple->getElements(); size_t tuple_size = tuple_columns.size(); for (size_t i = 0; i < tuple_size; ++i) - columns_of_key_columns.insert({ tuple_columns[i], tuple_types[i], "" }); + columns_of_key_columns.emplace_back(tuple_columns[i], tuple_types[i], "_" + toString(i)); } else - columns_of_key_columns.insert(left_arg); + columns_of_key_columns.emplace_back(left_arg); /// Replace single LowCardinality column to it's dictionary if possible. ColumnPtr lc_indexes = nullptr; - if (columns_of_key_columns.columns() == 1) + if (columns_of_key_columns.size() == 1) { - auto & arg = columns_of_key_columns.safeGetByPosition(0); + auto & arg = columns_of_key_columns.at(0); const auto * col = arg.column.get(); if (const auto * const_col = typeid_cast(col)) col = &const_col->getDataColumn(); diff --git a/src/Functions/registerFunctions.cpp b/src/Functions/registerFunctions.cpp index 7e8f35bc0c4..9b1a7faebbe 100644 --- a/src/Functions/registerFunctions.cpp +++ b/src/Functions/registerFunctions.cpp @@ -51,6 +51,7 @@ void registerFunctionBitHammingDistance(FunctionFactory & factory); void registerFunctionTupleHammingDistance(FunctionFactory & factory); void registerFunctionsStringHash(FunctionFactory & factory); void registerFunctionValidateNestedArraySizes(FunctionFactory & factory); +void registerFunctionsSnowflake(FunctionFactory & factory); #if !defined(ARCADIA_BUILD) void registerFunctionBayesAB(FunctionFactory &); #endif @@ -115,6 +116,7 @@ void registerFunctions() registerFunctionTupleHammingDistance(factory); registerFunctionsStringHash(factory); registerFunctionValidateNestedArraySizes(factory); + registerFunctionsSnowflake(factory); #if !defined(ARCADIA_BUILD) registerFunctionBayesAB(factory); diff --git a/src/Functions/registerFunctionsMiscellaneous.cpp b/src/Functions/registerFunctionsMiscellaneous.cpp index 12c54aeeefd..aed63717155 100644 --- a/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/src/Functions/registerFunctionsMiscellaneous.cpp @@ -78,6 +78,7 @@ void registerFunctionPartitionId(FunctionFactory & factory); void registerFunctionIsIPAddressContainedIn(FunctionFactory &); void registerFunctionQueryID(FunctionFactory & factory); void registerFunctionInitialQueryID(FunctionFactory & factory); +void registerFunctionServerUUID(FunctionFactory &); #if USE_ICU void registerFunctionConvertCharset(FunctionFactory &); @@ -156,6 +157,7 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionIsIPAddressContainedIn(factory); registerFunctionQueryID(factory); registerFunctionInitialQueryID(factory); + registerFunctionServerUUID(factory); #if USE_ICU registerFunctionConvertCharset(factory); diff --git a/src/Functions/registerFunctionsSnowflake.cpp b/src/Functions/registerFunctionsSnowflake.cpp new file mode 100644 index 00000000000..7a0569ee16a --- /dev/null +++ b/src/Functions/registerFunctionsSnowflake.cpp @@ -0,0 +1,22 @@ +namespace DB +{ + +class FunctionFactory; + +void registerDateTimeToSnowflake(FunctionFactory &); +void registerSnowflakeToDateTime(FunctionFactory &); + +void registerDateTime64ToSnowflake(FunctionFactory &); +void registerSnowflakeToDateTime64(FunctionFactory &); + + +void registerFunctionsSnowflake(FunctionFactory & factory) +{ + registerDateTimeToSnowflake(factory); + registerSnowflakeToDateTime(factory); + + registerDateTime64ToSnowflake(factory); + registerSnowflakeToDateTime64(factory); +} + +} diff --git a/src/Functions/serverUUID.cpp b/src/Functions/serverUUID.cpp new file mode 100644 index 00000000000..4b70b1576ac --- /dev/null +++ b/src/Functions/serverUUID.cpp @@ -0,0 +1,60 @@ +#include +#include +#include +#include + + +namespace DB +{ + +namespace +{ + +class FunctionServerUUID : public IFunction + { + public: + static constexpr auto name = "serverUUID"; + + static FunctionPtr create(ContextPtr context) + { + return std::make_shared(context->isDistributed(), ServerUUID::get()); + } + + explicit FunctionServerUUID(bool is_distributed_, UUID server_uuid_) + : is_distributed(is_distributed_), server_uuid(server_uuid_) + { + } + + String getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 0; } + + DataTypePtr getReturnTypeImpl(const DataTypes &) const override { return std::make_shared(); } + + bool isDeterministic() const override { return false; } + + bool isDeterministicInScopeOfQuery() const override { return true; } + + bool isSuitableForConstantFolding() const override { return !is_distributed; } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return false; } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName &, const DataTypePtr &, size_t input_rows_count) const override + { + return DataTypeUUID().createColumnConst(input_rows_count, server_uuid); + } + + private: + bool is_distributed; + const UUID server_uuid; + }; + +} + +void registerFunctionServerUUID(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + diff --git a/src/Functions/snowflake.cpp b/src/Functions/snowflake.cpp new file mode 100644 index 00000000000..5ac1d229d17 --- /dev/null +++ b/src/Functions/snowflake.cpp @@ -0,0 +1,34 @@ +#include +#include + +namespace DB +{ + +void registerDateTimeToSnowflake(FunctionFactory & factory) +{ + factory.registerFunction("dateTimeToSnowflake", + [](ContextPtr){ return std::make_unique( + std::make_shared("dateTimeToSnowflake")); }); +} + +void registerDateTime64ToSnowflake(FunctionFactory & factory) +{ + factory.registerFunction("dateTime64ToSnowflake", + [](ContextPtr){ return std::make_unique( + std::make_shared("dateTime64ToSnowflake")); }); +} + +void registerSnowflakeToDateTime(FunctionFactory & factory) +{ + factory.registerFunction("snowflakeToDateTime", + [](ContextPtr){ return std::make_unique( + std::make_shared("snowflakeToDateTime")); }); +} +void registerSnowflakeToDateTime64(FunctionFactory & factory) +{ + factory.registerFunction("snowflakeToDateTime64", + [](ContextPtr){ return std::make_unique( + std::make_shared("snowflakeToDateTime64")); }); +} + +} diff --git a/src/IO/ReadBufferFromFileDescriptor.cpp b/src/IO/ReadBufferFromFileDescriptor.cpp index fdb538d4a49..e60ec335ca1 100644 --- a/src/IO/ReadBufferFromFileDescriptor.cpp +++ b/src/IO/ReadBufferFromFileDescriptor.cpp @@ -12,6 +12,7 @@ #include #include #include +#include namespace ProfileEvents diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 61e484ff6f1..9a27043160f 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -374,8 +374,8 @@ SetPtr makeExplicitSet( SetPtr set = std::make_shared(size_limits, create_ordered_set, context->getSettingsRef().transform_null_in); - set->setHeader(block.cloneEmpty()); - set->insertFromBlock(block); + set->setHeader(block.cloneEmpty().getColumnsWithTypeAndName()); + set->insertFromBlock(block.getColumnsWithTypeAndName()); set->finishInsert(); prepared_sets[set_key] = set; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index bd15af76db0..583f59c20b5 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -59,6 +59,7 @@ #include #include #include +#include #include #include #include @@ -100,7 +101,6 @@ namespace CurrentMetrics extern const Metric BackgroundMessageBrokerSchedulePoolTask; } - namespace DB { @@ -115,189 +115,11 @@ namespace ErrorCodes extern const int THERE_IS_NO_QUERY; extern const int NO_ELEMENTS_IN_CONFIG; extern const int TABLE_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT; - extern const int SESSION_NOT_FOUND; - extern const int SESSION_IS_LOCKED; extern const int LOGICAL_ERROR; extern const int NOT_IMPLEMENTED; } -class NamedSessions -{ -public: - using Key = NamedSessionKey; - - ~NamedSessions() - { - try - { - { - std::lock_guard lock{mutex}; - quit = true; - } - - cond.notify_one(); - thread.join(); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } - } - - /// Find existing session or create a new. - std::shared_ptr acquireSession( - const String & session_id, - ContextMutablePtr context, - std::chrono::steady_clock::duration timeout, - bool throw_if_not_found) - { - std::unique_lock lock(mutex); - - auto & user_name = context->client_info.current_user; - - if (user_name.empty()) - throw Exception("Empty user name.", ErrorCodes::LOGICAL_ERROR); - - Key key(user_name, session_id); - - auto it = sessions.find(key); - if (it == sessions.end()) - { - if (throw_if_not_found) - throw Exception("Session not found.", ErrorCodes::SESSION_NOT_FOUND); - - /// Create a new session from current context. - it = sessions.insert(std::make_pair(key, std::make_shared(key, context, timeout, *this))).first; - } - else if (it->second->key.first != context->client_info.current_user) - { - throw Exception("Session belongs to a different user", ErrorCodes::SESSION_IS_LOCKED); - } - - /// Use existing session. - const auto & session = it->second; - - if (!session.unique()) - throw Exception("Session is locked by a concurrent client.", ErrorCodes::SESSION_IS_LOCKED); - - session->context->client_info = context->client_info; - - return session; - } - - void releaseSession(NamedSession & session) - { - std::unique_lock lock(mutex); - scheduleCloseSession(session, lock); - } - -private: - class SessionKeyHash - { - public: - size_t operator()(const Key & key) const - { - SipHash hash; - hash.update(key.first); - hash.update(key.second); - return hash.get64(); - } - }; - - /// TODO it's very complicated. Make simple std::map with time_t or boost::multi_index. - using Container = std::unordered_map, SessionKeyHash>; - using CloseTimes = std::deque>; - Container sessions; - CloseTimes close_times; - std::chrono::steady_clock::duration close_interval = std::chrono::seconds(1); - std::chrono::steady_clock::time_point close_cycle_time = std::chrono::steady_clock::now(); - UInt64 close_cycle = 0; - - void scheduleCloseSession(NamedSession & session, std::unique_lock &) - { - /// Push it on a queue of sessions to close, on a position corresponding to the timeout. - /// (timeout is measured from current moment of time) - - const UInt64 close_index = session.timeout / close_interval + 1; - const auto new_close_cycle = close_cycle + close_index; - - if (session.close_cycle != new_close_cycle) - { - session.close_cycle = new_close_cycle; - if (close_times.size() < close_index + 1) - close_times.resize(close_index + 1); - close_times[close_index].emplace_back(session.key); - } - } - - void cleanThread() - { - setThreadName("SessionCleaner"); - std::unique_lock lock{mutex}; - - while (true) - { - auto interval = closeSessions(lock); - - if (cond.wait_for(lock, interval, [this]() -> bool { return quit; })) - break; - } - } - - /// Close sessions, that has been expired. Returns how long to wait for next session to be expired, if no new sessions will be added. - std::chrono::steady_clock::duration closeSessions(std::unique_lock & lock) - { - const auto now = std::chrono::steady_clock::now(); - - /// The time to close the next session did not come - if (now < close_cycle_time) - return close_cycle_time - now; /// Will sleep until it comes. - - const auto current_cycle = close_cycle; - - ++close_cycle; - close_cycle_time = now + close_interval; - - if (close_times.empty()) - return close_interval; - - auto & sessions_to_close = close_times.front(); - - for (const auto & key : sessions_to_close) - { - const auto session = sessions.find(key); - - if (session != sessions.end() && session->second->close_cycle <= current_cycle) - { - if (!session->second.unique()) - { - /// Skip but move it to close on the next cycle. - session->second->timeout = std::chrono::steady_clock::duration{0}; - scheduleCloseSession(*session->second, lock); - } - else - sessions.erase(session); - } - } - - close_times.pop_front(); - return close_interval; - } - - std::mutex mutex; - std::condition_variable cond; - std::atomic quit{false}; - ThreadFromGlobalPool thread{&NamedSessions::cleanThread, this}; -}; - - -void NamedSession::release() -{ - parent.releaseSession(*this); -} - - /** Set of known objects (environment), that could be used in query. * Shared (global) part. Order of members (especially, order of destruction) is very important. */ @@ -359,7 +181,7 @@ struct ContextSharedPart String default_profile_name; /// Default profile name used for default values. String system_profile_name; /// Profile used by system processes String buffer_profile_name; /// Profile used by Buffer engine for flushing to the underlying - AccessControlManager access_control_manager; + std::unique_ptr access_control_manager; mutable UncompressedCachePtr uncompressed_cache; /// The cache of decompressed blocks. mutable MarkCachePtr mark_cache; /// Cache of marks in compressed files. mutable MMappedFileCachePtr mmap_cache; /// Cache of mmapped files to avoid frequent open/map/unmap/close and to reuse from several threads. @@ -399,7 +221,6 @@ struct ContextSharedPart RemoteHostFilter remote_host_filter; /// Allowed URL from config.xml std::optional trace_collector; /// Thread collecting traces from threads executing queries - std::optional named_sessions; /// Controls named HTTP sessions. /// Clusters for distributed tables /// Initialized on demand (on distributed storages initialization) since Settings should be initialized @@ -419,7 +240,7 @@ struct ContextSharedPart Context::ConfigReloadCallback config_reload_callback; ContextSharedPart() - : macros(std::make_unique()) + : access_control_manager(std::make_unique()), macros(std::make_unique()) { /// TODO: make it singleton (?) static std::atomic num_calls{0}; @@ -453,6 +274,8 @@ struct ContextSharedPart return; shutdown_called = true; + Session::shutdownNamedSessions(); + /** After system_logs have been shut down it is guaranteed that no system table gets created or written to. * Note that part changes at shutdown won't be logged to part log. */ @@ -498,6 +321,7 @@ struct ContextSharedPart distributed_schedule_pool.reset(); message_broker_schedule_pool.reset(); ddl_worker.reset(); + access_control_manager.reset(); /// Stop trace collector if any trace_collector.reset(); @@ -587,7 +411,6 @@ void Context::copyFrom(const ContextPtr & other) Context::~Context() = default; - InterserverIOHandler & Context::getInterserverIOHandler() { return shared->interserver_io_handler; } std::unique_lock Context::getLock() const @@ -604,21 +427,6 @@ const MergeList & Context::getMergeList() const { return shared->merge_list; } ReplicatedFetchList & Context::getReplicatedFetchList() { return shared->replicated_fetch_list; } const ReplicatedFetchList & Context::getReplicatedFetchList() const { return shared->replicated_fetch_list; } - -void Context::enableNamedSessions() -{ - shared->named_sessions.emplace(); -} - -std::shared_ptr -Context::acquireNamedSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check) -{ - if (!shared->named_sessions) - throw Exception("Support for named sessions is not enabled", ErrorCodes::NOT_IMPLEMENTED); - - return shared->named_sessions->acquireSession(session_id, shared_from_this(), timeout, session_check); -} - String Context::resolveDatabase(const String & database_name) const { String res = database_name.empty() ? getCurrentDatabase() : database_name; @@ -738,7 +546,7 @@ void Context::setConfig(const ConfigurationPtr & config) { auto lock = getLock(); shared->config = config; - shared->access_control_manager.setExternalAuthenticatorsConfig(*shared->config); + shared->access_control_manager->setExternalAuthenticatorsConfig(*shared->config); } const Poco::Util::AbstractConfiguration & Context::getConfigRef() const @@ -750,31 +558,31 @@ const Poco::Util::AbstractConfiguration & Context::getConfigRef() const AccessControlManager & Context::getAccessControlManager() { - return shared->access_control_manager; + return *shared->access_control_manager; } const AccessControlManager & Context::getAccessControlManager() const { - return shared->access_control_manager; + return *shared->access_control_manager; } void Context::setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config) { auto lock = getLock(); - shared->access_control_manager.setExternalAuthenticatorsConfig(config); + shared->access_control_manager->setExternalAuthenticatorsConfig(config); } std::unique_ptr Context::makeGSSAcceptorContext() const { auto lock = getLock(); - return std::make_unique(shared->access_control_manager.getExternalAuthenticators().getKerberosParams()); + return std::make_unique(shared->access_control_manager->getExternalAuthenticators().getKerberosParams()); } void Context::setUsersConfig(const ConfigurationPtr & config) { auto lock = getLock(); shared->users_config = config; - shared->access_control_manager.setUsersConfig(*shared->users_config); + shared->access_control_manager->setUsersConfig(*shared->users_config); } ConfigurationPtr Context::getUsersConfig() @@ -784,48 +592,24 @@ ConfigurationPtr Context::getUsersConfig() } -void Context::setUser(const Credentials & credentials, const Poco::Net::SocketAddress & address) +void Context::setUser(const UUID & user_id_) { auto lock = getLock(); - client_info.current_user = credentials.getUserName(); - client_info.current_address = address; + user_id = user_id_; -#if defined(ARCADIA_BUILD) - /// This is harmful field that is used only in foreign "Arcadia" build. - client_info.current_password.clear(); - if (const auto * basic_credentials = dynamic_cast(&credentials)) - client_info.current_password = basic_credentials->getPassword(); -#endif - - /// Find a user with such name and check the credentials. - auto new_user_id = getAccessControlManager().login(credentials, address.host()); - auto new_access = getAccessControlManager().getContextAccess( - new_user_id, /* current_roles = */ {}, /* use_default_roles = */ true, - settings, current_database, client_info); - - user_id = new_user_id; - access = std::move(new_access); + access = getAccessControlManager().getContextAccess( + user_id_, /* current_roles = */ {}, /* use_default_roles = */ true, settings, current_database, client_info); auto user = access->getUser(); current_roles = std::make_shared>(user->granted_roles.findGranted(user->default_roles)); - if (!user->default_database.empty()) - setCurrentDatabase(user->default_database); - auto default_profile_info = access->getDefaultProfileInfo(); settings_constraints_and_current_profiles = default_profile_info->getConstraintsAndProfileIDs(); applySettingsChanges(default_profile_info->settings); -} -void Context::setUser(const String & name, const String & password, const Poco::Net::SocketAddress & address) -{ - setUser(BasicCredentials(name, password), address); -} - -void Context::setUserWithoutCheckingPassword(const String & name, const Poco::Net::SocketAddress & address) -{ - setUser(AlwaysAllowCredentials(name), address); + if (!user->default_database.empty()) + setCurrentDatabase(user->default_database); } std::shared_ptr Context::getUser() const @@ -833,12 +617,6 @@ std::shared_ptr Context::getUser() const return getAccess()->getUser(); } -void Context::setQuotaKey(String quota_key_) -{ - auto lock = getLock(); - client_info.quota_key = std::move(quota_key_); -} - String Context::getUserName() const { return getAccess()->getUserName(); @@ -851,6 +629,13 @@ std::optional Context::getUserID() const } +void Context::setQuotaKey(String quota_key_) +{ + auto lock = getLock(); + client_info.quota_key = std::move(quota_key_); +} + + void Context::setCurrentRoles(const std::vector & current_roles_) { auto lock = getLock(); @@ -932,10 +717,13 @@ ASTPtr Context::getRowPolicyCondition(const String & database, const String & ta void Context::setInitialRowPolicy() { auto lock = getLock(); - auto initial_user_id = getAccessControlManager().find(client_info.initial_user); initial_row_policy = nullptr; - if (initial_user_id) - initial_row_policy = getAccessControlManager().getEnabledRowPolicies(*initial_user_id, {}); + if (client_info.initial_user == client_info.current_user) + return; + auto initial_user_id = getAccessControlManager().find(client_info.initial_user); + if (!initial_user_id) + return; + initial_row_policy = getAccessControlManager().getEnabledRowPolicies(*initial_user_id, {}); } @@ -1376,6 +1164,9 @@ void Context::setCurrentQueryId(const String & query_id) } client_info.current_query_id = query_id_to_set; + + if (client_info.query_kind == ClientInfo::QueryKind::INITIAL_QUERY) + client_info.initial_query_id = client_info.current_query_id; } void Context::killCurrentQuery() @@ -2012,8 +1803,8 @@ std::shared_ptr Context::getCluster(const std::string & cluster_name) c auto res = getClusters()->getCluster(cluster_name); if (res) return res; - - res = tryGetReplicatedDatabaseCluster(cluster_name); + if (!cluster_name.empty()) + res = tryGetReplicatedDatabaseCluster(cluster_name); if (res) return res; diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index d3a77e0039b..1b636deb532 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -14,21 +14,16 @@ #include #include #include -#include #include #if !defined(ARCADIA_BUILD) # include "config_core.h" #endif -#include -#include -#include #include #include #include #include -#include namespace Poco::Net { class IPAddress; } @@ -67,6 +62,7 @@ class ProcessList; class QueryStatus; class Macros; struct Progress; +struct FileProgress; class Clusters; class QueryLog; class QueryThreadLog; @@ -107,6 +103,7 @@ using StoragePolicySelectorPtr = std::shared_ptr; struct PartUUIDs; using PartUUIDsPtr = std::shared_ptr; class KeeperStorageDispatcher; +class Session; class IOutputFormat; using OutputFormatPtr = std::shared_ptr; @@ -287,8 +284,6 @@ public: OpenTelemetryTraceContext query_trace_context; private: - friend class NamedSessions; - using SampleBlockCache = std::unordered_map; mutable SampleBlockCache sample_block_cache; @@ -367,23 +362,17 @@ public: void setUsersConfig(const ConfigurationPtr & config); ConfigurationPtr getUsersConfig(); - /// Sets the current user, checks the credentials and that the specified host is allowed. - /// Must be called before getClientInfo() can be called. - void setUser(const Credentials & credentials, const Poco::Net::SocketAddress & address); - void setUser(const String & name, const String & password, const Poco::Net::SocketAddress & address); - - /// Sets the current user, *does not check the password/credentials and that the specified host is allowed*. - /// Must be called before getClientInfo. - /// - /// (Used only internally in cluster, if the secret matches) - void setUserWithoutCheckingPassword(const String & name, const Poco::Net::SocketAddress & address); - - void setQuotaKey(String quota_key_); + /// Sets the current user assuming that he/she is already authenticated. + /// WARNING: This function doesn't check password! + /// Normally you shouldn't call this function. Use the Session class to do authentication instead. + void setUser(const UUID & user_id_); UserPtr getUser() const; String getUserName() const; std::optional getUserID() const; + void setQuotaKey(String quota_key_); + void setCurrentRoles(const std::vector & current_roles_); void setCurrentRolesDefault(); boost::container::flat_set getCurrentRoles() const; @@ -591,12 +580,6 @@ public: std::optional getTCPPortSecure() const; - /// Allow to use named sessions. The thread will be run to cleanup sessions after timeout has expired. - /// The method must be called at the server startup. - void enableNamedSessions(); - - std::shared_ptr acquireNamedSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check); - /// For methods below you may need to acquire the context lock by yourself. ContextMutablePtr getQueryContext() const; @@ -852,32 +835,6 @@ private: StoragePolicySelectorPtr getStoragePolicySelector(std::lock_guard & lock) const; DiskSelectorPtr getDiskSelector(std::lock_guard & /* lock */) const; - - /// If the password is not set, the password will not be checked - void setUserImpl(const String & name, const std::optional & password, const Poco::Net::SocketAddress & address); -}; - - -class NamedSessions; - -/// User name and session identifier. Named sessions are local to users. -using NamedSessionKey = std::pair; - -/// Named sessions. The user could specify session identifier to reuse settings and temporary tables in subsequent requests. -struct NamedSession -{ - NamedSessionKey key; - UInt64 close_cycle = 0; - ContextMutablePtr context; - std::chrono::steady_clock::duration timeout; - NamedSessions & parent; - - NamedSession(NamedSessionKey key_, ContextPtr context_, std::chrono::steady_clock::duration timeout_, NamedSessions & parent_) - : key(key_), context(Context::createCopy(context_)), timeout(timeout_), parent(parent_) - { - } - - void release(); }; } diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 856d8713560..c2de6ecbaf1 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -634,7 +634,7 @@ void DDLWorker::processTask(DDLTaskBase & task, const ZooKeeperPtr & zookeeper) String dummy; if (zookeeper->tryGet(active_node_path, dummy, nullptr, eph_node_disappeared)) { - constexpr int timeout_ms = 30 * 1000; + constexpr int timeout_ms = 60 * 1000; if (!eph_node_disappeared->tryWait(timeout_ms)) throw Exception(ErrorCodes::LOGICAL_ERROR, "Ephemeral node {} still exists, " "probably it's owned by someone else", active_node_path); diff --git a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp index f46cbdd2465..2b858512b98 100644 --- a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp +++ b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp @@ -80,9 +80,13 @@ void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr Block scalar; if (data.getContext()->hasQueryContext() && data.getContext()->getQueryContext()->hasScalar(scalar_query_hash_str)) + { scalar = data.getContext()->getQueryContext()->getScalar(scalar_query_hash_str); + } else if (data.scalars.count(scalar_query_hash_str)) + { scalar = data.scalars[scalar_query_hash_str]; + } else { auto subquery_context = Context::createCopy(data.getContext()); @@ -149,7 +153,8 @@ void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr throw Exception("Scalar subquery returned more than one row", ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY); Block tmp_block; - while (tmp_block.rows() == 0 && executor.pull(tmp_block)); + while (tmp_block.rows() == 0 && executor.pull(tmp_block)) + ; if (tmp_block.rows() != 0) throw Exception("Scalar subquery returned more than one row", ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY); @@ -173,10 +178,10 @@ void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr } else { - ColumnWithTypeAndName ctn; - ctn.type = std::make_shared(block.getDataTypes()); - ctn.column = ColumnTuple::create(block.getColumns()); - scalar.insert(ctn); + scalar.insert({ + ColumnTuple::create(block.getColumns()), + std::make_shared(block.getDataTypes()), + "tuple"}); } } diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 5b25ad9609e..f3c51487a05 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -1037,12 +1037,15 @@ ExpressionActionsChain::JoinStep::JoinStep( : Step({}) , analyzed_join(std::move(analyzed_join_)) , join(std::move(join_)) - , result_columns(std::move(required_columns_)) { - for (const auto & column : result_columns) + for (const auto & column : required_columns_) required_columns.emplace_back(column.name, column.type); - analyzed_join->addJoinedColumnsAndCorrectTypes(result_columns); + NamesAndTypesList result_names_and_types = required_columns; + analyzed_join->addJoinedColumnsAndCorrectTypes(result_names_and_types); + for (const auto & [name, type] : result_names_and_types) + /// `column` is `nullptr` because we don't care on constness here, it may be changed in join + result_columns.emplace_back(nullptr, type, name); } void ExpressionActionsChain::JoinStep::finalize(const NameSet & required_output_) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 77598e69c00..dbb1addf202 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -162,6 +162,36 @@ ExpressionAnalyzer::ExpressionAnalyzer( analyzeAggregation(); } +static ASTPtr checkPositionalArgument(ASTPtr argument, const ASTSelectQuery * select_query, ASTSelectQuery::Expression expression) +{ + auto columns = select_query->select()->children; + + /// Case when GROUP BY element is position. + /// Do not consider case when GROUP BY element is not a literal, but expression, even if all values are constants. + if (const auto * ast_literal = typeid_cast(argument.get())) + { + auto which = ast_literal->value.getType(); + if (which == Field::Types::UInt64) + { + auto pos = ast_literal->value.get(); + if (pos > 0 && pos <= columns.size()) + { + const auto & column = columns[--pos]; + if (const auto * literal_ast = typeid_cast(column.get())) + { + return std::make_shared(literal_ast->name()); + } + else + { + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal value for positional argument in {}", + ASTSelectQuery::expressionToString(expression)); + } + } + /// Do not throw if out of bounds, see appendUnusedGroupByColumn. + } + } + return nullptr; +} void ExpressionAnalyzer::analyzeAggregation() { @@ -216,7 +246,7 @@ void ExpressionAnalyzer::analyzeAggregation() if (join) { getRootActionsNoMakeSet(analyzedJoin().leftKeysList(), true, temp_actions, false); - auto sample_columns = temp_actions->getResultColumns(); + auto sample_columns = temp_actions->getNamesAndTypesList(); analyzedJoin().addJoinedColumnsAndCorrectTypes(sample_columns); temp_actions = std::make_shared(sample_columns); } @@ -238,13 +268,22 @@ void ExpressionAnalyzer::analyzeAggregation() { NameSet unique_keys; ASTs & group_asts = select_query->groupBy()->children; + for (ssize_t i = 0; i < ssize_t(group_asts.size()); ++i) { ssize_t size = group_asts.size(); getRootActionsNoMakeSet(group_asts[i], true, temp_actions, false); + if (getContext()->getSettingsRef().enable_positional_arguments) + { + auto new_argument = checkPositionalArgument(group_asts[i], select_query, ASTSelectQuery::Expression::GROUP_BY); + if (new_argument) + group_asts[i] = new_argument; + } + const auto & column_name = group_asts[i]->getColumnName(); const auto * node = temp_actions->tryFindInIndex(column_name); + if (!node) throw Exception("Unknown identifier (in GROUP BY): " + column_name, ErrorCodes::UNKNOWN_IDENTIFIER); @@ -337,7 +376,7 @@ void ExpressionAnalyzer::tryMakeSetForIndexFromSubquery(const ASTPtr & subquery_ PullingAsyncPipelineExecutor executor(io.pipeline); SetPtr set = std::make_shared(settings.size_limits_for_set, true, getContext()->getSettingsRef().transform_null_in); - set->setHeader(executor.getHeader()); + set->setHeader(executor.getHeader().getColumnsWithTypeAndName()); Block block; while (executor.pull(block)) @@ -346,7 +385,7 @@ void ExpressionAnalyzer::tryMakeSetForIndexFromSubquery(const ASTPtr & subquery_ continue; /// If the limits have been exceeded, give up and let the default subquery processing actions take place. - if (!set->insertFromBlock(block)) + if (!set->insertFromBlock(block.getColumnsWithTypeAndName())) return; } @@ -1206,7 +1245,7 @@ void SelectQueryExpressionAnalyzer::appendSelect(ExpressionActionsChain & chain, } ActionsDAGPtr SelectQueryExpressionAnalyzer::appendOrderBy(ExpressionActionsChain & chain, bool only_types, bool optimize_read_in_order, - ManyExpressionActions & order_by_elements_actions) + ManyExpressionActions & order_by_elements_actions) { const auto * select_query = getSelectQuery(); @@ -1223,11 +1262,20 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendOrderBy(ExpressionActionsChai bool with_fill = false; NameSet order_by_keys; + for (auto & child : select_query->orderBy()->children) { - const auto * ast = child->as(); + auto * ast = child->as(); if (!ast || ast->children.empty()) throw Exception("Bad order expression AST", ErrorCodes::UNKNOWN_TYPE_OF_AST_NODE); + + if (getContext()->getSettingsRef().enable_positional_arguments) + { + auto new_argument = checkPositionalArgument(ast->children.at(0), select_query, ASTSelectQuery::Expression::ORDER_BY); + if (new_argument) + ast->children[0] = new_argument; + } + ASTPtr order_expression = ast->children.at(0); step.addRequiredOutput(order_expression->getColumnName()); @@ -1277,8 +1325,16 @@ bool SelectQueryExpressionAnalyzer::appendLimitBy(ExpressionActionsChain & chain aggregated_names.insert(column.name); } - for (const auto & child : select_query->limitBy()->children) + auto & children = select_query->limitBy()->children; + for (auto & child : children) { + if (getContext()->getSettingsRef().enable_positional_arguments) + { + auto new_argument = checkPositionalArgument(child, select_query, ASTSelectQuery::Expression::LIMIT_BY); + if (new_argument) + child = new_argument; + } + auto child_name = child->getColumnName(); if (!aggregated_names.count(child_name)) step.addRequiredOutput(std::move(child_name)); diff --git a/src/Interpreters/ExternalDictionariesLoader.cpp b/src/Interpreters/ExternalDictionariesLoader.cpp index 83931649443..cbb0e52b91b 100644 --- a/src/Interpreters/ExternalDictionariesLoader.cpp +++ b/src/Interpreters/ExternalDictionariesLoader.cpp @@ -45,12 +45,20 @@ ExternalLoader::LoadablePtr ExternalDictionariesLoader::create( ExternalDictionariesLoader::DictPtr ExternalDictionariesLoader::getDictionary(const std::string & dictionary_name, ContextPtr local_context) const { std::string resolved_dictionary_name = resolveDictionaryName(dictionary_name, local_context->getCurrentDatabase()); + + if (local_context->hasQueryContext() && local_context->getSettingsRef().log_queries) + local_context->addQueryFactoriesInfo(Context::QueryLogFactories::Dictionary, resolved_dictionary_name); + return std::static_pointer_cast(load(resolved_dictionary_name)); } ExternalDictionariesLoader::DictPtr ExternalDictionariesLoader::tryGetDictionary(const std::string & dictionary_name, ContextPtr local_context) const { std::string resolved_dictionary_name = resolveDictionaryName(dictionary_name, local_context->getCurrentDatabase()); + + if (local_context->hasQueryContext() && local_context->getSettingsRef().log_queries) + local_context->addQueryFactoriesInfo(Context::QueryLogFactories::Dictionary, resolved_dictionary_name); + return std::static_pointer_cast(tryLoad(resolved_dictionary_name)); } diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index e0aad706966..46f8c9aac79 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -21,7 +21,6 @@ #include -#include #include #include @@ -194,6 +193,13 @@ HashJoin::HashJoin(std::shared_ptr table_join_, const Block & right_s required_right_keys = table_join->getRequiredRightKeys(right_table_keys, required_right_keys_sources); + LOG_DEBUG(log, "Right keys: [{}] (required: [{}]), left keys: [{}]", + fmt::join(key_names_right, ", "), + fmt::join(required_right_keys.getNames(), ", "), + fmt::join(table_join->keyNamesLeft(), ", ")); + + LOG_DEBUG(log, "Columns to add: [{}]", sample_block_with_columns_to_add.dumpStructure()); + std::tie(condition_mask_column_name_left, condition_mask_column_name_right) = table_join->joinConditionColumnNames(); JoinCommon::removeLowCardinalityInplace(right_table_keys); @@ -629,7 +635,7 @@ bool HashJoin::addJoinedBlock(const Block & source_block, bool check_limits) ConstNullMapPtr null_map{}; ColumnPtr null_map_holder = extractNestedColumnsAndNullMap(key_columns, null_map); - /// If RIGHT or FULL save blocks with nulls for NonJoinedBlockInputStream + /// If RIGHT or FULL save blocks with nulls for NotJoinedBlocks UInt8 save_nullmap = 0; if (isRightOrFull(kind) && null_map) { @@ -1468,40 +1474,17 @@ struct AdderNonJoined /// Stream from not joined earlier rows of the right table. -class NonJoinedBlockInputStream : private NotJoined, public IBlockInputStream +class NotJoinedHash final : public NotJoinedBlocks::RightColumnsFiller { public: - NonJoinedBlockInputStream(const HashJoin & parent_, const Block & result_sample_block_, UInt64 max_block_size_) - : NotJoined(*parent_.table_join, - parent_.savedBlockSample(), - parent_.right_sample_block, - result_sample_block_) - , parent(parent_) - , max_block_size(max_block_size_) + NotJoinedHash(const HashJoin & parent_, UInt64 max_block_size_) + : parent(parent_), max_block_size(max_block_size_) {} - String getName() const override { return "NonJoined"; } - Block getHeader() const override { return result_sample_block; } + Block getEmptyBlock() override { return parent.savedBlockSample().cloneEmpty(); } -protected: - Block readImpl() override + size_t fillColumns(MutableColumns & columns_right) override { - if (parent.data->blocks.empty()) - return Block(); - return createBlock(); - } - -private: - const HashJoin & parent; - UInt64 max_block_size; - - std::any position; - std::optional nulls_position; - - Block createBlock() - { - MutableColumns columns_right = saved_block_sample.cloneEmptyColumns(); - size_t rows_added = 0; auto fill_callback = [&](auto, auto strictness, auto & map) @@ -1513,22 +1496,16 @@ private: throw Exception("Logical error: unknown JOIN strictness (must be on of: ANY, ALL, ASOF)", ErrorCodes::LOGICAL_ERROR); fillNullsFromBlocks(columns_right, rows_added); - if (!rows_added) - return {}; - - Block res = result_sample_block.cloneEmpty(); - addLeftColumns(res, rows_added); - addRightColumns(res, columns_right); - copySameKeys(res); - correctLowcardAndNullability(res); - -#ifndef NDEBUG - assertBlocksHaveEqualStructure(res, result_sample_block, getName()); -#endif - - return res; + return rows_added; } +private: + const HashJoin & parent; + UInt64 max_block_size; + + std::any position; + std::optional nulls_position; + template size_t fillColumnsFromMap(const Maps & maps, MutableColumns & columns_keys_and_right) { @@ -1607,15 +1584,18 @@ private: }; -BlockInputStreamPtr HashJoin::createStreamWithNonJoinedRows(const Block & result_sample_block, UInt64 max_block_size) const +std::shared_ptr HashJoin::getNonJoinedBlocks(const Block & result_sample_block, UInt64 max_block_size) const { if (table_join->strictness() == ASTTableJoin::Strictness::Asof || - table_join->strictness() == ASTTableJoin::Strictness::Semi) + table_join->strictness() == ASTTableJoin::Strictness::Semi || + !isRightOrFull(table_join->kind())) + { return {}; + } - if (isRightOrFull(table_join->kind())) - return std::make_shared(*this, result_sample_block, max_block_size); - return {}; + size_t left_columns_count = result_sample_block.columns() - required_right_keys.columns() - sample_block_with_columns_to_add.columns(); + auto non_joined = std::make_unique(*this, max_block_size); + return std::make_shared(std::move(non_joined), result_sample_block, left_columns_count, table_join->leftToRightKeyRemap()); } void HashJoin::reuseJoinedData(const HashJoin & join) diff --git a/src/Interpreters/HashJoin.h b/src/Interpreters/HashJoin.h index 65e3f5dbabe..2e691f189c4 100644 --- a/src/Interpreters/HashJoin.h +++ b/src/Interpreters/HashJoin.h @@ -20,7 +20,6 @@ #include #include -#include #include @@ -164,7 +163,7 @@ public: * Use only after all calls to joinBlock was done. * left_sample_block is passed without account of 'use_nulls' setting (columns will be converted to Nullable inside). */ - BlockInputStreamPtr createStreamWithNonJoinedRows(const Block & result_sample_block, UInt64 max_block_size) const override; + std::shared_ptr getNonJoinedBlocks(const Block & result_sample_block, UInt64 max_block_size) const override; /// Number of keys in all built JOIN maps. size_t getTotalRowCount() const final; @@ -337,7 +336,7 @@ public: bool isUsed(size_t off) const { return used_flags.getUsedSafe(off); } private: - friend class NonJoinedBlockInputStream; + friend class NotJoinedHash; friend class JoinSource; std::shared_ptr table_join; diff --git a/src/Interpreters/IJoin.h b/src/Interpreters/IJoin.h index 8fa85de4951..2215402e1d4 100644 --- a/src/Interpreters/IJoin.h +++ b/src/Interpreters/IJoin.h @@ -5,7 +5,6 @@ #include #include -#include namespace DB { @@ -15,6 +14,7 @@ struct ExtraBlock; using ExtraBlockPtr = std::shared_ptr; class TableJoin; +class NotJoinedBlocks; class IJoin { @@ -43,7 +43,7 @@ public: /// Different query plan is used for such joins. virtual bool isFilled() const { return false; } - virtual BlockInputStreamPtr createStreamWithNonJoinedRows(const Block &, UInt64) const { return {}; } + virtual std::shared_ptr getNonJoinedBlocks(const Block &, UInt64) const = 0; }; using JoinPtr = std::shared_ptr; diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 4c1a3064c3d..76cb6c783ba 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -223,7 +223,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) if (create.storage->engine->name == "MaterializedPostgreSQL" && !getContext()->getSettingsRef().allow_experimental_database_materialized_postgresql && !internal) { throw Exception("MaterializedPostgreSQL is an experimental database engine. " - "Enable allow_experimental_database_postgresql_replica to use it.", ErrorCodes::UNKNOWN_DATABASE_ENGINE); + "Enable allow_experimental_database_materialized_postgresql to use it.", ErrorCodes::UNKNOWN_DATABASE_ENGINE); } DatabasePtr database = DatabaseFactory::get(create, metadata_path / "", getContext()); diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 297e41c9546..6f64c83f8bf 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -2080,7 +2080,9 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac settings.group_by_two_level_threshold, settings.group_by_two_level_threshold_bytes, settings.max_bytes_before_external_group_by, - settings.empty_result_for_aggregation_by_empty_set || (keys.empty() && query_analyzer->hasConstAggregationKeys()), + settings.empty_result_for_aggregation_by_empty_set + || (settings.empty_result_for_aggregation_by_constant_keys_on_empty_set && keys.empty() + && query_analyzer->hasConstAggregationKeys()), context->getTemporaryVolume(), settings.max_threads, settings.min_free_disk_space_for_temporary_data, diff --git a/src/Interpreters/JoinSwitcher.h b/src/Interpreters/JoinSwitcher.h index a89ac6d5d98..e750bc5eed0 100644 --- a/src/Interpreters/JoinSwitcher.h +++ b/src/Interpreters/JoinSwitcher.h @@ -56,9 +56,9 @@ public: return join->alwaysReturnsEmptySet(); } - BlockInputStreamPtr createStreamWithNonJoinedRows(const Block & block, UInt64 max_block_size) const override + std::shared_ptr getNonJoinedBlocks(const Block & block, UInt64 max_block_size) const override { - return join->createStreamWithNonJoinedRows(block, max_block_size); + return join->getNonJoinedBlocks(block, max_block_size); } private: @@ -74,38 +74,4 @@ private: void switchJoin(); }; - -/// Creates NonJoinedBlockInputStream on the first read. Allows to swap join algo before it. -class LazyNonJoinedBlockInputStream : public IBlockInputStream -{ -public: - LazyNonJoinedBlockInputStream(const IJoin & join_, const Block & block, UInt64 max_block_size_) - : join(join_) - , result_sample_block(block) - , max_block_size(max_block_size_) - {} - - String getName() const override { return "LazyNonMergeJoined"; } - Block getHeader() const override { return result_sample_block; } - -protected: - Block readImpl() override - { - if (!stream) - { - stream = join.createStreamWithNonJoinedRows(result_sample_block, max_block_size); - if (!stream) - return {}; - } - - return stream->read(); - } - -private: - BlockInputStreamPtr stream; - const IJoin & join; - Block result_sample_block; - UInt64 max_block_size; -}; - } diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index a2c63a4693b..0150bbe1d93 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -1,7 +1,8 @@ #include #include -#include +#include + #include #include #include @@ -723,15 +724,7 @@ void MergeJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed) if (needConditionJoinColumn()) block.erase(deriveTempName(mask_column_name_left)); - for (const auto & column_name : lowcard_keys) - { - if (!block.has(column_name)) - continue; - if (auto & col = block.getByName(column_name); !col.type->lowCardinality()) - JoinCommon::changeLowCardinalityInplace(col); - } - - JoinCommon::restoreLowCardinalityInplace(block); + JoinCommon::restoreLowCardinalityInplace(block, lowcard_keys); } template @@ -1035,55 +1028,16 @@ void MergeJoin::initRightTableWriter() } /// Stream from not joined earlier rows of the right table. -class NonMergeJoinedBlockInputStream : private NotJoined, public IBlockInputStream +class NotJoinedMerge final : public NotJoinedBlocks::RightColumnsFiller { public: - NonMergeJoinedBlockInputStream(const MergeJoin & parent_, - const Block & result_sample_block_, - const Names & key_names_right_, - UInt64 max_block_size_) - : NotJoined(*parent_.table_join, - parent_.modifyRightBlock(parent_.right_sample_block), - parent_.right_sample_block, - result_sample_block_, - {}, key_names_right_) - , parent(parent_) - , max_block_size(max_block_size_) + NotJoinedMerge(const MergeJoin & parent_, UInt64 max_block_size_) + : parent(parent_), max_block_size(max_block_size_) {} - String getName() const override { return "NonMergeJoined"; } - Block getHeader() const override { return result_sample_block; } + Block getEmptyBlock() override { return parent.modifyRightBlock(parent.right_sample_block).cloneEmpty(); } -protected: - Block readImpl() override - { - if (parent.getRightBlocksCount()) - return createBlock(); - return {}; - } - -private: - const MergeJoin & parent; - size_t max_block_size; - size_t block_number = 0; - - Block createBlock() - { - MutableColumns columns_right = saved_block_sample.cloneEmptyColumns(); - - size_t rows_added = fillColumns(columns_right); - if (!rows_added) - return {}; - - Block res = result_sample_block.cloneEmpty(); - addLeftColumns(res, rows_added); - addRightColumns(res, columns_right); - copySameKeys(res); - correctLowcardAndNullability(res); - return res; - } - - size_t fillColumns(MutableColumns & columns_right) + size_t fillColumns(MutableColumns & columns_right) override { const RowBitmaps & bitmaps = *parent.used_rows_bitmap; size_t rows_added = 0; @@ -1127,14 +1081,23 @@ private: return rows_added; } + +private: + const MergeJoin & parent; + size_t max_block_size; + size_t block_number = 0; }; -BlockInputStreamPtr MergeJoin::createStreamWithNonJoinedRows(const Block & result_sample_block, UInt64 max_block_size) const +std::shared_ptr MergeJoin::getNonJoinedBlocks(const Block & result_sample_block, UInt64 max_block_size) const { if (table_join->strictness() == ASTTableJoin::Strictness::All && (is_right || is_full)) - return std::make_shared(*this, result_sample_block, key_names_right, max_block_size); - return {}; + { + size_t left_columns_count = result_sample_block.columns() - right_columns_to_add.columns(); + auto non_joined = std::make_unique(*this, max_block_size); + return std::make_shared(std::move(non_joined), result_sample_block, left_columns_count, table_join->leftToRightKeyRemap()); + } + return nullptr; } bool MergeJoin::needConditionJoinColumn() const diff --git a/src/Interpreters/MergeJoin.h b/src/Interpreters/MergeJoin.h index 844c730de4f..9e765041846 100644 --- a/src/Interpreters/MergeJoin.h +++ b/src/Interpreters/MergeJoin.h @@ -35,10 +35,10 @@ public: /// Has to be called only after setTotals()/mergeRightBlocks() bool alwaysReturnsEmptySet() const override { return (is_right || is_inner) && min_max_right_blocks.empty(); } - BlockInputStreamPtr createStreamWithNonJoinedRows(const Block & result_sample_block, UInt64 max_block_size) const override; + std::shared_ptr getNonJoinedBlocks(const Block & result_sample_block, UInt64 max_block_size) const override; private: - friend class NonMergeJoinedBlockInputStream; + friend class NotJoinedMerge; struct NotProcessed : public ExtraBlock { @@ -78,6 +78,7 @@ private: SortDescription right_merge_description; Block right_sample_block; Block right_table_keys; + /// Columns from right side of join, both key and additional Block right_columns_to_add; SortedBlocksWriter::Blocks right_blocks; diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp new file mode 100644 index 00000000000..c0e08395eff --- /dev/null +++ b/src/Interpreters/Session.cpp @@ -0,0 +1,427 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int SESSION_NOT_FOUND; + extern const int SESSION_IS_LOCKED; +} + + +class NamedSessionsStorage; + +/// User ID and session identifier. Named sessions are local to users. +using NamedSessionKey = std::pair; + +/// Named sessions. The user could specify session identifier to reuse settings and temporary tables in subsequent requests. +struct NamedSessionData +{ + NamedSessionKey key; + UInt64 close_cycle = 0; + ContextMutablePtr context; + std::chrono::steady_clock::duration timeout; + NamedSessionsStorage & parent; + + NamedSessionData(NamedSessionKey key_, ContextPtr context_, std::chrono::steady_clock::duration timeout_, NamedSessionsStorage & parent_) + : key(std::move(key_)), context(Context::createCopy(context_)), timeout(timeout_), parent(parent_) + {} + + void release(); +}; + +class NamedSessionsStorage +{ +public: + using Key = NamedSessionKey; + + static NamedSessionsStorage & instance() + { + static NamedSessionsStorage the_instance; + return the_instance; + } + + ~NamedSessionsStorage() + { + try + { + shutdown(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } + + void shutdown() + { + { + std::lock_guard lock{mutex}; + sessions.clear(); + if (!thread.joinable()) + return; + quit = true; + } + + cond.notify_one(); + thread.join(); + } + + /// Find existing session or create a new. + std::pair, bool> acquireSession( + const ContextPtr & global_context, + const UUID & user_id, + const String & session_id, + std::chrono::steady_clock::duration timeout, + bool throw_if_not_found) + { + std::unique_lock lock(mutex); + + Key key{user_id, session_id}; + + auto it = sessions.find(key); + if (it == sessions.end()) + { + if (throw_if_not_found) + throw Exception("Session not found.", ErrorCodes::SESSION_NOT_FOUND); + + /// Create a new session from current context. + auto context = Context::createCopy(global_context); + it = sessions.insert(std::make_pair(key, std::make_shared(key, context, timeout, *this))).first; + const auto & session = it->second; + + if (!thread.joinable()) + thread = ThreadFromGlobalPool{&NamedSessionsStorage::cleanThread, this}; + + return {session, true}; + } + else + { + /// Use existing session. + const auto & session = it->second; + + if (!session.unique()) + throw Exception("Session is locked by a concurrent client.", ErrorCodes::SESSION_IS_LOCKED); + return {session, false}; + } + } + + void releaseSession(NamedSessionData & session) + { + std::unique_lock lock(mutex); + scheduleCloseSession(session, lock); + } + +private: + class SessionKeyHash + { + public: + size_t operator()(const Key & key) const + { + SipHash hash; + hash.update(key.first); + hash.update(key.second); + return hash.get64(); + } + }; + + /// TODO it's very complicated. Make simple std::map with time_t or boost::multi_index. + using Container = std::unordered_map, SessionKeyHash>; + using CloseTimes = std::deque>; + Container sessions; + CloseTimes close_times; + std::chrono::steady_clock::duration close_interval = std::chrono::seconds(1); + std::chrono::steady_clock::time_point close_cycle_time = std::chrono::steady_clock::now(); + UInt64 close_cycle = 0; + + void scheduleCloseSession(NamedSessionData & session, std::unique_lock &) + { + /// Push it on a queue of sessions to close, on a position corresponding to the timeout. + /// (timeout is measured from current moment of time) + + const UInt64 close_index = session.timeout / close_interval + 1; + const auto new_close_cycle = close_cycle + close_index; + + if (session.close_cycle != new_close_cycle) + { + session.close_cycle = new_close_cycle; + if (close_times.size() < close_index + 1) + close_times.resize(close_index + 1); + close_times[close_index].emplace_back(session.key); + } + } + + void cleanThread() + { + setThreadName("SessionCleaner"); + std::unique_lock lock{mutex}; + while (!quit) + { + auto interval = closeSessions(lock); + if (cond.wait_for(lock, interval, [this]() -> bool { return quit; })) + break; + } + } + + /// Close sessions, that has been expired. Returns how long to wait for next session to be expired, if no new sessions will be added. + std::chrono::steady_clock::duration closeSessions(std::unique_lock & lock) + { + const auto now = std::chrono::steady_clock::now(); + + /// The time to close the next session did not come + if (now < close_cycle_time) + return close_cycle_time - now; /// Will sleep until it comes. + + const auto current_cycle = close_cycle; + + ++close_cycle; + close_cycle_time = now + close_interval; + + if (close_times.empty()) + return close_interval; + + auto & sessions_to_close = close_times.front(); + + for (const auto & key : sessions_to_close) + { + const auto session = sessions.find(key); + + if (session != sessions.end() && session->second->close_cycle <= current_cycle) + { + if (!session->second.unique()) + { + /// Skip but move it to close on the next cycle. + session->second->timeout = std::chrono::steady_clock::duration{0}; + scheduleCloseSession(*session->second, lock); + } + else + sessions.erase(session); + } + } + + close_times.pop_front(); + return close_interval; + } + + std::mutex mutex; + std::condition_variable cond; + ThreadFromGlobalPool thread; + bool quit = false; +}; + + +void NamedSessionData::release() +{ + parent.releaseSession(*this); +} + +void Session::shutdownNamedSessions() +{ + NamedSessionsStorage::instance().shutdown(); +} + + +Session::Session(const ContextPtr & global_context_, ClientInfo::Interface interface_) + : global_context(global_context_) +{ + prepared_client_info.emplace(); + prepared_client_info->interface = interface_; +} + +Session::Session(Session &&) = default; + +Session::~Session() +{ + /// Early release a NamedSessionData. + if (named_session) + named_session->release(); +} + +Authentication::Type Session::getAuthenticationType(const String & user_name) const +{ + return global_context->getAccessControlManager().read(user_name)->authentication.getType(); +} + +Authentication::Digest Session::getPasswordDoubleSHA1(const String & user_name) const +{ + return global_context->getAccessControlManager().read(user_name)->authentication.getPasswordDoubleSHA1(); +} + +void Session::authenticate(const String & user_name, const String & password, const Poco::Net::SocketAddress & address) +{ + authenticate(BasicCredentials{user_name, password}, address); +} + +void Session::authenticate(const Credentials & credentials_, const Poco::Net::SocketAddress & address_) +{ + if (session_context) + throw Exception("If there is a session context it must be created after authentication", ErrorCodes::LOGICAL_ERROR); + + user_id = global_context->getAccessControlManager().login(credentials_, address_.host()); + + prepared_client_info->current_user = credentials_.getUserName(); + prepared_client_info->current_address = address_; + +#if defined(ARCADIA_BUILD) + /// This is harmful field that is used only in foreign "Arcadia" build. + if (const auto * basic_credentials = dynamic_cast(&credentials_)) + session_client_info->current_password = basic_credentials->getPassword(); +#endif +} + +ClientInfo & Session::getClientInfo() +{ + return session_context ? session_context->getClientInfo() : *prepared_client_info; +} + +const ClientInfo & Session::getClientInfo() const +{ + return session_context ? session_context->getClientInfo() : *prepared_client_info; +} + +ContextMutablePtr Session::makeSessionContext() +{ + if (session_context) + throw Exception("Session context already exists", ErrorCodes::LOGICAL_ERROR); + if (query_context_created) + throw Exception("Session context must be created before any query context", ErrorCodes::LOGICAL_ERROR); + + /// Make a new session context. + ContextMutablePtr new_session_context; + new_session_context = Context::createCopy(global_context); + new_session_context->makeSessionContext(); + + /// Copy prepared client info to the new session context. + auto & res_client_info = new_session_context->getClientInfo(); + res_client_info = std::move(prepared_client_info).value(); + prepared_client_info.reset(); + + /// Set user information for the new context: current profiles, roles, access rights. + if (user_id) + new_session_context->setUser(*user_id); + + /// Session context is ready. + session_context = new_session_context; + user = session_context->getUser(); + + return session_context; +} + +ContextMutablePtr Session::makeSessionContext(const String & session_id_, std::chrono::steady_clock::duration timeout_, bool session_check_) +{ + if (session_context) + throw Exception("Session context already exists", ErrorCodes::LOGICAL_ERROR); + if (query_context_created) + throw Exception("Session context must be created before any query context", ErrorCodes::LOGICAL_ERROR); + + /// Make a new session context OR + /// if the `session_id` and `user_id` were used before then just get a previously created session context. + std::shared_ptr new_named_session; + bool new_named_session_created = false; + std::tie(new_named_session, new_named_session_created) + = NamedSessionsStorage::instance().acquireSession(global_context, user_id.value_or(UUID{}), session_id_, timeout_, session_check_); + + auto new_session_context = new_named_session->context; + new_session_context->makeSessionContext(); + + /// Copy prepared client info to the session context, no matter it's been just created or not. + /// If we continue using a previously created session context found by session ID + /// it's necessary to replace the client info in it anyway, because it contains actual connection information (client address, etc.) + auto & res_client_info = new_session_context->getClientInfo(); + res_client_info = std::move(prepared_client_info).value(); + prepared_client_info.reset(); + + /// Set user information for the new context: current profiles, roles, access rights. + if (user_id && !new_session_context->getUser()) + new_session_context->setUser(*user_id); + + /// Session context is ready. + session_context = new_session_context; + session_id = session_id_; + named_session = new_named_session; + named_session_created = new_named_session_created; + user = session_context->getUser(); + + return session_context; +} + +ContextMutablePtr Session::makeQueryContext(const ClientInfo & query_client_info) const +{ + return makeQueryContextImpl(&query_client_info, nullptr); +} + +ContextMutablePtr Session::makeQueryContext(ClientInfo && query_client_info) const +{ + return makeQueryContextImpl(nullptr, &query_client_info); +} + +ContextMutablePtr Session::makeQueryContextImpl(const ClientInfo * client_info_to_copy, ClientInfo * client_info_to_move) const +{ + /// We can create a query context either from a session context or from a global context. + bool from_session_context = static_cast(session_context); + + /// Create a new query context. + ContextMutablePtr query_context = Context::createCopy(from_session_context ? session_context : global_context); + query_context->makeQueryContext(); + + /// Copy the specified client info to the new query context. + auto & res_client_info = query_context->getClientInfo(); + if (client_info_to_move) + res_client_info = std::move(*client_info_to_move); + else if (client_info_to_copy && (client_info_to_copy != &getClientInfo())) + res_client_info = *client_info_to_copy; + + /// Copy current user's name and address if it was authenticated after query_client_info was initialized. + if (prepared_client_info && !prepared_client_info->current_user.empty()) + { + res_client_info.current_user = prepared_client_info->current_user; + res_client_info.current_address = prepared_client_info->current_address; +#if defined(ARCADIA_BUILD) + res_client_info.current_password = prepared_client_info->current_password; +#endif + } + + /// Set parameters of initial query. + if (res_client_info.query_kind == ClientInfo::QueryKind::NO_QUERY) + res_client_info.query_kind = ClientInfo::QueryKind::INITIAL_QUERY; + + if (res_client_info.query_kind == ClientInfo::QueryKind::INITIAL_QUERY) + { + res_client_info.initial_user = res_client_info.current_user; + res_client_info.initial_address = res_client_info.current_address; + } + + /// Sets that row policies from the initial user should be used too. + query_context->setInitialRowPolicy(); + + /// Set user information for the new context: current profiles, roles, access rights. + if (user_id && !query_context->getUser()) + query_context->setUser(*user_id); + + /// Query context is ready. + query_context_created = true; + user = query_context->getUser(); + + return query_context; +} + +} diff --git a/src/Interpreters/Session.h b/src/Interpreters/Session.h new file mode 100644 index 00000000000..d104e250099 --- /dev/null +++ b/src/Interpreters/Session.h @@ -0,0 +1,87 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +namespace Poco::Net { class SocketAddress; } + +namespace DB +{ +class Credentials; +class Authentication; +struct NamedSessionData; +class NamedSessionsStorage; +struct User; +using UserPtr = std::shared_ptr; + +/** Represents user-session from the server perspective, + * basically it is just a smaller subset of Context API, simplifies Context management. + * + * Holds session context, facilitates acquisition of NamedSession and proper creation of query contexts. + */ +class Session +{ +public: + /// Stops using named sessions. The method must be called at the server shutdown. + static void shutdownNamedSessions(); + + Session(const ContextPtr & global_context_, ClientInfo::Interface interface_); + Session(Session &&); + ~Session(); + + Session(const Session &) = delete; + Session& operator=(const Session &) = delete; + + /// Provides information about the authentication type of a specified user. + Authentication::Type getAuthenticationType(const String & user_name) const; + Authentication::Digest getPasswordDoubleSHA1(const String & user_name) const; + + /// Sets the current user, checks the credentials and that the specified address is allowed to connect from. + /// The function throws an exception if there is no such user or password is wrong. + void authenticate(const String & user_name, const String & password, const Poco::Net::SocketAddress & address); + void authenticate(const Credentials & credentials_, const Poco::Net::SocketAddress & address_); + + /// Returns a reference to session ClientInfo. + ClientInfo & getClientInfo(); + const ClientInfo & getClientInfo() const; + + /// Makes a session context, can be used one or zero times. + /// The function also assigns an user to this context. + ContextMutablePtr makeSessionContext(); + ContextMutablePtr makeSessionContext(const String & session_id_, std::chrono::steady_clock::duration timeout_, bool session_check_); + ContextMutablePtr sessionContext() { return session_context; } + ContextPtr sessionContext() const { return session_context; } + + /// Makes a query context, can be used multiple times, with or without makeSession() called earlier. + /// The query context will be created from a copy of a session context if it exists, or from a copy of + /// a global context otherwise. In the latter case the function also assigns an user to this context. + ContextMutablePtr makeQueryContext() const { return makeQueryContext(getClientInfo()); } + ContextMutablePtr makeQueryContext(const ClientInfo & query_client_info) const; + ContextMutablePtr makeQueryContext(ClientInfo && query_client_info) const; + +private: + ContextMutablePtr makeQueryContextImpl(const ClientInfo * client_info_to_copy, ClientInfo * client_info_to_move) const; + + const ContextPtr global_context; + + /// ClientInfo that will be copied to a session context when it's created. + std::optional prepared_client_info; + + mutable UserPtr user; + std::optional user_id; + + ContextMutablePtr session_context; + mutable bool query_context_created = false; + + String session_id; + std::shared_ptr named_session; + bool named_session_created = false; +}; + +} diff --git a/src/Interpreters/Set.cpp b/src/Interpreters/Set.cpp index ff502b499cd..5ab59ba3f07 100644 --- a/src/Interpreters/Set.cpp +++ b/src/Interpreters/Set.cpp @@ -99,14 +99,14 @@ void NO_INLINE Set::insertFromBlockImplCase( } -void Set::setHeader(const Block & header) +void Set::setHeader(const ColumnsWithTypeAndName & header) { std::unique_lock lock(rwlock); if (!data.empty()) return; - keys_size = header.columns(); + keys_size = header.size(); ColumnRawPtrs key_columns; key_columns.reserve(keys_size); data_types.reserve(keys_size); @@ -118,10 +118,10 @@ void Set::setHeader(const Block & header) /// Remember the columns we will work with for (size_t i = 0; i < keys_size; ++i) { - materialized_columns.emplace_back(header.safeGetByPosition(i).column->convertToFullColumnIfConst()); + materialized_columns.emplace_back(header.at(i).column->convertToFullColumnIfConst()); key_columns.emplace_back(materialized_columns.back().get()); - data_types.emplace_back(header.safeGetByPosition(i).type); - set_elements_types.emplace_back(header.safeGetByPosition(i).type); + data_types.emplace_back(header.at(i).type); + set_elements_types.emplace_back(header.at(i).type); /// Convert low cardinality column to full. if (const auto * low_cardinality_type = typeid_cast(data_types.back().get())) @@ -161,7 +161,7 @@ void Set::setHeader(const Block & header) } -bool Set::insertFromBlock(const Block & block) +bool Set::insertFromBlock(const ColumnsWithTypeAndName & columns) { std::unique_lock lock(rwlock); @@ -177,11 +177,11 @@ bool Set::insertFromBlock(const Block & block) /// Remember the columns we will work with for (size_t i = 0; i < keys_size; ++i) { - materialized_columns.emplace_back(block.safeGetByPosition(i).column->convertToFullColumnIfConst()->convertToFullColumnIfLowCardinality()); + materialized_columns.emplace_back(columns.at(i).column->convertToFullColumnIfConst()->convertToFullColumnIfLowCardinality()); key_columns.emplace_back(materialized_columns.back().get()); } - size_t rows = block.rows(); + size_t rows = columns.at(0).column->size(); /// We will insert to the Set only keys, where all components are not NULL. ConstNullMapPtr null_map{}; @@ -192,7 +192,7 @@ bool Set::insertFromBlock(const Block & block) /// Filter to extract distinct values from the block. ColumnUInt8::MutablePtr filter; if (fill_set_elements) - filter = ColumnUInt8::create(block.rows()); + filter = ColumnUInt8::create(rows); switch (data.type) { @@ -224,16 +224,16 @@ bool Set::insertFromBlock(const Block & block) } -ColumnPtr Set::execute(const Block & block, bool negative) const +ColumnPtr Set::execute(const ColumnsWithTypeAndName & columns, bool negative) const { - size_t num_key_columns = block.columns(); + size_t num_key_columns = columns.size(); if (0 == num_key_columns) throw Exception("Logical error: no columns passed to Set::execute method.", ErrorCodes::LOGICAL_ERROR); auto res = ColumnUInt8::create(); ColumnUInt8::Container & vec_res = res->getData(); - vec_res.resize(block.safeGetByPosition(0).column->size()); + vec_res.resize(columns.at(0).column->size()); if (vec_res.empty()) return res; @@ -264,7 +264,7 @@ ColumnPtr Set::execute(const Block & block, bool negative) const { ColumnPtr result; - const auto & column_before_cast = block.safeGetByPosition(i); + const auto & column_before_cast = columns.at(i); ColumnWithTypeAndName column_to_cast = {column_before_cast.column->convertToFullColumnIfConst(), column_before_cast.type, column_before_cast.name}; @@ -428,7 +428,7 @@ MergeTreeSetIndex::MergeTreeSetIndex(const Columns & set_elements, std::vectorsecond; } if (correct_nullability && leftBecomeNullable(col.type)) - { - /// No need to nullify constants - bool is_column_const = col.column && isColumnConst(*col.column); - if (!is_column_const) - col.type = JoinCommon::convertTypeToNullable(col.type); - } + col.type = JoinCommon::convertTypeToNullable(col.type); } /// Types in columns_added_by_join already converted and set nullable if needed for (const auto & col : columns_added_by_join) - columns.emplace_back(nullptr, col.type, col.name); + names_and_types.emplace_back(col.name, col.type); } bool TableJoin::sameStrictnessAndKind(ASTTableJoin::Strictness strictness_, ASTTableJoin::Kind kind_) const @@ -472,6 +454,24 @@ void TableJoin::addJoinCondition(const ASTPtr & ast, bool is_left) on_filter_condition_asts_right.push_back(ast); } +std::unordered_map TableJoin::leftToRightKeyRemap() const +{ + std::unordered_map left_to_right_key_remap; + if (hasUsing()) + { + const auto & required_right_keys = requiredRightKeys(); + for (size_t i = 0; i < key_names_left.size(); ++i) + { + const String & left_key_name = key_names_left[i]; + const String & right_key_name = key_names_right[i]; + + if (!required_right_keys.contains(right_key_name)) + left_to_right_key_remap[left_key_name] = right_key_name; + } + } + return left_to_right_key_remap; +} + /// Returns all conditions related to one table joined with 'and' function static ASTPtr buildJoinConditionColumn(const ASTs & on_filter_condition_asts) { diff --git a/src/Interpreters/TableJoin.h b/src/Interpreters/TableJoin.h index 4c8c16028f5..5d26a14113c 100644 --- a/src/Interpreters/TableJoin.h +++ b/src/Interpreters/TableJoin.h @@ -191,7 +191,6 @@ public: void addJoinedColumn(const NameAndTypePair & joined_column); void addJoinedColumnsAndCorrectTypes(NamesAndTypesList & names_and_types, bool correct_nullability = true) const; - void addJoinedColumnsAndCorrectTypes(ColumnsWithTypeAndName & columns, bool correct_nullability = true) const; /// Calculates common supertypes for corresponding join key columns. bool inferJoinKeyCommonType(const NamesAndTypesList & left, const NamesAndTypesList & right); @@ -230,6 +229,7 @@ public: Block getRequiredRightKeys(const Block & right_table_keys, std::vector & keys_sources) const; String renamedRightColumnName(const String & name) const; + std::unordered_map leftToRightKeyRemap() const; }; } diff --git a/src/Interpreters/TreeOptimizer.cpp b/src/Interpreters/TreeOptimizer.cpp index c1a265d9a06..518c041d785 100644 --- a/src/Interpreters/TreeOptimizer.cpp +++ b/src/Interpreters/TreeOptimizer.cpp @@ -69,7 +69,9 @@ const std::unordered_set possibly_injective_function_names void appendUnusedGroupByColumn(ASTSelectQuery * select_query, const NameSet & source_columns) { /// You must insert a constant that is not the name of the column in the table. Such a case is rare, but it happens. - UInt64 unused_column = 0; + /// Also start unused_column integer from source_columns.size() + 1, because lower numbers ([1, source_columns.size()]) + /// might be in positional GROUP BY. + UInt64 unused_column = source_columns.size() + 1; String unused_column_name = toString(unused_column); while (source_columns.count(unused_column_name)) @@ -111,6 +113,8 @@ void optimizeGroupBy(ASTSelectQuery * select_query, const NameSet & source_colum group_exprs.pop_back(); }; + const auto & settings = context->getSettingsRef(); + /// iterate over each GROUP BY expression, eliminate injective function calls and literals for (size_t i = 0; i < group_exprs.size();) { @@ -166,7 +170,22 @@ void optimizeGroupBy(ASTSelectQuery * select_query, const NameSet & source_colum } else if (is_literal(group_exprs[i])) { - remove_expr_at_index(i); + bool keep_position = false; + if (settings.enable_positional_arguments) + { + const auto & value = group_exprs[i]->as()->value; + if (value.getType() == Field::Types::UInt64) + { + auto pos = value.get(); + if (pos > 0 && pos <= select_query->children.size()) + keep_position = true; + } + } + + if (keep_position) + ++i; + else + remove_expr_at_index(i); } else { diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index e46f644e836..c05118b7c6a 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -18,6 +18,7 @@ #include #include #include +#include namespace DB { @@ -121,6 +122,7 @@ std::tuple evaluateDatabaseNameForMergeEngine(const ASTPtr & node, return std::tuple{false, ast}; } + namespace { using Conjunction = ColumnsWithTypeAndName; @@ -213,7 +215,7 @@ namespace Disjunction result; - auto add_dnf = [&](const auto &dnf) + auto add_dnf = [&](const auto & dnf) { if (dnf.size() > limit) { @@ -338,6 +340,7 @@ std::optional evaluateExpressionOverConstantCondition(const ASTPtr & nod if (const auto * fn = node->as()) { + std::unordered_map always_false_map; const auto dnf = analyzeFunction(fn, target_expr, limit); if (dnf.empty() || !limit) @@ -368,7 +371,41 @@ std::optional evaluateExpressionOverConstantCondition(const ASTPtr & nod for (const auto & conjunct : dnf) { - Block block(conjunct); + Block block; + + for (const auto & elem : conjunct) + { + if (!block.has(elem.name)) + { + block.insert(elem); + } + else + { + /// Conjunction of condition on column equality to distinct values can never be satisfied. + + const ColumnWithTypeAndName & prev = block.getByName(elem.name); + + if (isColumnConst(*prev.column) && isColumnConst(*elem.column)) + { + Field prev_value = assert_cast(*prev.column).getField(); + Field curr_value = assert_cast(*elem.column).getField(); + + if (!always_false_map.count(elem.name)) + { + always_false_map[elem.name] = prev_value != curr_value; + } + else + { + auto & always_false = always_false_map[elem.name]; + /// If at least one of conjunct is not always false, we should preserve this. + if (always_false) + { + always_false = prev_value != curr_value; + } + } + } + } + } // Block should contain all required columns from `target_expr` if (!has_required_columns(block)) @@ -393,6 +430,11 @@ std::optional evaluateExpressionOverConstantCondition(const ASTPtr & nod return {}; } } + + bool any_always_false = std::any_of(always_false_map.begin(), always_false_map.end(), [](const auto & v) { return v.second; }); + if (any_always_false) + return Blocks{}; + } else if (const auto * literal = node->as()) { diff --git a/src/Interpreters/join_common.cpp b/src/Interpreters/join_common.cpp index e9f3e4f3fdd..9890a130c33 100644 --- a/src/Interpreters/join_common.cpp +++ b/src/Interpreters/join_common.cpp @@ -314,8 +314,16 @@ void removeLowCardinalityInplace(Block & block, const Names & names, bool change } } -void restoreLowCardinalityInplace(Block & block) +void restoreLowCardinalityInplace(Block & block, const Names & lowcard_keys) { + for (const auto & column_name : lowcard_keys) + { + if (!block.has(column_name)) + continue; + if (auto & col = block.getByName(column_name); !col.type->lowCardinality()) + JoinCommon::changeLowCardinalityInplace(col); + } + for (size_t i = 0; i < block.columns(); ++i) { auto & col = block.getByPosition(i); @@ -484,49 +492,23 @@ void splitAdditionalColumns(const Names & key_names, const Block & sample_block, } - -NotJoined::NotJoined(const TableJoin & table_join, const Block & saved_block_sample_, const Block & right_sample_block, - const Block & result_sample_block_, const Names & key_names_left_, const Names & key_names_right_) - : saved_block_sample(saved_block_sample_) +NotJoinedBlocks::NotJoinedBlocks(std::unique_ptr filler_, + const Block & result_sample_block_, + size_t left_columns_count, + const LeftToRightKeyRemap & left_to_right_key_remap) + : filler(std::move(filler_)) + , saved_block_sample(filler->getEmptyBlock()) , result_sample_block(materializeBlock(result_sample_block_)) - , key_names_left(key_names_left_.empty() ? table_join.keyNamesLeft() : key_names_left_) - , key_names_right(key_names_right_.empty() ? table_join.keyNamesRight() : key_names_right_) { - std::vector tmp; - Block right_table_keys; - Block sample_block_with_columns_to_add; - - JoinCommon::splitAdditionalColumns(key_names_right, right_sample_block, right_table_keys, - sample_block_with_columns_to_add); - Block required_right_keys = table_join.getRequiredRightKeys(right_table_keys, tmp); - - std::unordered_map left_to_right_key_remap; - - if (table_join.hasUsing()) - { - for (size_t i = 0; i < key_names_left.size(); ++i) - { - const String & left_key_name = key_names_left[i]; - const String & right_key_name = key_names_right[i]; - - size_t left_key_pos = result_sample_block.getPositionByName(left_key_name); - size_t right_key_pos = saved_block_sample.getPositionByName(right_key_name); - - if (!required_right_keys.has(right_key_name)) - left_to_right_key_remap[left_key_pos] = right_key_pos; - } - } - - /// result_sample_block: left_sample_block + left expressions, right not key columns, required right keys - size_t left_columns_count = result_sample_block.columns() - - sample_block_with_columns_to_add.columns() - required_right_keys.columns(); for (size_t left_pos = 0; left_pos < left_columns_count; ++left_pos) { - /// We need right 'x' for 'RIGHT JOIN ... USING(x)'. - if (left_to_right_key_remap.count(left_pos)) + /// We need right 'x' for 'RIGHT JOIN ... USING(x)' + auto left_name = result_sample_block.getByPosition(left_pos).name; + const auto & right_key = left_to_right_key_remap.find(left_name); + if (right_key != left_to_right_key_remap.end()) { - size_t right_key_pos = left_to_right_key_remap[left_pos]; + size_t right_key_pos = saved_block_sample.getPositionByName(right_key->second); setRightIndex(right_key_pos, left_pos); } else @@ -556,9 +538,9 @@ NotJoined::NotJoined(const TableJoin & table_join, const Block & saved_block_sam ErrorCodes::LOGICAL_ERROR); } -void NotJoined::setRightIndex(size_t right_pos, size_t result_position) +void NotJoinedBlocks::setRightIndex(size_t right_pos, size_t result_position) { - if (!column_indices_right.count(right_pos)) + if (!column_indices_right.contains(right_pos)) { column_indices_right[right_pos] = result_position; extractColumnChanges(right_pos, result_position); @@ -567,7 +549,7 @@ void NotJoined::setRightIndex(size_t right_pos, size_t result_position) same_result_keys[result_position] = column_indices_right[right_pos]; } -void NotJoined::extractColumnChanges(size_t right_pos, size_t result_pos) +void NotJoinedBlocks::extractColumnChanges(size_t right_pos, size_t result_pos) { auto src_props = getLowcardAndNullability(saved_block_sample.getByPosition(right_pos).column); auto dst_props = getLowcardAndNullability(result_sample_block.getByPosition(result_pos).column); @@ -579,7 +561,7 @@ void NotJoined::extractColumnChanges(size_t right_pos, size_t result_pos) right_lowcard_changes.push_back({result_pos, dst_props.is_lowcard}); } -void NotJoined::correctLowcardAndNullability(Block & block) +void NotJoinedBlocks::correctLowcardAndNullability(Block & block) { for (auto & [pos, added] : right_nullability_changes) { @@ -607,7 +589,7 @@ void NotJoined::correctLowcardAndNullability(Block & block) } } -void NotJoined::addLeftColumns(Block & block, size_t rows_added) const +void NotJoinedBlocks::addLeftColumns(Block & block, size_t rows_added) const { for (size_t pos : column_indices_left) { @@ -619,7 +601,7 @@ void NotJoined::addLeftColumns(Block & block, size_t rows_added) const } } -void NotJoined::addRightColumns(Block & block, MutableColumns & columns_right) const +void NotJoinedBlocks::addRightColumns(Block & block, MutableColumns & columns_right) const { for (const auto & pr : column_indices_right) { @@ -629,7 +611,7 @@ void NotJoined::addRightColumns(Block & block, MutableColumns & columns_right) c } } -void NotJoined::copySameKeys(Block & block) const +void NotJoinedBlocks::copySameKeys(Block & block) const { for (const auto & pr : same_result_keys) { @@ -639,4 +621,26 @@ void NotJoined::copySameKeys(Block & block) const } } +Block NotJoinedBlocks::read() +{ + Block result_block = result_sample_block.cloneEmpty(); + { + Block right_block = filler->getEmptyBlock(); + MutableColumns columns_right = right_block.cloneEmptyColumns(); + size_t rows_added = filler->fillColumns(columns_right); + if (rows_added == 0) + return {}; + + addLeftColumns(result_block, rows_added); + addRightColumns(result_block, columns_right); + } + copySameKeys(result_block); + correctLowcardAndNullability(result_block); + +#ifndef NDEBUG + assertBlocksHaveEqualStructure(result_block, result_sample_block, "NotJoinedBlocks"); +#endif + return result_block; +} + } diff --git a/src/Interpreters/join_common.h b/src/Interpreters/join_common.h index 1f037899155..ec2e1d3bd50 100644 --- a/src/Interpreters/join_common.h +++ b/src/Interpreters/join_common.h @@ -30,7 +30,7 @@ ColumnRawPtrs materializeColumnsInplace(Block & block, const Names & names); ColumnRawPtrs getRawPointers(const Columns & columns); void removeLowCardinalityInplace(Block & block); void removeLowCardinalityInplace(Block & block, const Names & names, bool change_type = true); -void restoreLowCardinalityInplace(Block & block); +void restoreLowCardinalityInplace(Block & block, const Names & lowcard_keys); ColumnRawPtrs extractKeysForJoin(const Block & block_keys, const Names & key_names_right); @@ -64,40 +64,58 @@ void changeLowCardinalityInplace(ColumnWithTypeAndName & column); } /// Creates result from right table data in RIGHT and FULL JOIN when keys are not present in left table. -class NotJoined +class NotJoinedBlocks final { public: - NotJoined(const TableJoin & table_join, const Block & saved_block_sample_, const Block & right_sample_block, - const Block & result_sample_block_, const Names & key_names_left_ = {}, const Names & key_names_right_ = {}); + using LeftToRightKeyRemap = std::unordered_map; + /// Returns non joined columns from right part of join + class RightColumnsFiller + { + public: + /// Create empty block for right part + virtual Block getEmptyBlock() = 0; + /// Fill columns from right part of join with not joined rows + virtual size_t fillColumns(MutableColumns & columns_right) = 0; + + virtual ~RightColumnsFiller() = default; + }; + + NotJoinedBlocks(std::unique_ptr filler_, + const Block & result_sample_block_, + size_t left_columns_count, + const LeftToRightKeyRemap & left_to_right_key_remap); + + Block read(); + +private: + void extractColumnChanges(size_t right_pos, size_t result_pos); void correctLowcardAndNullability(Block & block); void addLeftColumns(Block & block, size_t rows_added) const; void addRightColumns(Block & block, MutableColumns & columns_right) const; void copySameKeys(Block & block) const; -protected: + std::unique_ptr filler; + + /// Right block saved in Join Block saved_block_sample; + + /// Output of join Block result_sample_block; - Names key_names_left; - Names key_names_right; - - ~NotJoined() = default; - -private: /// Indices of columns in result_sample_block that should be generated std::vector column_indices_left; /// Indices of columns that come from the right-side table: right_pos -> result_pos std::unordered_map column_indices_right; - /// + std::unordered_map same_result_keys; - /// Which right columns (saved in parent) need nullability change before placing them in result block + + /// Which right columns (saved in parent) need Nullability/LowCardinality change + /// before placing them in result block std::vector> right_nullability_changes; - /// Which right columns (saved in parent) need LowCardinality change before placing them in result block std::vector> right_lowcard_changes; void setRightIndex(size_t right_pos, size_t result_position); - void extractColumnChanges(size_t right_pos, size_t result_pos); }; } diff --git a/src/Parsers/ASTColumnsTransformers.cpp b/src/Parsers/ASTColumnsTransformers.cpp index 359efbd03aa..451ecf0d4dd 100644 --- a/src/Parsers/ASTColumnsTransformers.cpp +++ b/src/Parsers/ASTColumnsTransformers.cpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB @@ -40,10 +41,18 @@ void ASTColumnsApplyTransformer::formatImpl(const FormatSettings & settings, For if (!column_name_prefix.empty()) settings.ostr << "("; - settings.ostr << func_name; - if (parameters) - parameters->formatImpl(settings, state, frame); + if (lambda) + { + lambda->formatImpl(settings, state, frame); + } + else + { + settings.ostr << func_name; + + if (parameters) + parameters->formatImpl(settings, state, frame); + } if (!column_name_prefix.empty()) settings.ostr << ", '" << column_name_prefix << "')"; @@ -64,9 +73,33 @@ void ASTColumnsApplyTransformer::transform(ASTs & nodes) const else name = column->getColumnName(); } - auto function = makeASTFunction(func_name, column); - function->parameters = parameters; - column = function; + if (lambda) + { + auto body = lambda->as().arguments->children.at(1)->clone(); + std::stack stack; + stack.push(body); + while (!stack.empty()) + { + auto ast = stack.top(); + stack.pop(); + for (auto & child : ast->children) + { + if (auto arg_name = tryGetIdentifierName(child); arg_name && arg_name == lambda_arg) + { + child = column->clone(); + continue; + } + stack.push(child); + } + } + column = body; + } + else + { + auto function = makeASTFunction(func_name, column); + function->parameters = parameters; + column = function; + } if (!column_name_prefix.empty()) column->setAlias(column_name_prefix + name); } diff --git a/src/Parsers/ASTColumnsTransformers.h b/src/Parsers/ASTColumnsTransformers.h index 49d29222f02..1064beb44bd 100644 --- a/src/Parsers/ASTColumnsTransformers.h +++ b/src/Parsers/ASTColumnsTransformers.h @@ -25,13 +25,22 @@ public: auto res = std::make_shared(*this); if (parameters) res->parameters = parameters->clone(); + if (lambda) + res->lambda = lambda->clone(); return res; } void transform(ASTs & nodes) const override; + + // Case 1 APPLY (quantile(0.9)) String func_name; - String column_name_prefix; ASTPtr parameters; + // Case 2 APPLY (x -> quantile(0.9)(x)) + ASTPtr lambda; + String lambda_arg; + + String column_name_prefix; + protected: void formatImpl(const FormatSettings & settings, FormatState &, FormatStateStacked) const override; }; diff --git a/src/Parsers/ASTInsertQuery.cpp b/src/Parsers/ASTInsertQuery.cpp index 8bfd3ccf1f2..39ae5f2a58a 100644 --- a/src/Parsers/ASTInsertQuery.cpp +++ b/src/Parsers/ASTInsertQuery.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -48,11 +49,15 @@ void ASTInsertQuery::formatImpl(const FormatSettings & settings, FormatState & s } else { + if (infile) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM INFILE " << (settings.hilite ? hilite_none : "") << infile->as().value.safeGet(); + } if (!format.empty()) { settings.ostr << (settings.hilite ? hilite_keyword : "") << " FORMAT " << (settings.hilite ? hilite_none : "") << format; } - else + else if (!infile) { settings.ostr << (settings.hilite ? hilite_keyword : "") << " VALUES" << (settings.hilite ? hilite_none : ""); } diff --git a/src/Parsers/ASTInsertQuery.h b/src/Parsers/ASTInsertQuery.h index a454f46c3f1..e98fe79dedb 100644 --- a/src/Parsers/ASTInsertQuery.h +++ b/src/Parsers/ASTInsertQuery.h @@ -2,6 +2,7 @@ #include #include +#include "Parsers/IAST_fwd.h" namespace DB { @@ -16,6 +17,7 @@ public: ASTPtr columns; String format; ASTPtr select; + ASTPtr infile; ASTPtr watch; ASTPtr table_function; ASTPtr settings_ast; diff --git a/src/Parsers/ASTSelectQuery.h b/src/Parsers/ASTSelectQuery.h index 12817199d13..2babc2f75c8 100644 --- a/src/Parsers/ASTSelectQuery.h +++ b/src/Parsers/ASTSelectQuery.h @@ -35,6 +35,44 @@ public: SETTINGS }; + static String expressionToString(Expression expr) + { + switch (expr) + { + case Expression::WITH: + return "WITH"; + case Expression::SELECT: + return "SELECT"; + case Expression::TABLES: + return "TABLES"; + case Expression::PREWHERE: + return "PREWHERE"; + case Expression::WHERE: + return "WHERE"; + case Expression::GROUP_BY: + return "GROUP BY"; + case Expression::HAVING: + return "HAVING"; + case Expression::WINDOW: + return "WINDOW"; + case Expression::ORDER_BY: + return "ORDER BY"; + case Expression::LIMIT_BY_OFFSET: + return "LIMIT BY OFFSET"; + case Expression::LIMIT_BY_LENGTH: + return "LIMIT BY LENGTH"; + case Expression::LIMIT_BY: + return "LIMIT BY"; + case Expression::LIMIT_OFFSET: + return "LIMIT OFFSET"; + case Expression::LIMIT_LENGTH: + return "LIMIT LENGTH"; + case Expression::SETTINGS: + return "SETTINGS"; + } + return ""; + } + /** Get the text that identifies this element. */ String getID(char) const override { return "SelectQuery"; } diff --git a/src/Parsers/CMakeLists.txt b/src/Parsers/CMakeLists.txt index 5aaa5c32f92..a20dd3567a9 100644 --- a/src/Parsers/CMakeLists.txt +++ b/src/Parsers/CMakeLists.txt @@ -12,3 +12,7 @@ endif () if(ENABLE_EXAMPLES) add_subdirectory(examples) endif() + +if (ENABLE_FUZZING) + add_subdirectory(fuzzers) +endif() diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 16f2b720b4a..0ae51b2e843 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -850,15 +850,24 @@ static bool isOneOf(TokenType token) return ((token == tokens) || ...); } - bool ParserCastOperator::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - /// Parse numbers (including decimals), strings and arrays of them. + /// Parse numbers (including decimals), strings, arrays and tuples of them. const char * data_begin = pos->begin; const char * data_end = pos->end; bool is_string_literal = pos->type == TokenType::StringLiteral; - if (pos->type == TokenType::Number || is_string_literal) + + if (pos->type == TokenType::Minus) + { + ++pos; + if (pos->type != TokenType::Number) + return false; + + data_end = pos->end; + ++pos; + } + else if (pos->type == TokenType::Number || is_string_literal) { ++pos; } @@ -876,7 +885,7 @@ bool ParserCastOperator::parseImpl(Pos & pos, ASTPtr & node, Expected & expected } else if (pos->type == TokenType::ClosingSquareBracket) { - if (isOneOf(last_token)) + if (isOneOf(last_token)) return false; if (stack.empty() || stack.back() != TokenType::OpeningSquareBracket) return false; @@ -884,7 +893,7 @@ bool ParserCastOperator::parseImpl(Pos & pos, ASTPtr & node, Expected & expected } else if (pos->type == TokenType::ClosingRoundBracket) { - if (isOneOf(last_token)) + if (isOneOf(last_token)) return false; if (stack.empty() || stack.back() != TokenType::OpeningRoundBracket) return false; @@ -892,10 +901,15 @@ bool ParserCastOperator::parseImpl(Pos & pos, ASTPtr & node, Expected & expected } else if (pos->type == TokenType::Comma) { - if (isOneOf(last_token)) + if (isOneOf(last_token)) return false; } - else if (isOneOf(pos->type)) + else if (pos->type == TokenType::Number) + { + if (!isOneOf(last_token)) + return false; + } + else if (isOneOf(pos->type)) { if (!isOneOf(last_token)) return false; @@ -915,6 +929,8 @@ bool ParserCastOperator::parseImpl(Pos & pos, ASTPtr & node, Expected & expected if (!stack.empty()) return false; } + else + return false; ASTPtr type_ast; if (ParserToken(TokenType::DoubleColon).ignore(pos, expected) @@ -1811,20 +1827,47 @@ bool ParserColumnsTransformers::parseImpl(Pos & pos, ASTPtr & node, Expected & e with_open_round_bracket = true; } + ASTPtr lambda; + String lambda_arg; ASTPtr func_name; - if (!ParserIdentifier().parse(pos, func_name, expected)) - return false; - ASTPtr expr_list_args; - if (pos->type == TokenType::OpeningRoundBracket) + auto opos = pos; + if (ParserLambdaExpression().parse(pos, lambda, expected)) { - ++pos; - if (!ParserExpressionList(false).parse(pos, expr_list_args, expected)) + if (const auto * func = lambda->as(); func && func->name == "lambda") + { + const auto * lambda_args_tuple = func->arguments->children.at(0)->as(); + const ASTs & lambda_arg_asts = lambda_args_tuple->arguments->children; + if (lambda_arg_asts.size() != 1) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "APPLY column transformer can only accept lambda with one argument"); + + if (auto opt_arg_name = tryGetIdentifierName(lambda_arg_asts[0]); opt_arg_name) + lambda_arg = *opt_arg_name; + else + throw Exception(ErrorCodes::BAD_ARGUMENTS, "lambda argument declarations must be identifiers"); + } + else + { + lambda = nullptr; + pos = opos; + } + } + + if (!lambda) + { + if (!ParserIdentifier().parse(pos, func_name, expected)) return false; - if (pos->type != TokenType::ClosingRoundBracket) - return false; - ++pos; + if (pos->type == TokenType::OpeningRoundBracket) + { + ++pos; + if (!ParserExpressionList(false).parse(pos, expr_list_args, expected)) + return false; + + if (pos->type != TokenType::ClosingRoundBracket) + return false; + ++pos; + } } String column_name_prefix; @@ -1848,8 +1891,16 @@ bool ParserColumnsTransformers::parseImpl(Pos & pos, ASTPtr & node, Expected & e } auto res = std::make_shared(); - res->func_name = getIdentifierName(func_name); - res->parameters = expr_list_args; + if (lambda) + { + res->lambda = lambda; + res->lambda_arg = lambda_arg; + } + else + { + res->func_name = getIdentifierName(func_name); + res->parameters = expr_list_args; + } res->column_name_prefix = column_name_prefix; node = std::move(res); return true; diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index 58f5e766905..3aa5c82884b 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -664,10 +664,12 @@ bool ParserUnaryExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (pos->type == TokenType::Minus) { - ParserLiteral lit_p; Pos begin = pos; + if (ParserCastOperator().parse(pos, node, expected)) + return true; - if (lit_p.parse(pos, node, expected)) + pos = begin; + if (ParserLiteral().parse(pos, node, expected)) return true; pos = begin; diff --git a/src/Parsers/ParserInsertQuery.cpp b/src/Parsers/ParserInsertQuery.cpp index 1f987edf13f..d597e572437 100644 --- a/src/Parsers/ParserInsertQuery.cpp +++ b/src/Parsers/ParserInsertQuery.cpp @@ -11,6 +11,7 @@ #include #include #include +#include "Parsers/IAST_fwd.h" namespace DB @@ -25,6 +26,7 @@ namespace ErrorCodes bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ParserKeyword s_insert_into("INSERT INTO"); + ParserKeyword s_from_infile("FROM INFILE"); ParserKeyword s_table("TABLE"); ParserKeyword s_function("FUNCTION"); ParserToken s_dot(TokenType::Dot); @@ -39,9 +41,11 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserIdentifier name_p; ParserList columns_p(std::make_unique(), std::make_unique(TokenType::Comma), false); ParserFunction table_function_p{false}; + ParserStringLiteral infile_name_p; ASTPtr database; ASTPtr table; + ASTPtr infile; ASTPtr columns; ASTPtr format; ASTPtr select; @@ -86,8 +90,14 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) Pos before_values = pos; - /// VALUES or FORMAT or SELECT - if (s_values.ignore(pos, expected)) + if (s_from_infile.ignore(pos, expected)) + { + if (!infile_name_p.parse(pos, infile, expected)) + return false; + } + + /// VALUES or FROM INFILE or FORMAT or SELECT + if (!infile && s_values.ignore(pos, expected)) { data = pos->begin; } @@ -136,7 +146,7 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } - if (format) + if (format && !infile) { Pos last_token = pos; --last_token; @@ -167,6 +177,9 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) auto query = std::make_shared(); node = query; + if (infile) + query->infile = infile; + if (table_function) { query->table_function = table_function; diff --git a/src/Parsers/examples/CMakeLists.txt b/src/Parsers/examples/CMakeLists.txt index 3ff40b3cc3b..3e1d6ae559f 100644 --- a/src/Parsers/examples/CMakeLists.txt +++ b/src/Parsers/examples/CMakeLists.txt @@ -8,14 +8,3 @@ target_link_libraries(select_parser PRIVATE clickhouse_parsers) add_executable(create_parser create_parser.cpp ${SRCS}) target_link_libraries(create_parser PRIVATE clickhouse_parsers) - -if (ENABLE_FUZZING) - add_executable(lexer_fuzzer lexer_fuzzer.cpp ${SRCS}) - target_link_libraries(lexer_fuzzer PRIVATE clickhouse_parsers ${LIB_FUZZING_ENGINE}) - - add_executable(select_parser_fuzzer select_parser_fuzzer.cpp ${SRCS}) - target_link_libraries(select_parser_fuzzer PRIVATE clickhouse_parsers ${LIB_FUZZING_ENGINE}) - - add_executable(create_parser_fuzzer create_parser_fuzzer.cpp ${SRCS}) - target_link_libraries(create_parser_fuzzer PRIVATE clickhouse_parsers ${LIB_FUZZING_ENGINE}) -endif () diff --git a/src/Parsers/fuzzers/CMakeLists.txt b/src/Parsers/fuzzers/CMakeLists.txt new file mode 100644 index 00000000000..0dd541e663f --- /dev/null +++ b/src/Parsers/fuzzers/CMakeLists.txt @@ -0,0 +1,8 @@ +add_executable(lexer_fuzzer lexer_fuzzer.cpp ${SRCS}) +target_link_libraries(lexer_fuzzer PRIVATE clickhouse_parsers ${LIB_FUZZING_ENGINE}) + +add_executable(select_parser_fuzzer select_parser_fuzzer.cpp ${SRCS}) +target_link_libraries(select_parser_fuzzer PRIVATE clickhouse_parsers ${LIB_FUZZING_ENGINE}) + +add_executable(create_parser_fuzzer create_parser_fuzzer.cpp ${SRCS}) +target_link_libraries(create_parser_fuzzer PRIVATE clickhouse_parsers ${LIB_FUZZING_ENGINE}) diff --git a/src/Parsers/examples/create_parser_fuzzer.cpp b/src/Parsers/fuzzers/create_parser_fuzzer.cpp similarity index 82% rename from src/Parsers/examples/create_parser_fuzzer.cpp rename to src/Parsers/fuzzers/create_parser_fuzzer.cpp index 169f7b765b7..b0adec7d634 100644 --- a/src/Parsers/examples/create_parser_fuzzer.cpp +++ b/src/Parsers/fuzzers/create_parser_fuzzer.cpp @@ -15,7 +15,10 @@ try DB::ParserCreateQuery parser; DB::ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); - DB::formatAST(*ast, std::cerr); + DB::WriteBufferFromOwnString wb; + DB::formatAST(*ast, wb); + + std::cerr << wb.str() << std::endl; return 0; } diff --git a/src/Parsers/examples/lexer_fuzzer.cpp b/src/Parsers/fuzzers/lexer_fuzzer.cpp similarity index 100% rename from src/Parsers/examples/lexer_fuzzer.cpp rename to src/Parsers/fuzzers/lexer_fuzzer.cpp diff --git a/src/Parsers/examples/select_parser_fuzzer.cpp b/src/Parsers/fuzzers/select_parser_fuzzer.cpp similarity index 82% rename from src/Parsers/examples/select_parser_fuzzer.cpp rename to src/Parsers/fuzzers/select_parser_fuzzer.cpp index 4848b285c07..bdba552dfa7 100644 --- a/src/Parsers/examples/select_parser_fuzzer.cpp +++ b/src/Parsers/fuzzers/select_parser_fuzzer.cpp @@ -14,7 +14,10 @@ try DB::ParserQueryWithOutput parser(input.data() + input.size()); DB::ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); - DB::formatAST(*ast, std::cerr); + DB::WriteBufferFromOwnString wb; + DB::formatAST(*ast, wb); + + std::cerr << wb.str() << std::endl; return 0; } diff --git a/src/Processors/Formats/Impl/MySQLOutputFormat.cpp b/src/Processors/Formats/Impl/MySQLOutputFormat.cpp index 6fdcc544a18..4c6f4624ad4 100644 --- a/src/Processors/Formats/Impl/MySQLOutputFormat.cpp +++ b/src/Processors/Formats/Impl/MySQLOutputFormat.cpp @@ -64,7 +64,6 @@ void MySQLOutputFormat::initialize() } } - void MySQLOutputFormat::consume(Chunk chunk) { initialize(); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 4276160f514..dc3e863b841 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -40,18 +40,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -struct ReadFromMergeTree::AnalysisResult -{ - RangesInDataParts parts_with_ranges; - MergeTreeDataSelectSamplingData sampling; - IndexStats index_stats; - Names column_names_to_read; - ReadFromMergeTree::ReadType read_type = ReadFromMergeTree::ReadType::Default; - UInt64 selected_rows = 0; - UInt64 selected_marks = 0; - UInt64 selected_parts = 0; -}; - static MergeTreeReaderSettings getMergeTreeReaderSettings(const ContextPtr & context) { const auto & settings = context->getSettingsRef(); @@ -84,7 +72,8 @@ ReadFromMergeTree::ReadFromMergeTree( size_t num_streams_, bool sample_factor_column_queried_, std::shared_ptr max_block_numbers_to_read_, - Poco::Logger * log_) + Poco::Logger * log_, + MergeTreeDataSelectAnalysisResultPtr analyzed_result_ptr_) : ISourceStep(DataStream{.header = MergeTreeBaseSelectProcessor::transformHeader( metadata_snapshot_->getSampleBlockForColumns(real_column_names_, data_.getVirtuals(), data_.getStorageID()), getPrewhereInfo(query_info_), @@ -108,6 +97,7 @@ ReadFromMergeTree::ReadFromMergeTree( , sample_factor_column_queried(sample_factor_column_queried_) , max_block_numbers_to_read(std::move(max_block_numbers_to_read_)) , log(log_) + , analyzed_result_ptr(analyzed_result_ptr_) { if (sample_factor_column_queried) { @@ -779,7 +769,34 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( return Pipe::unitePipes(std::move(partition_pipes)); } -ReadFromMergeTree::AnalysisResult ReadFromMergeTree::selectRangesToRead(MergeTreeData::DataPartsVector parts) const +MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToRead(MergeTreeData::DataPartsVector parts) const +{ + return selectRangesToRead( + std::move(parts), + metadata_snapshot_base, + metadata_snapshot, + query_info, + context, + requested_num_streams, + max_block_numbers_to_read, + data, + real_column_names, + sample_factor_column_queried, + log); +} + +MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToRead( + MergeTreeData::DataPartsVector parts, + const StorageMetadataPtr & metadata_snapshot_base, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + ContextPtr context, + unsigned num_streams, + std::shared_ptr max_block_numbers_to_read, + const MergeTreeData & data, + const Names & real_column_names, + bool sample_factor_column_queried, + Poco::Logger * log) { AnalysisResult result; const auto & settings = context->getSettingsRef(); @@ -788,7 +805,7 @@ ReadFromMergeTree::AnalysisResult ReadFromMergeTree::selectRangesToRead(MergeTre auto part_values = MergeTreeDataSelectExecutor::filterPartsByVirtualColumns(data, parts, query_info.query, context); if (part_values && part_values->empty()) - return result; + return std::make_shared(MergeTreeDataSelectAnalysisResult{.result = std::move(result)}); result.column_names_to_read = real_column_names; @@ -808,44 +825,66 @@ ReadFromMergeTree::AnalysisResult ReadFromMergeTree::selectRangesToRead(MergeTre if (settings.force_primary_key && key_condition.alwaysUnknownOrTrue()) { - throw Exception( - ErrorCodes::INDEX_NOT_USED, - "Primary key ({}) is not used and setting 'force_primary_key' is set.", - fmt::join(primary_key_columns, ", ")); + return std::make_shared(MergeTreeDataSelectAnalysisResult{ + .result = std::make_exception_ptr(Exception( + ErrorCodes::INDEX_NOT_USED, + "Primary key ({}) is not used and setting 'force_primary_key' is set.", + fmt::join(primary_key_columns, ", ")))}); } LOG_DEBUG(log, "Key condition: {}", key_condition.toString()); const auto & select = query_info.query->as(); - MergeTreeDataSelectExecutor::filterPartsByPartition( - parts, part_values, metadata_snapshot_base, data, query_info, context, - max_block_numbers_to_read.get(), log, result.index_stats); - - result.sampling = MergeTreeDataSelectExecutor::getSampling( - select, metadata_snapshot->getColumns().getAllPhysical(), parts, key_condition, - data, metadata_snapshot, context, sample_factor_column_queried, log); - - if (result.sampling.read_nothing) - return result; - size_t total_marks_pk = 0; - for (const auto & part : parts) - total_marks_pk += part->index_granularity.getMarksCountWithoutFinal(); + size_t parts_before_pk = 0; + try + { + MergeTreeDataSelectExecutor::filterPartsByPartition( + parts, + part_values, + metadata_snapshot_base, + data, + query_info, + context, + max_block_numbers_to_read.get(), + log, + result.index_stats); - size_t parts_before_pk = parts.size(); + result.sampling = MergeTreeDataSelectExecutor::getSampling( + select, + metadata_snapshot->getColumns().getAllPhysical(), + parts, + key_condition, + data, + metadata_snapshot, + context, + sample_factor_column_queried, + log); - result.parts_with_ranges = MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipIndexes( - std::move(parts), - metadata_snapshot, - query_info, - context, - key_condition, - reader_settings, - log, - requested_num_streams, - result.index_stats, - true /* use_skip_indexes */, - true /* check_limits */); + if (result.sampling.read_nothing) + return std::make_shared(MergeTreeDataSelectAnalysisResult{.result = std::move(result)}); + + for (const auto & part : parts) + total_marks_pk += part->index_granularity.getMarksCountWithoutFinal(); + parts_before_pk = parts.size(); + + auto reader_settings = getMergeTreeReaderSettings(context); + result.parts_with_ranges = MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipIndexes( + std::move(parts), + metadata_snapshot, + query_info, + context, + key_condition, + reader_settings, + log, + num_streams, + result.index_stats, + true /* use_skip_indexes */); + } + catch (...) + { + return std::make_shared(MergeTreeDataSelectAnalysisResult{.result = std::current_exception()}); + } size_t sum_marks_pk = total_marks_pk; for (const auto & stat : result.index_stats) @@ -862,23 +901,15 @@ ReadFromMergeTree::AnalysisResult ReadFromMergeTree::selectRangesToRead(MergeTre sum_marks += part.getMarksCount(); sum_rows += part.getRowsCount(); } - result.selected_parts = result.parts_with_ranges.size(); - result.selected_marks = sum_marks; - result.selected_rows = sum_rows; - LOG_DEBUG( - log, - "Selected {}/{} parts by partition key, {} parts by primary key, {}/{} marks by primary key, {} marks to read from {} ranges", - parts_before_pk, - total_parts, - result.parts_with_ranges.size(), - sum_marks_pk, - total_marks_pk, - sum_marks, - sum_ranges); - ProfileEvents::increment(ProfileEvents::SelectedParts, result.parts_with_ranges.size()); - ProfileEvents::increment(ProfileEvents::SelectedRanges, sum_ranges); - ProfileEvents::increment(ProfileEvents::SelectedMarks, sum_marks); + result.total_parts = total_parts; + result.parts_before_pk = parts_before_pk; + result.selected_parts = result.parts_with_ranges.size(); + result.selected_ranges = sum_ranges; + result.selected_marks = sum_marks; + result.selected_marks_pk = sum_marks_pk; + result.total_marks_pk = total_marks_pk; + result.selected_rows = sum_rows; const auto & input_order_info = query_info.input_order_info ? query_info.input_order_info @@ -888,12 +919,36 @@ ReadFromMergeTree::AnalysisResult ReadFromMergeTree::selectRangesToRead(MergeTre result.read_type = (input_order_info->direction > 0) ? ReadType::InOrder : ReadType::InReverseOrder; - return result; + return std::make_shared(MergeTreeDataSelectAnalysisResult{.result = std::move(result)}); +} + +ReadFromMergeTree::AnalysisResult ReadFromMergeTree::getAnalysisResult() const +{ + auto result_ptr = analyzed_result_ptr ? analyzed_result_ptr : selectRangesToRead(prepared_parts); + if (std::holds_alternative(result_ptr->result)) + std::rethrow_exception(std::move(std::get(result_ptr->result))); + + return std::get(result_ptr->result); } void ReadFromMergeTree::initializePipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings &) { - auto result = selectRangesToRead(prepared_parts); + auto result = getAnalysisResult(); + LOG_DEBUG( + log, + "Selected {}/{} parts by partition key, {} parts by primary key, {}/{} marks by primary key, {} marks to read from {} ranges", + result.parts_before_pk, + result.total_parts, + result.selected_parts, + result.selected_marks_pk, + result.total_marks_pk, + result.selected_marks, + result.selected_ranges); + + ProfileEvents::increment(ProfileEvents::SelectedParts, result.selected_parts); + ProfileEvents::increment(ProfileEvents::SelectedRanges, result.selected_ranges); + ProfileEvents::increment(ProfileEvents::SelectedMarks, result.selected_marks); + auto query_id_holder = MergeTreeDataSelectExecutor::checkLimits(data, result.parts_with_ranges, context); if (result.parts_with_ranges.empty()) @@ -1084,7 +1139,7 @@ static const char * readTypeToString(ReadFromMergeTree::ReadType type) void ReadFromMergeTree::describeActions(FormatSettings & format_settings) const { - auto result = selectRangesToRead(prepared_parts); + auto result = getAnalysisResult(); std::string prefix(format_settings.offset, format_settings.indent_char); format_settings.out << prefix << "ReadType: " << readTypeToString(result.read_type) << '\n'; @@ -1097,7 +1152,7 @@ void ReadFromMergeTree::describeActions(FormatSettings & format_settings) const void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const { - auto result = selectRangesToRead(prepared_parts); + auto result = getAnalysisResult(); map.add("Read Type", readTypeToString(result.read_type)); if (!result.index_stats.empty()) { @@ -1108,7 +1163,7 @@ void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const void ReadFromMergeTree::describeIndexes(FormatSettings & format_settings) const { - auto result = selectRangesToRead(prepared_parts); + auto result = getAnalysisResult(); auto index_stats = std::move(result.index_stats); std::string prefix(format_settings.offset, format_settings.indent_char); @@ -1160,7 +1215,7 @@ void ReadFromMergeTree::describeIndexes(FormatSettings & format_settings) const void ReadFromMergeTree::describeIndexes(JSONBuilder::JSONMap & map) const { - auto result = selectRangesToRead(prepared_parts); + auto result = getAnalysisResult(); auto index_stats = std::move(result.index_stats); if (!index_stats.empty()) @@ -1215,4 +1270,20 @@ void ReadFromMergeTree::describeIndexes(JSONBuilder::JSONMap & map) const } } +bool MergeTreeDataSelectAnalysisResult::error() const +{ + return std::holds_alternative(result); +} + +size_t MergeTreeDataSelectAnalysisResult::marks() const +{ + if (std::holds_alternative(result)) + std::rethrow_exception(std::move(std::get(result))); + + const auto & index_stats = std::get(result).index_stats; + if (index_stats.empty()) + return 0; + return index_stats.back().num_granules_after; +} + } diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index e83746c3ff0..fc06314ee0c 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -9,6 +9,18 @@ using PartitionIdToMaxBlock = std::unordered_map; class Pipe; +struct MergeTreeDataSelectSamplingData +{ + bool use_sampling = false; + bool read_nothing = false; + Float64 used_sample_factor = 1.0; + std::shared_ptr filter_function; + ActionsDAGPtr filter_expression; +}; + +struct MergeTreeDataSelectAnalysisResult; +using MergeTreeDataSelectAnalysisResultPtr = std::shared_ptr; + /// This step is created to read from MergeTree* table. /// For now, it takes a list of parts and creates source from it. class ReadFromMergeTree final : public ISourceStep @@ -54,6 +66,23 @@ public: InReverseOrder, }; + struct AnalysisResult + { + RangesInDataParts parts_with_ranges; + MergeTreeDataSelectSamplingData sampling; + IndexStats index_stats; + Names column_names_to_read; + ReadFromMergeTree::ReadType read_type = ReadFromMergeTree::ReadType::Default; + UInt64 total_parts = 0; + UInt64 parts_before_pk = 0; + UInt64 selected_parts = 0; + UInt64 selected_ranges = 0; + UInt64 selected_marks = 0; + UInt64 selected_marks_pk = 0; + UInt64 total_marks_pk = 0; + UInt64 selected_rows = 0; + }; + ReadFromMergeTree( MergeTreeData::DataPartsVector parts_, Names real_column_names_, @@ -67,7 +96,8 @@ public: size_t num_streams_, bool sample_factor_column_queried_, std::shared_ptr max_block_numbers_to_read_, - Poco::Logger * log_ + Poco::Logger * log_, + MergeTreeDataSelectAnalysisResultPtr analyzed_result_ptr_ ); String getName() const override { return "ReadFromMergeTree"; } @@ -84,6 +114,20 @@ public: UInt64 getSelectedParts() const { return selected_parts; } UInt64 getSelectedRows() const { return selected_rows; } UInt64 getSelectedMarks() const { return selected_marks; } + + static MergeTreeDataSelectAnalysisResultPtr selectRangesToRead( + MergeTreeData::DataPartsVector parts, + const StorageMetadataPtr & metadata_snapshot_base, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + ContextPtr context, + unsigned num_streams, + std::shared_ptr max_block_numbers_to_read, + const MergeTreeData & data, + const Names & real_column_names, + bool sample_factor_column_queried, + Poco::Logger * log); + private: const MergeTreeReaderSettings reader_settings; @@ -137,8 +181,17 @@ private: const Names & column_names, ActionsDAGPtr & out_projection); - struct AnalysisResult; - AnalysisResult selectRangesToRead(MergeTreeData::DataPartsVector parts) const; + MergeTreeDataSelectAnalysisResultPtr selectRangesToRead(MergeTreeData::DataPartsVector parts) const; + ReadFromMergeTree::AnalysisResult getAnalysisResult() const; + MergeTreeDataSelectAnalysisResultPtr analyzed_result_ptr; +}; + +struct MergeTreeDataSelectAnalysisResult +{ + std::variant result; + + bool error() const; + size_t marks() const; }; } diff --git a/src/Processors/Transforms/CreatingSetsTransform.cpp b/src/Processors/Transforms/CreatingSetsTransform.cpp index 6f69765ee23..d9b383030d3 100644 --- a/src/Processors/Transforms/CreatingSetsTransform.cpp +++ b/src/Processors/Transforms/CreatingSetsTransform.cpp @@ -84,7 +84,7 @@ void CreatingSetsTransform::init() is_initialized = true; if (subquery.set) - subquery.set->setHeader(getInputPort().getHeader()); + subquery.set->setHeader(getInputPort().getHeader().getColumnsWithTypeAndName()); watch.restart(); startSubquery(); @@ -97,7 +97,7 @@ void CreatingSetsTransform::consume(Chunk chunk) if (!done_with_set) { - if (!subquery.set->insertFromBlock(block)) + if (!subquery.set->insertFromBlock(block.getColumnsWithTypeAndName())) done_with_set = true; } diff --git a/src/Processors/Transforms/JoiningTransform.cpp b/src/Processors/Transforms/JoiningTransform.cpp index e402fd788bc..95a12e2291d 100644 --- a/src/Processors/Transforms/JoiningTransform.cpp +++ b/src/Processors/Transforms/JoiningTransform.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include @@ -114,7 +113,7 @@ void JoiningTransform::work() } else { - if (!non_joined_stream) + if (!non_joined_blocks) { if (!finish_counter || !finish_counter->isLast()) { @@ -122,15 +121,15 @@ void JoiningTransform::work() return; } - non_joined_stream = join->createStreamWithNonJoinedRows(outputs.front().getHeader(), max_block_size); - if (!non_joined_stream) + non_joined_blocks = join->getNonJoinedBlocks(outputs.front().getHeader(), max_block_size); + if (!non_joined_blocks) { process_non_joined = false; return; } } - auto block = non_joined_stream->read(); + Block block = non_joined_blocks->read(); if (!block) { process_non_joined = false; diff --git a/src/Processors/Transforms/JoiningTransform.h b/src/Processors/Transforms/JoiningTransform.h index 98038946f3b..96c4032dabc 100644 --- a/src/Processors/Transforms/JoiningTransform.h +++ b/src/Processors/Transforms/JoiningTransform.h @@ -8,8 +8,7 @@ namespace DB class IJoin; using JoinPtr = std::shared_ptr; -class IBlockInputStream; -using BlockInputStreamPtr = std::shared_ptr; +class NotJoinedBlocks; /// Join rows to chunk form left table. /// This transform usually has two input ports and one output. @@ -76,7 +75,7 @@ private: ExtraBlockPtr not_processed; FinishCounterPtr finish_counter; - BlockInputStreamPtr non_joined_stream; + std::shared_ptr non_joined_blocks; size_t max_block_size; Block readExecute(Chunk & chunk); diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp index 1b8406682ea..132bdb7b327 100644 --- a/src/Processors/Transforms/WindowTransform.cpp +++ b/src/Processors/Transforms/WindowTransform.cpp @@ -1764,21 +1764,21 @@ void registerWindowFunctions(AggregateFunctionFactory & factory) { return std::make_shared(name, argument_types, parameters); - }, properties}); + }, properties}, AggregateFunctionFactory::CaseInsensitive); factory.registerFunction("dense_rank", {[](const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *) { return std::make_shared(name, argument_types, parameters); - }, properties}); + }, properties}, AggregateFunctionFactory::CaseInsensitive); factory.registerFunction("row_number", {[](const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *) { return std::make_shared(name, argument_types, parameters); - }, properties}); + }, properties}, AggregateFunctionFactory::CaseInsensitive); factory.registerFunction("lagInFrame", {[](const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *) @@ -1799,7 +1799,7 @@ void registerWindowFunctions(AggregateFunctionFactory & factory) { return std::make_shared( name, argument_types, parameters); - }, properties}); + }, properties}, AggregateFunctionFactory::CaseInsensitive); } } diff --git a/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp b/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp index 8d8a4761657..9e64bd954fa 100644 --- a/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp +++ b/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -11,6 +12,8 @@ #include #include #include +#include "IO/CompressionMethod.h" +#include "Parsers/ASTLiteral.h" namespace DB @@ -20,6 +23,7 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int INVALID_USAGE_OF_INPUT; + extern const int UNKNOWN_TYPE_OF_QUERY; } @@ -35,6 +39,9 @@ Pipe getSourceFromFromASTInsertQuery( if (!ast_insert_query) throw Exception("Logical error: query requires data to insert, but it is not INSERT query", ErrorCodes::LOGICAL_ERROR); + if (ast_insert_query->infile && context->getApplicationType() == Context::ApplicationType::SERVER) + throw Exception("Query has infile and was send directly to server", ErrorCodes::UNKNOWN_TYPE_OF_QUERY); + String format = ast_insert_query->format; if (format.empty()) { @@ -48,20 +55,33 @@ Pipe getSourceFromFromASTInsertQuery( auto input_buffer_ast_part = std::make_unique( ast_insert_query->data, ast_insert_query->data ? ast_insert_query->end - ast_insert_query->data : 0); - ConcatReadBuffer::ReadBuffers buffers; - if (ast_insert_query->data) - buffers.push_back(input_buffer_ast_part.get()); + std::unique_ptr input_buffer; - if (input_buffer_tail_part) - buffers.push_back(input_buffer_tail_part); + if (ast_insert_query->infile) + { + /// Data can be from infile + const auto & in_file_node = ast_insert_query->infile->as(); + const auto in_file = in_file_node.value.safeGet(); - /** NOTE Must not read from 'input_buffer_tail_part' before read all between 'ast_insert_query.data' and 'ast_insert_query.end'. - * - because 'query.data' could refer to memory piece, used as buffer for 'input_buffer_tail_part'. - */ + input_buffer = wrapReadBufferWithCompressionMethod(std::make_unique(in_file), chooseCompressionMethod(in_file, "")); + } + else + { + ConcatReadBuffer::ReadBuffers buffers; + if (ast_insert_query->data) + buffers.push_back(input_buffer_ast_part.get()); - auto input_buffer_contacenated = std::make_unique(buffers); + if (input_buffer_tail_part) + buffers.push_back(input_buffer_tail_part); - auto source = FormatFactory::instance().getInput(format, *input_buffer_contacenated, header, context, context->getSettings().max_insert_block_size); + /** NOTE Must not read from 'input_buffer_tail_part' before read all between 'ast_insert_query.data' and 'ast_insert_query.end'. + * - because 'query.data' could refer to memory piece, used as buffer for 'input_buffer_tail_part'. + */ + + input_buffer = std::make_unique(buffers); + } + + auto source = FormatFactory::instance().getInput(format, *input_buffer, header, context, context->getSettings().max_insert_block_size); Pipe pipe(source); if (context->getSettingsRef().input_format_defaults_for_omitted_fields && ast_insert_query->table_id && !input_function) @@ -79,7 +99,7 @@ Pipe getSourceFromFromASTInsertQuery( } source->addBuffer(std::move(input_buffer_ast_part)); - source->addBuffer(std::move(input_buffer_contacenated)); + source->addBuffer(std::move(input_buffer)); return pipe; } diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index b90b0c33f17..f0c6e208323 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -54,7 +55,6 @@ namespace ErrorCodes extern const int NETWORK_ERROR; extern const int NO_DATA_TO_INSERT; extern const int SUPPORT_IS_DISABLED; - extern const int UNKNOWN_DATABASE; } namespace @@ -560,7 +560,7 @@ namespace IServer & iserver; Poco::Logger * log = nullptr; - std::shared_ptr session; + std::optional session; ContextMutablePtr query_context; std::optional query_scope; String query_text; @@ -689,34 +689,20 @@ namespace password = ""; } - /// Create context. - query_context = Context::createCopy(iserver.context()); - /// Authentication. - query_context->setUser(user, password, user_address); - query_context->setCurrentQueryId(query_info.query_id()); - if (!quota_key.empty()) - query_context->setQuotaKey(quota_key); + session.emplace(iserver.context(), ClientInfo::Interface::GRPC); + session->authenticate(user, password, user_address); + session->getClientInfo().quota_key = quota_key; /// The user could specify session identifier and session timeout. /// It allows to modify settings, create temporary tables and reuse them in subsequent requests. if (!query_info.session_id().empty()) { - session = query_context->acquireNamedSession( + session->makeSessionContext( query_info.session_id(), getSessionTimeout(query_info, iserver.config()), query_info.session_check()); - query_context = Context::createCopy(session->context); - query_context->setSessionContext(session->context); } - query_scope.emplace(query_context); - - /// Set client info. - ClientInfo & client_info = query_context->getClientInfo(); - client_info.query_kind = ClientInfo::QueryKind::INITIAL_QUERY; - client_info.interface = ClientInfo::Interface::GRPC; - client_info.initial_user = client_info.current_user; - client_info.initial_query_id = client_info.current_query_id; - client_info.initial_address = client_info.current_address; + query_context = session->makeQueryContext(); /// Prepare settings. SettingsChanges settings_changes; @@ -726,11 +712,14 @@ namespace } query_context->checkSettingsConstraints(settings_changes); query_context->applySettingsChanges(settings_changes); - const Settings & settings = query_context->getSettingsRef(); + + query_context->setCurrentQueryId(query_info.query_id()); + query_scope.emplace(query_context); /// Prepare for sending exceptions and logs. - send_exception_with_stacktrace = query_context->getSettingsRef().calculate_text_stack_trace; - const auto client_logs_level = query_context->getSettingsRef().send_logs_level; + const Settings & settings = query_context->getSettingsRef(); + send_exception_with_stacktrace = settings.calculate_text_stack_trace; + const auto client_logs_level = settings.send_logs_level; if (client_logs_level != LogsLevel::none) { logs_queue = std::make_shared(); @@ -741,14 +730,10 @@ namespace /// Set the current database if specified. if (!query_info.database().empty()) - { - if (!DatabaseCatalog::instance().isDatabaseExist(query_info.database())) - throw Exception("Database " + query_info.database() + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); query_context->setCurrentDatabase(query_info.database()); - } /// The interactive delay will be used to show progress. - interactive_delay = query_context->getSettingsRef().interactive_delay; + interactive_delay = settings.interactive_delay; query_context->setProgressCallback([this](const Progress & value) { return progress.incrementPiecewiseAtomically(value); }); /// Parse the query. @@ -1254,8 +1239,6 @@ namespace io = {}; query_scope.reset(); query_context.reset(); - if (session) - session->release(); session.reset(); } diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 7c9ff0637f0..0492b58dc88 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -261,6 +262,7 @@ void HTTPHandler::pushDelayedResults(Output & used_output) HTTPHandler::HTTPHandler(IServer & server_, const std::string & name) : server(server_) , log(&Poco::Logger::get(name)) + , default_settings(server.context()->getSettingsRef()) { server_display_name = server.config().getString("display_name", getFQDNOrHostName()); } @@ -268,14 +270,10 @@ HTTPHandler::HTTPHandler(IServer & server_, const std::string & name) /// We need d-tor to be present in this translation unit to make it play well with some /// forward decls in the header. Other than that, the default d-tor would be OK. -HTTPHandler::~HTTPHandler() -{ - (void)this; -} +HTTPHandler::~HTTPHandler() = default; bool HTTPHandler::authenticateUser( - ContextMutablePtr context, HTTPServerRequest & request, HTMLForm & params, HTTPServerResponse & response) @@ -352,7 +350,7 @@ bool HTTPHandler::authenticateUser( else { if (!request_credentials) - request_credentials = request_context->makeGSSAcceptorContext(); + request_credentials = server.context()->makeGSSAcceptorContext(); auto * gss_acceptor_context = dynamic_cast(request_credentials.get()); if (!gss_acceptor_context) @@ -378,10 +376,7 @@ bool HTTPHandler::authenticateUser( } /// Set client info. It will be used for quota accounting parameters in 'setUser' method. - - ClientInfo & client_info = context->getClientInfo(); - client_info.query_kind = ClientInfo::QueryKind::INITIAL_QUERY; - client_info.interface = ClientInfo::Interface::HTTP; + ClientInfo & client_info = session->getClientInfo(); ClientInfo::HTTPMethod http_method = ClientInfo::HTTPMethod::UNKNOWN; if (request.getMethod() == HTTPServerRequest::HTTP_GET) @@ -393,10 +388,11 @@ bool HTTPHandler::authenticateUser( client_info.http_user_agent = request.get("User-Agent", ""); client_info.http_referer = request.get("Referer", ""); client_info.forwarded_for = request.get("X-Forwarded-For", ""); + client_info.quota_key = quota_key; try { - context->setUser(*request_credentials, request.clientAddress()); + session->authenticate(*request_credentials, request.clientAddress()); } catch (const Authentication::Require & required_credentials) { @@ -413,7 +409,7 @@ bool HTTPHandler::authenticateUser( } catch (const Authentication::Require & required_credentials) { - request_credentials = request_context->makeGSSAcceptorContext(); + request_credentials = server.context()->makeGSSAcceptorContext(); if (required_credentials.getRealm().empty()) response.set("WWW-Authenticate", "Negotiate"); @@ -426,20 +422,11 @@ bool HTTPHandler::authenticateUser( } request_credentials.reset(); - - if (!quota_key.empty()) - context->setQuotaKey(quota_key); - - /// Query sent through HTTP interface is initial. - client_info.initial_user = client_info.current_user; - client_info.initial_address = client_info.current_address; - return true; } void HTTPHandler::processQuery( - ContextMutablePtr context, HTTPServerRequest & request, HTMLForm & params, HTTPServerResponse & response, @@ -450,13 +437,11 @@ void HTTPHandler::processQuery( LOG_TRACE(log, "Request URI: {}", request.getURI()); - if (!authenticateUser(context, request, params, response)) + if (!authenticateUser(request, params, response)) return; // '401 Unauthorized' response with 'Negotiate' has been sent at this point. /// The user could specify session identifier and session timeout. /// It allows to modify settings, create temporary tables and reuse them in subsequent requests. - - std::shared_ptr session; String session_id; std::chrono::steady_clock::duration session_timeout; bool session_is_set = params.has("session_id"); @@ -467,43 +452,30 @@ void HTTPHandler::processQuery( session_id = params.get("session_id"); session_timeout = parseSessionTimeout(config, params); std::string session_check = params.get("session_check", ""); - - session = context->acquireNamedSession(session_id, session_timeout, session_check == "1"); - - context->copyFrom(session->context); /// FIXME: maybe move this part to HandleRequest(), copyFrom() is used only here. - context->setSessionContext(session->context); + session->makeSessionContext(session_id, session_timeout, session_check == "1"); } - SCOPE_EXIT({ - if (session) - session->release(); - }); - // Parse the OpenTelemetry traceparent header. // Disable in Arcadia -- it interferes with the // test_clickhouse.TestTracing.test_tracing_via_http_proxy[traceparent] test. + ClientInfo client_info = session->getClientInfo(); #if !defined(ARCADIA_BUILD) if (request.has("traceparent")) { std::string opentelemetry_traceparent = request.get("traceparent"); std::string error; - if (!context->getClientInfo().client_trace_context.parseTraceparentHeader( + if (!client_info.client_trace_context.parseTraceparentHeader( opentelemetry_traceparent, error)) { throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, "Failed to parse OpenTelemetry traceparent header '{}': {}", opentelemetry_traceparent, error); } - - context->getClientInfo().client_trace_context.tracestate = request.get("tracestate", ""); + client_info.client_trace_context.tracestate = request.get("tracestate", ""); } #endif - // Set the query id supplied by the user, if any, and also update the OpenTelemetry fields. - context->setCurrentQueryId(params.get("query_id", request.get("X-ClickHouse-Query-Id", ""))); - - ClientInfo & client_info = context->getClientInfo(); - client_info.initial_query_id = client_info.current_query_id; + auto context = session->makeQueryContext(std::move(client_info)); /// The client can pass a HTTP header indicating supported compression method (gzip or deflate). String http_response_compression_methods = request.get("Accept-Encoding", ""); @@ -568,7 +540,7 @@ void HTTPHandler::processQuery( if (buffer_until_eof) { - const std::string tmp_path(context->getTemporaryVolume()->getDisk()->getPath()); + const std::string tmp_path(server.context()->getTemporaryVolume()->getDisk()->getPath()); const std::string tmp_path_template(tmp_path + "http_buffers/"); auto create_tmp_disk_buffer = [tmp_path_template] (const WriteBufferPtr &) @@ -714,6 +686,9 @@ void HTTPHandler::processQuery( context->checkSettingsConstraints(settings_changes); context->applySettingsChanges(settings_changes); + // Set the query id supplied by the user, if any, and also update the OpenTelemetry fields. + context->setCurrentQueryId(params.get("query_id", request.get("X-ClickHouse-Query-Id", ""))); + const auto & query = getQuery(request, params, context); std::unique_ptr in_param = std::make_unique(query); in = has_external_data ? std::move(in_param) : std::make_unique(*in_param, *in_post_maybe_compressed); @@ -864,23 +839,10 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse setThreadName("HTTPHandler"); ThreadStatus thread_status; - SCOPE_EXIT({ - // If there is no request_credentials instance waiting for the next round, then the request is processed, - // so no need to preserve request_context either. - // Needs to be performed with respect to the other destructors in the scope though. - if (!request_credentials) - request_context.reset(); - }); - - if (!request_context) - { - // Context should be initialized before anything, for correct memory accounting. - request_context = Context::createCopy(server.context()); - request_credentials.reset(); - } - - /// Cannot be set here, since query_id is unknown. + session = std::make_unique(server.context(), ClientInfo::Interface::HTTP); + SCOPE_EXIT({ session.reset(); }); std::optional query_scope; + Output used_output; /// In case of exception, send stack trace to client. @@ -894,7 +856,7 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse if (request.getVersion() == HTTPServerRequest::HTTP_1_1) response.setChunkedTransferEncoding(true); - HTMLForm params(request_context->getSettingsRef(), request); + HTMLForm params(default_settings, request); with_stacktrace = params.getParsed("stacktrace", false); /// FIXME: maybe this check is already unnecessary. @@ -906,7 +868,7 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse ErrorCodes::HTTP_LENGTH_REQUIRED); } - processQuery(request_context, request, params, response, used_output, query_scope); + processQuery(request, params, response, used_output, query_scope); LOG_DEBUG(log, (request_credentials ? "Authentication in progress..." : "Done processing query")); } catch (...) diff --git a/src/Server/HTTPHandler.h b/src/Server/HTTPHandler.h index 2149a7ca55c..98f573f8cef 100644 --- a/src/Server/HTTPHandler.h +++ b/src/Server/HTTPHandler.h @@ -18,8 +18,10 @@ namespace Poco { class Logger; } namespace DB { +class Session; class Credentials; class IServer; +struct Settings; class WriteBufferFromHTTPServerResponse; using CompiledRegexPtr = std::shared_ptr; @@ -71,25 +73,30 @@ private: CurrentMetrics::Increment metric_increment{CurrentMetrics::HTTPConnection}; - // The request_context and the request_credentials instances may outlive a single request/response loop. + /// Reference to the immutable settings in the global context. + /// Those settings are used only to extract a http request's parameters. + /// See settings http_max_fields, http_max_field_name_size, http_max_field_value_size in HTMLForm. + const Settings & default_settings; + + // session is reset at the end of each request/response. + std::unique_ptr session; + + // The request_credential instance may outlive a single request/response loop. // This happens only when the authentication mechanism requires more than a single request/response exchange (e.g., SPNEGO). - ContextMutablePtr request_context; std::unique_ptr request_credentials; // Returns true when the user successfully authenticated, - // the request_context instance will be configured accordingly, and the request_credentials instance will be dropped. + // the session instance will be configured accordingly, and the request_credentials instance will be dropped. // Returns false when the user is not authenticated yet, and the 'Negotiate' response is sent, - // the request_context and request_credentials instances are preserved. + // the session and request_credentials instances are preserved. // Throws an exception if authentication failed. bool authenticateUser( - ContextMutablePtr context, HTTPServerRequest & request, HTMLForm & params, HTTPServerResponse & response); /// Also initializes 'used_output'. void processQuery( - ContextMutablePtr context, HTTPServerRequest & request, HTMLForm & params, HTTPServerResponse & response, diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp index 52182257ac9..93f4bff46c2 100644 --- a/src/Server/MySQLHandler.cpp +++ b/src/Server/MySQLHandler.cpp @@ -3,11 +3,12 @@ #include #include #include -#include #include #include #include #include +#include +#include #include #include #include @@ -18,9 +19,8 @@ #include #include #include -#include -#include #include +#include #if !defined(ARCADIA_BUILD) # include @@ -70,7 +70,6 @@ MySQLHandler::MySQLHandler(IServer & server_, const Poco::Net::StreamSocket & so , server(server_) , log(&Poco::Logger::get("MySQLHandler")) , connection_id(connection_id_) - , connection_context(Context::createCopy(server.context())) , auth_plugin(new MySQLProtocol::Authentication::Native41()) { server_capabilities = CLIENT_PROTOCOL_41 | CLIENT_SECURE_CONNECTION | CLIENT_PLUGIN_AUTH | CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA | CLIENT_CONNECT_WITH_DB | CLIENT_DEPRECATE_EOF; @@ -87,11 +86,11 @@ void MySQLHandler::run() { setThreadName("MySQLHandler"); ThreadStatus thread_status; - connection_context->makeSessionContext(); - connection_context->getClientInfo().interface = ClientInfo::Interface::MYSQL; - connection_context->setDefaultFormat("MySQLWire"); - connection_context->getClientInfo().connection_id = connection_id; - connection_context->getClientInfo().query_kind = ClientInfo::QueryKind::INITIAL_QUERY; + + session = std::make_unique(server.context(), ClientInfo::Interface::MYSQL); + SCOPE_EXIT({ session.reset(); }); + + session->getClientInfo().connection_id = connection_id; in = std::make_shared(socket()); out = std::make_shared(socket()); @@ -125,14 +124,12 @@ void MySQLHandler::run() authenticate(handshake_response.username, handshake_response.auth_plugin_name, handshake_response.auth_response); - connection_context->getClientInfo().initial_user = handshake_response.username; - try { + session->makeSessionContext(); + session->sessionContext()->setDefaultFormat("MySQLWire"); if (!handshake_response.database.empty()) - connection_context->setCurrentDatabase(handshake_response.database); - connection_context->setCurrentQueryId(Poco::format("mysql:%lu", connection_id)); - + session->sessionContext()->setCurrentDatabase(handshake_response.database); } catch (const Exception & exc) { @@ -249,15 +246,13 @@ void MySQLHandler::authenticate(const String & user_name, const String & auth_pl try { // For compatibility with JavaScript MySQL client, Native41 authentication plugin is used when possible (if password is specified using double SHA1). Otherwise SHA256 plugin is used. - auto user = connection_context->getAccessControlManager().read(user_name); - const DB::Authentication::Type user_auth_type = user->authentication.getType(); - if (user_auth_type == DB::Authentication::SHA256_PASSWORD) + if (session->getAuthenticationType(user_name) == DB::Authentication::SHA256_PASSWORD) { authPluginSSL(); } std::optional auth_response = auth_plugin_name == auth_plugin->getName() ? std::make_optional(initial_auth_response) : std::nullopt; - auth_plugin->authenticate(user_name, auth_response, connection_context, packet_endpoint, secure_connection, socket().peerAddress()); + auth_plugin->authenticate(user_name, *session, auth_response, packet_endpoint, secure_connection, socket().peerAddress()); } catch (const Exception & exc) { @@ -273,7 +268,7 @@ void MySQLHandler::comInitDB(ReadBuffer & payload) String database; readStringUntilEOF(database, payload); LOG_DEBUG(log, "Setting current database to {}", database); - connection_context->setCurrentDatabase(database); + session->sessionContext()->setCurrentDatabase(database); packet_endpoint->sendPacket(OKPacket(0, client_capabilities, 0, 0, 1), true); } @@ -281,8 +276,9 @@ void MySQLHandler::comFieldList(ReadBuffer & payload) { ComFieldList packet; packet.readPayloadWithUnpacked(payload); - String database = connection_context->getCurrentDatabase(); - StoragePtr table_ptr = DatabaseCatalog::instance().getTable({database, packet.table}, connection_context); + const auto session_context = session->sessionContext(); + String database = session_context->getCurrentDatabase(); + StoragePtr table_ptr = DatabaseCatalog::instance().getTable({database, packet.table}, session_context); auto metadata_snapshot = table_ptr->getInMemoryMetadataPtr(); for (const NameAndTypePair & column : metadata_snapshot->getColumns().getAll()) { @@ -329,7 +325,9 @@ void MySQLHandler::comQuery(ReadBuffer & payload) ReadBufferFromString replacement(replacement_query); - auto query_context = Context::createCopy(connection_context); + auto query_context = session->makeQueryContext(); + query_context->setCurrentQueryId(Poco::format("mysql:%lu", connection_id)); + CurrentThread::QueryScope query_scope{query_context}; std::atomic affected_rows {0}; auto prev = query_context->getProgressCallback(); @@ -341,8 +339,6 @@ void MySQLHandler::comQuery(ReadBuffer & payload) affected_rows += progress.written_rows; }); - CurrentThread::QueryScope query_scope{query_context}; - FormatSettings format_settings; format_settings.mysql_wire.client_capabilities = client_capabilities; format_settings.mysql_wire.max_packet_size = max_packet_size; diff --git a/src/Server/MySQLHandler.h b/src/Server/MySQLHandler.h index 96467797105..5258862cf23 100644 --- a/src/Server/MySQLHandler.h +++ b/src/Server/MySQLHandler.h @@ -17,6 +17,8 @@ # include #endif +#include + namespace CurrentMetrics { extern const Metric MySQLConnection; @@ -61,7 +63,7 @@ protected: uint8_t sequence_id = 0; MySQLProtocol::PacketEndpointPtr packet_endpoint; - ContextMutablePtr connection_context; + std::unique_ptr session; using ReplacementFn = std::function; using Replacements = std::unordered_map; diff --git a/src/Server/PostgreSQLHandler.cpp b/src/Server/PostgreSQLHandler.cpp index 1e98ed2e134..0716d828520 100644 --- a/src/Server/PostgreSQLHandler.cpp +++ b/src/Server/PostgreSQLHandler.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include "PostgreSQLHandler.h" #include @@ -33,7 +34,6 @@ PostgreSQLHandler::PostgreSQLHandler( std::vector> & auth_methods_) : Poco::Net::TCPServerConnection(socket_) , server(server_) - , connection_context(Context::createCopy(server.context())) , ssl_enabled(ssl_enabled_) , connection_id(connection_id_) , authentication_manager(auth_methods_) @@ -52,10 +52,9 @@ void PostgreSQLHandler::run() { setThreadName("PostgresHandler"); ThreadStatus thread_status; - connection_context->makeSessionContext(); - connection_context->getClientInfo().interface = ClientInfo::Interface::POSTGRESQL; - connection_context->setDefaultFormat("PostgreSQLWire"); - connection_context->getClientInfo().query_kind = ClientInfo::QueryKind::INITIAL_QUERY; + + session = std::make_unique(server.context(), ClientInfo::Interface::POSTGRESQL); + SCOPE_EXIT({ session.reset(); }); try { @@ -123,18 +122,15 @@ bool PostgreSQLHandler::startup() } std::unique_ptr start_up_msg = receiveStartupMessage(payload_size); - authentication_manager.authenticate(start_up_msg->user, connection_context, *message_transport, socket().peerAddress()); - - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_int_distribution dis(0, INT32_MAX); - secret_key = dis(gen); + const auto & user_name = start_up_msg->user; + authentication_manager.authenticate(user_name, *session, *message_transport, socket().peerAddress()); try { + session->makeSessionContext(); + session->sessionContext()->setDefaultFormat("PostgreSQLWire"); if (!start_up_msg->database.empty()) - connection_context->setCurrentDatabase(start_up_msg->database); - connection_context->setCurrentQueryId(Poco::format("postgres:%d:%d", connection_id, secret_key)); + session->sessionContext()->setCurrentDatabase(start_up_msg->database); } catch (const Exception & exc) { @@ -214,16 +210,15 @@ void PostgreSQLHandler::sendParameterStatusData(PostgreSQLProtocol::Messaging::S void PostgreSQLHandler::cancelRequest() { - connection_context->setCurrentQueryId(""); - connection_context->setDefaultFormat("Null"); - std::unique_ptr msg = message_transport->receiveWithPayloadSize(8); String query = Poco::format("KILL QUERY WHERE query_id = 'postgres:%d:%d'", msg->process_id, msg->secret_key); ReadBufferFromString replacement(query); - executeQuery(replacement, *out, true, connection_context, {}); + auto query_context = session->makeQueryContext(); + query_context->setCurrentQueryId(""); + executeQuery(replacement, *out, true, query_context, {}); } inline std::unique_ptr PostgreSQLHandler::receiveStartupMessage(int payload_size) @@ -269,18 +264,25 @@ void PostgreSQLHandler::processQuery() return; } - const auto & settings = connection_context->getSettingsRef(); + const auto & settings = session->sessionContext()->getSettingsRef(); std::vector queries; auto parse_res = splitMultipartQuery(query->query, queries, settings.max_query_size, settings.max_parser_depth); if (!parse_res.second) throw Exception("Cannot parse and execute the following part of query: " + String(parse_res.first), ErrorCodes::SYNTAX_ERROR); + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dis(0, INT32_MAX); + for (const auto & spl_query : queries) { - /// FIXME why do we execute all queries in a single connection context? - CurrentThread::QueryScope query_scope{connection_context}; + secret_key = dis(gen); + auto query_context = session->makeQueryContext(); + query_context->setCurrentQueryId(Poco::format("postgres:%d:%d", connection_id, secret_key)); + + CurrentThread::QueryScope query_scope{query_context}; ReadBufferFromString read_buf(spl_query); - executeQuery(read_buf, *out, false, connection_context, {}); + executeQuery(read_buf, *out, false, query_context, {}); PostgreSQLProtocol::Messaging::CommandComplete::Command command = PostgreSQLProtocol::Messaging::CommandComplete::classifyQuery(spl_query); diff --git a/src/Server/PostgreSQLHandler.h b/src/Server/PostgreSQLHandler.h index 9aaad1d7aa7..36dd62d3dec 100644 --- a/src/Server/PostgreSQLHandler.h +++ b/src/Server/PostgreSQLHandler.h @@ -18,6 +18,8 @@ namespace CurrentMetrics namespace DB { +class Session; + /** PostgreSQL wire protocol implementation. * For more info see https://www.postgresql.org/docs/current/protocol.html */ @@ -37,7 +39,7 @@ private: Poco::Logger * log = &Poco::Logger::get("PostgreSQLHandler"); IServer & server; - ContextMutablePtr connection_context; + std::unique_ptr session; bool ssl_enabled = false; Int32 connection_id = 0; Int32 secret_key = 0; diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 0339b144f09..beca726e95f 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -24,10 +24,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -47,33 +49,11 @@ namespace DB { -namespace -{ -std::string formatHTTPErrorResponse(const Poco::Util::AbstractConfiguration& config) -{ - std::string result = fmt::format( - "HTTP/1.0 400 Bad Request\r\n\r\n" - "Port {} is for clickhouse-client program\r\n", - config.getString("tcp_port")); - - if (config.has("http_port")) - { - result += fmt::format( - "You must use port {} for HTTP.\r\n", - config.getString("http_port")); - } - - return result; -} -} - - namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int ATTEMPT_TO_READ_AFTER_EOF; extern const int CLIENT_HAS_CONNECTED_TO_WRONG_PORT; - extern const int UNKNOWN_DATABASE; extern const int UNKNOWN_EXCEPTION; extern const int UNKNOWN_PACKET_FROM_CLIENT; extern const int POCO_EXCEPTION; @@ -88,11 +68,10 @@ TCPHandler::TCPHandler(IServer & server_, const Poco::Net::StreamSocket & socket , server(server_) , parse_proxy_protocol(parse_proxy_protocol_) , log(&Poco::Logger::get("TCPHandler")) - , connection_context(Context::createCopy(server.context())) - , query_context(Context::createCopy(server.context())) , server_display_name(std::move(server_display_name_)) { } + TCPHandler::~TCPHandler() { try @@ -112,16 +91,11 @@ void TCPHandler::runImpl() setThreadName("TCPHandler"); ThreadStatus thread_status; - connection_context = Context::createCopy(server.context()); - connection_context->makeSessionContext(); + session = std::make_unique(server.context(), ClientInfo::Interface::TCP); + extractConnectionSettingsFromContext(server.context()); - /// These timeouts can be changed after receiving query. - - auto global_receive_timeout = connection_context->getSettingsRef().receive_timeout; - auto global_send_timeout = connection_context->getSettingsRef().send_timeout; - - socket().setReceiveTimeout(global_receive_timeout); - socket().setSendTimeout(global_send_timeout); + socket().setReceiveTimeout(receive_timeout); + socket().setSendTimeout(send_timeout); socket().setNoDelay(true); in = std::make_shared(socket()); @@ -159,34 +133,27 @@ void TCPHandler::runImpl() try { /// We try to send error information to the client. - sendException(e, connection_context->getSettingsRef().calculate_text_stack_trace); + sendException(e, send_exception_with_stack_trace); } catch (...) {} throw; } - /// When connecting, the default database can be specified. - if (!default_database.empty()) - { - if (!DatabaseCatalog::instance().isDatabaseExist(default_database)) - { - Exception e("Database " + backQuote(default_database) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); - LOG_ERROR(log, getExceptionMessage(e, true)); - sendException(e, connection_context->getSettingsRef().calculate_text_stack_trace); - return; - } - - connection_context->setCurrentDatabase(default_database); - } - - Settings connection_settings = connection_context->getSettings(); - UInt64 idle_connection_timeout = connection_settings.idle_connection_timeout; - UInt64 poll_interval = connection_settings.poll_interval; - sendHello(); - connection_context->setProgressCallback([this] (const Progress & value) { return this->updateProgress(value); }); + if (!is_interserver_mode) /// In interserver mode queries are executed without a session context. + { + session->makeSessionContext(); + + /// If session created, then settings in session context has been updated. + /// So it's better to update the connection settings for flexibility. + extractConnectionSettingsFromContext(session->sessionContext()); + + /// When connecting, the default database could be specified. + if (!default_database.empty()) + session->sessionContext()->setCurrentDatabase(default_database); + } while (true) { @@ -208,9 +175,6 @@ void TCPHandler::runImpl() if (server.isCancelled() || in->eof()) break; - /// Set context of request. - query_context = Context::createCopy(connection_context); - Stopwatch watch; state.reset(); @@ -223,8 +187,6 @@ void TCPHandler::runImpl() std::optional exception; bool network_error = false; - bool send_exception_with_stack_trace = true; - try { /// If a user passed query-local timeouts, reset socket to initial state at the end of the query @@ -237,23 +199,22 @@ void TCPHandler::runImpl() if (!receivePacket()) continue; - /** If Query received, then settings in query_context has been updated - * So, update some other connection settings, for flexibility. - */ - { - const Settings & settings = query_context->getSettingsRef(); - idle_connection_timeout = settings.idle_connection_timeout; - poll_interval = settings.poll_interval; - } - /** If part_uuids got received in previous packet, trying to read again. */ - if (state.empty() && state.part_uuids && !receivePacket()) + if (state.empty() && state.part_uuids_to_ignore && !receivePacket()) continue; query_scope.emplace(query_context); - send_exception_with_stack_trace = query_context->getSettingsRef().calculate_text_stack_trace; + /// If query received, then settings in query_context has been updated. + /// So it's better to update the connection settings for flexibility. + extractConnectionSettingsFromContext(query_context); + + /// Sync timeouts on client and server during current query to avoid dangling queries on server + /// NOTE: We use send_timeout for the receive timeout and vice versa (change arguments ordering in TimeoutSetter), + /// because send_timeout is client-side setting which has opposite meaning on the server side. + /// NOTE: these settings are applied only for current connection (not for distributed tables' connections) + state.timeout_setter = std::make_unique(socket(), receive_timeout, send_timeout); /// Should we send internal logs to client? const auto client_logs_level = query_context->getSettingsRef().send_logs_level; @@ -266,20 +227,18 @@ void TCPHandler::runImpl() CurrentThread::setFatalErrorCallback([this]{ sendLogs(); }); } - query_context->setExternalTablesInitializer([&connection_settings, this] (ContextPtr context) + query_context->setExternalTablesInitializer([this] (ContextPtr context) { if (context != query_context) throw Exception("Unexpected context in external tables initializer", ErrorCodes::LOGICAL_ERROR); /// Get blocks of temporary tables - readData(connection_settings); + readData(); /// Reset the input stream, as we received an empty block while receiving external table data. /// So, the stream has been marked as cancelled and we can't read from it anymore. state.block_in.reset(); state.maybe_compressed_in.reset(); /// For more accurate accounting by MemoryTracker. - - state.temporary_tables_read = true; }); /// Send structure of columns to client for function input() @@ -303,15 +262,12 @@ void TCPHandler::runImpl() sendData(state.input_header); }); - query_context->setInputBlocksReaderCallback([&connection_settings, this] (ContextPtr context) -> Block + query_context->setInputBlocksReaderCallback([this] (ContextPtr context) -> Block { if (context != query_context) throw Exception("Unexpected context in InputBlocksReader", ErrorCodes::LOGICAL_ERROR); - size_t poll_interval_ms; - int receive_timeout; - std::tie(poll_interval_ms, receive_timeout) = getReadTimeouts(connection_settings); - if (!readDataNext(poll_interval_ms, receive_timeout)) + if (!readDataNext()) { state.block_in.reset(); state.maybe_compressed_in.reset(); @@ -334,15 +290,13 @@ void TCPHandler::runImpl() /// Processing Query state.io = executeQuery(state.query, query_context, false, state.stage, may_have_embedded_data); - unknown_packet_in_send_data = query_context->getSettingsRef().unknown_packet_in_send_data; - after_check_cancelled.restart(); after_send_progress.restart(); if (state.io.out) { state.need_receive_data_for_insert = true; - processInsertQuery(connection_settings); + processInsertQuery(); } else if (state.need_receive_data_for_input) // It implies pipeline execution { @@ -458,16 +412,17 @@ void TCPHandler::runImpl() try { - if (exception && !state.temporary_tables_read) - query_context->initializeExternalTablesIfSet(); + /// A query packet is always followed by one or more data packets. + /// If some of those data packets are left, try to skip them. + if (exception && !state.empty() && !state.read_all_data) + skipData(); } catch (...) { network_error = true; - LOG_WARNING(log, "Can't read external tables after query failure."); + LOG_WARNING(log, "Can't skip data packets after query failure."); } - try { /// QueryState should be cleared before QueryScope, since otherwise @@ -498,75 +453,94 @@ void TCPHandler::runImpl() } -bool TCPHandler::readDataNext(size_t poll_interval, time_t receive_timeout) +void TCPHandler::extractConnectionSettingsFromContext(const ContextPtr & context) +{ + const auto & settings = context->getSettingsRef(); + send_exception_with_stack_trace = settings.calculate_text_stack_trace; + send_timeout = settings.send_timeout; + receive_timeout = settings.receive_timeout; + poll_interval = settings.poll_interval; + idle_connection_timeout = settings.idle_connection_timeout; + interactive_delay = settings.interactive_delay; + sleep_in_send_tables_status = settings.sleep_in_send_tables_status_ms; + unknown_packet_in_send_data = settings.unknown_packet_in_send_data; + sleep_in_receive_cancel = settings.sleep_in_receive_cancel_ms; +} + + +bool TCPHandler::readDataNext() { Stopwatch watch(CLOCK_MONOTONIC_COARSE); + /// Poll interval should not be greater than receive_timeout + constexpr UInt64 min_timeout_ms = 5000; // 5 ms + UInt64 timeout_ms = std::max(min_timeout_ms, std::min(poll_interval * 1000000, static_cast(receive_timeout.totalMicroseconds()))); + bool read_ok = false; + /// We are waiting for a packet from the client. Thus, every `POLL_INTERVAL` seconds check whether we need to shut down. while (true) { - if (static_cast(*in).poll(poll_interval)) + if (static_cast(*in).poll(timeout_ms)) + { + /// If client disconnected. + if (in->eof()) + { + LOG_INFO(log, "Client has dropped the connection, cancel the query."); + state.is_connection_closed = true; + break; + } + + /// We accept and process data. + read_ok = receivePacket(); break; + } /// Do we need to shut down? if (server.isCancelled()) - return false; + break; /** Have we waited for data for too long? * If we periodically poll, the receive_timeout of the socket itself does not work. * Therefore, an additional check is added. */ Float64 elapsed = watch.elapsedSeconds(); - if (elapsed > static_cast(receive_timeout)) + if (elapsed > static_cast(receive_timeout.totalSeconds())) { throw Exception(ErrorCodes::SOCKET_TIMEOUT, "Timeout exceeded while receiving data from client. Waited for {} seconds, timeout is {} seconds.", - static_cast(elapsed), receive_timeout); + static_cast(elapsed), receive_timeout.totalSeconds()); } } - /// If client disconnected. - if (in->eof()) - { - LOG_INFO(log, "Client has dropped the connection, cancel the query."); - state.is_connection_closed = true; - return false; - } + if (read_ok) + sendLogs(); + else + state.read_all_data = true; - /// We accept and process data. And if they are over, then we leave. - if (!receivePacket()) - return false; - - sendLogs(); - return true; + return read_ok; } -std::tuple TCPHandler::getReadTimeouts(const Settings & connection_settings) +void TCPHandler::readData() { - const auto receive_timeout = query_context->getSettingsRef().receive_timeout.value; - - /// Poll interval should not be greater than receive_timeout - const size_t default_poll_interval = connection_settings.poll_interval * 1000000; - size_t current_poll_interval = static_cast(receive_timeout.totalMicroseconds()); - constexpr size_t min_poll_interval = 5000; // 5 ms - size_t poll_interval = std::max(min_poll_interval, std::min(default_poll_interval, current_poll_interval)); - - return std::make_tuple(poll_interval, receive_timeout.totalSeconds()); -} - - -void TCPHandler::readData(const Settings & connection_settings) -{ - auto [poll_interval, receive_timeout] = getReadTimeouts(connection_settings); sendLogs(); - while (readDataNext(poll_interval, receive_timeout)) + while (readDataNext()) ; } -void TCPHandler::processInsertQuery(const Settings & connection_settings) +void TCPHandler::skipData() +{ + state.skipping_data = true; + SCOPE_EXIT({ state.skipping_data = false; }); + + while (readDataNext()) + ; +} + + +void TCPHandler::processInsertQuery() { /** Made above the rest of the lines, so that in case of `writePrefix` function throws an exception, * client receive exception before sending data. @@ -592,7 +566,7 @@ void TCPHandler::processInsertQuery(const Settings & connection_settings) try { - readData(connection_settings); + readData(); } catch (...) { @@ -631,7 +605,7 @@ void TCPHandler::processOrdinaryQuery() break; } - if (after_send_progress.elapsed() / 1000 >= query_context->getSettingsRef().interactive_delay) + if (after_send_progress.elapsed() / 1000 >= interactive_delay) { /// Some time passed. after_send_progress.restart(); @@ -640,7 +614,7 @@ void TCPHandler::processOrdinaryQuery() sendLogs(); - if (async_in.poll(query_context->getSettingsRef().interactive_delay / 1000)) + if (async_in.poll(interactive_delay / 1000)) { const auto block = async_in.read(); if (!block) @@ -695,7 +669,7 @@ void TCPHandler::processOrdinaryQueryWithProcessors() CurrentMetrics::Increment query_thread_metric_increment{CurrentMetrics::QueryThread}; Block block; - while (executor.pull(block, query_context->getSettingsRef().interactive_delay / 1000)) + while (executor.pull(block, interactive_delay / 1000)) { std::lock_guard lock(task_callback_mutex); @@ -706,7 +680,7 @@ void TCPHandler::processOrdinaryQueryWithProcessors() break; } - if (after_send_progress.elapsed() / 1000 >= query_context->getSettingsRef().interactive_delay) + if (after_send_progress.elapsed() / 1000 >= interactive_delay) { /// Some time passed and there is a progress. after_send_progress.restart(); @@ -753,11 +727,13 @@ void TCPHandler::processTablesStatusRequest() TablesStatusRequest request; request.read(*in, client_tcp_protocol_version); + ContextPtr context_to_resolve_table_names = session->sessionContext() ? session->sessionContext() : server.context(); + TablesStatusResponse response; for (const QualifiedTableName & table_name: request.tables) { - auto resolved_id = connection_context->tryResolveStorageID({table_name.database, table_name.table}); - StoragePtr table = DatabaseCatalog::instance().tryGetTable(resolved_id, connection_context); + auto resolved_id = context_to_resolve_table_names->tryResolveStorageID({table_name.database, table_name.table}); + StoragePtr table = DatabaseCatalog::instance().tryGetTable(resolved_id, context_to_resolve_table_names); if (!table) continue; @@ -777,11 +753,10 @@ void TCPHandler::processTablesStatusRequest() writeVarUInt(Protocol::Server::TablesStatusResponse, *out); /// For testing hedged requests - const Settings & settings = query_context->getSettingsRef(); - if (settings.sleep_in_send_tables_status_ms.totalMilliseconds()) + if (sleep_in_send_tables_status.totalMilliseconds()) { out->next(); - std::chrono::milliseconds ms(settings.sleep_in_send_tables_status_ms.totalMilliseconds()); + std::chrono::milliseconds ms(sleep_in_send_tables_status.totalMilliseconds()); std::this_thread::sleep_for(ms); } @@ -924,11 +899,34 @@ bool TCPHandler::receiveProxyHeader() } LOG_TRACE(log, "Forwarded client address from PROXY header: {}", forwarded_address); - connection_context->getClientInfo().forwarded_for = forwarded_address; + session->getClientInfo().forwarded_for = forwarded_address; return true; } +namespace +{ + +std::string formatHTTPErrorResponseWhenUserIsConnectedToWrongPort(const Poco::Util::AbstractConfiguration& config) +{ + std::string result = fmt::format( + "HTTP/1.0 400 Bad Request\r\n\r\n" + "Port {} is for clickhouse-client program\r\n", + config.getString("tcp_port")); + + if (config.has("http_port")) + { + result += fmt::format( + "You must use port {} for HTTP.\r\n", + config.getString("http_port")); + } + + return result; +} + +} + + void TCPHandler::receiveHello() { /// Receive `hello` packet. @@ -944,9 +942,7 @@ void TCPHandler::receiveHello() */ if (packet_type == 'G' || packet_type == 'P') { - writeString(formatHTTPErrorResponse(server.config()), - *out); - + writeString(formatHTTPErrorResponseWhenUserIsConnectedToWrongPort(server.config()), *out); throw Exception("Client has connected to wrong port", ErrorCodes::CLIENT_HAS_CONNECTED_TO_WRONG_PORT); } else @@ -973,14 +969,21 @@ void TCPHandler::receiveHello() (!user.empty() ? ", user: " + user : "") ); - if (user != USER_INTERSERVER_MARKER) - { - connection_context->setUser(user, password, socket().peerAddress()); - } - else + auto & client_info = session->getClientInfo(); + client_info.client_name = client_name; + client_info.client_version_major = client_version_major; + client_info.client_version_minor = client_version_minor; + client_info.client_version_patch = client_version_patch; + client_info.client_tcp_protocol_version = client_tcp_protocol_version; + + is_interserver_mode = (user == USER_INTERSERVER_MARKER); + if (is_interserver_mode) { receiveClusterNameAndSalt(); + return; } + + session->authenticate(user, password, socket().peerAddress()); } @@ -1027,8 +1030,11 @@ bool TCPHandler::receivePacket() { case Protocol::Client::IgnoredPartUUIDs: /// Part uuids packet if any comes before query. + if (!state.empty() || state.part_uuids_to_ignore) + receiveUnexpectedIgnoredPartUUIDs(); receiveIgnoredPartUUIDs(); return true; + case Protocol::Client::Query: if (!state.empty()) receiveUnexpectedQuery(); @@ -1037,8 +1043,10 @@ bool TCPHandler::receivePacket() case Protocol::Client::Data: case Protocol::Client::Scalar: + if (state.skipping_data) + return receiveUnexpectedData(false); if (state.empty()) - receiveUnexpectedData(); + receiveUnexpectedData(true); return receiveData(packet_type == Protocol::Client::Scalar); case Protocol::Client::Ping: @@ -1049,10 +1057,9 @@ bool TCPHandler::receivePacket() case Protocol::Client::Cancel: { /// For testing connection collector. - const Settings & settings = query_context->getSettingsRef(); - if (settings.sleep_in_receive_cancel_ms.totalMilliseconds()) + if (sleep_in_receive_cancel.totalMilliseconds()) { - std::chrono::milliseconds ms(settings.sleep_in_receive_cancel_ms.totalMilliseconds()); + std::chrono::milliseconds ms(sleep_in_receive_cancel.totalMilliseconds()); std::this_thread::sleep_for(ms); } @@ -1074,14 +1081,18 @@ bool TCPHandler::receivePacket() } } + void TCPHandler::receiveIgnoredPartUUIDs() { - state.part_uuids = true; - std::vector uuids; - readVectorBinary(uuids, *in); + readVectorBinary(state.part_uuids_to_ignore.emplace(), *in); +} - if (!uuids.empty()) - query_context->getIgnoredPartUUIDs()->add(uuids); + +void TCPHandler::receiveUnexpectedIgnoredPartUUIDs() +{ + std::vector skip_part_uuids; + readVectorBinary(skip_part_uuids, *in); + throw NetException("Unexpected packet IgnoredPartUUIDs received from client", ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT); } @@ -1095,10 +1106,9 @@ String TCPHandler::receiveReadTaskResponseAssumeLocked() { state.is_cancelled = true; /// For testing connection collector. - const Settings & settings = query_context->getSettingsRef(); - if (settings.sleep_in_receive_cancel_ms.totalMilliseconds()) + if (sleep_in_receive_cancel.totalMilliseconds()) { - std::chrono::milliseconds ms(settings.sleep_in_receive_cancel_ms.totalMilliseconds()); + std::chrono::milliseconds ms(sleep_in_receive_cancel.totalMilliseconds()); std::this_thread::sleep_for(ms); } return {}; @@ -1129,14 +1139,14 @@ void TCPHandler::receiveClusterNameAndSalt() if (salt.empty()) throw NetException("Empty salt is not allowed", ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT); - cluster_secret = query_context->getCluster(cluster)->getSecret(); + cluster_secret = server.context()->getCluster(cluster)->getSecret(); } catch (const Exception & e) { try { /// We try to send error information to the client. - sendException(e, connection_context->getSettingsRef().calculate_text_stack_trace); + sendException(e, send_exception_with_stack_trace); } catch (...) {} @@ -1152,25 +1162,11 @@ void TCPHandler::receiveQuery() state.is_empty = false; readStringBinary(state.query_id, *in); - /// Client info - ClientInfo & client_info = query_context->getClientInfo(); + /// Read client info. + ClientInfo client_info = session->getClientInfo(); if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_CLIENT_INFO) client_info.read(*in, client_tcp_protocol_version); - /// For better support of old clients, that does not send ClientInfo. - if (client_info.query_kind == ClientInfo::QueryKind::NO_QUERY) - { - client_info.query_kind = ClientInfo::QueryKind::INITIAL_QUERY; - client_info.client_name = client_name; - client_info.client_version_major = client_version_major; - client_info.client_version_minor = client_version_minor; - client_info.client_version_patch = client_version_patch; - client_info.client_tcp_protocol_version = client_tcp_protocol_version; - } - - /// Set fields, that are known apriori. - client_info.interface = ClientInfo::Interface::TCP; - /// Per query settings are also passed via TCP. /// We need to check them before applying due to they can violate the settings constraints. auto settings_format = (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS) @@ -1191,12 +1187,11 @@ void TCPHandler::receiveQuery() readVarUInt(compression, *in); state.compression = static_cast(compression); + last_block_in.compression = state.compression; readStringBinary(state.query, *in); - /// It is OK to check only when query != INITIAL_QUERY, - /// since only in that case the actions will be done. - if (!cluster.empty() && client_info.query_kind != ClientInfo::QueryKind::INITIAL_QUERY) + if (is_interserver_mode) { #if USE_SSL std::string data(salt); @@ -1218,26 +1213,33 @@ void TCPHandler::receiveQuery() /// i.e. when the INSERT is done with the global context (w/o user). if (!client_info.initial_user.empty()) { - query_context->setUserWithoutCheckingPassword(client_info.initial_user, client_info.initial_address); - LOG_DEBUG(log, "User (initial): {}", query_context->getUserName()); + LOG_DEBUG(log, "User (initial): {}", client_info.initial_user); + session->authenticate(AlwaysAllowCredentials{client_info.initial_user}, client_info.initial_address); } - /// No need to update connection_context, since it does not requires user (it will not be used for query execution) #else throw Exception( "Inter-server secret support is disabled, because ClickHouse was built without SSL library", ErrorCodes::SUPPORT_IS_DISABLED); #endif } - else - { - query_context->setInitialRowPolicy(); - } + + query_context = session->makeQueryContext(std::move(client_info)); + + /// Sets the default database if it wasn't set earlier for the session context. + if (!default_database.empty() && !session->sessionContext()) + query_context->setCurrentDatabase(default_database); + + if (state.part_uuids_to_ignore) + query_context->getIgnoredPartUUIDs()->add(*state.part_uuids_to_ignore); + + query_context->setProgressCallback([this] (const Progress & value) { return this->updateProgress(value); }); /// /// Settings /// auto settings_changes = passed_settings.changes(); - if (client_info.query_kind == ClientInfo::QueryKind::INITIAL_QUERY) + auto query_kind = query_context->getClientInfo().query_kind; + if (query_kind == ClientInfo::QueryKind::INITIAL_QUERY) { /// Throw an exception if the passed settings violate the constraints. query_context->checkSettingsConstraints(settings_changes); @@ -1249,40 +1251,24 @@ void TCPHandler::receiveQuery() } query_context->applySettingsChanges(settings_changes); + /// Use the received query id, or generate a random default. It is convenient + /// to also generate the default OpenTelemetry trace id at the same time, and + /// set the trace parent. + /// Notes: + /// 1) ClientInfo might contain upstream trace id, so we decide whether to use + /// the default ids after we have received the ClientInfo. + /// 2) There is the opentelemetry_start_trace_probability setting that + /// controls when we start a new trace. It can be changed via Native protocol, + /// so we have to apply the changes first. + query_context->setCurrentQueryId(state.query_id); + /// Disable function name normalization when it's a secondary query, because queries are either /// already normalized on initiator node, or not normalized and should remain unnormalized for /// compatibility. - if (client_info.query_kind == ClientInfo::QueryKind::SECONDARY_QUERY) + if (query_kind == ClientInfo::QueryKind::SECONDARY_QUERY) { query_context->setSetting("normalize_function_names", Field(0)); } - - // Use the received query id, or generate a random default. It is convenient - // to also generate the default OpenTelemetry trace id at the same time, and - // set the trace parent. - // Why is this done here and not earlier: - // 1) ClientInfo might contain upstream trace id, so we decide whether to use - // the default ids after we have received the ClientInfo. - // 2) There is the opentelemetry_start_trace_probability setting that - // controls when we start a new trace. It can be changed via Native protocol, - // so we have to apply the changes first. - query_context->setCurrentQueryId(state.query_id); - - // Set parameters of initial query. - if (client_info.query_kind == ClientInfo::QueryKind::INITIAL_QUERY) - { - /// 'Current' fields was set at receiveHello. - client_info.initial_user = client_info.current_user; - client_info.initial_query_id = client_info.current_query_id; - client_info.initial_address = client_info.current_address; - } - - /// Sync timeouts on client and server during current query to avoid dangling queries on server - /// NOTE: We use settings.send_timeout for the receive timeout and vice versa (change arguments ordering in TimeoutSetter), - /// because settings.send_timeout is client-side setting which has opposite meaning on the server side. - /// NOTE: these settings are applied only for current connection (not for distributed tables' connections) - const Settings & settings = query_context->getSettingsRef(); - state.timeout_setter = std::make_unique(socket(), settings.receive_timeout, settings.send_timeout); } void TCPHandler::receiveUnexpectedQuery() @@ -1307,7 +1293,10 @@ void TCPHandler::receiveUnexpectedQuery() readStringBinary(skip_hash, *in, 32); readVarUInt(skip_uint_64, *in); + readVarUInt(skip_uint_64, *in); + last_block_in.compression = static_cast(skip_uint_64); + readStringBinary(skip_string, *in); throw NetException("Unexpected packet Query received from client", ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT); @@ -1324,73 +1313,77 @@ bool TCPHandler::receiveData(bool scalar) /// Read one block from the network and write it down Block block = state.block_in->read(); - if (block) + if (!block) { - if (scalar) - { - /// Scalar value - query_context->addScalar(temporary_id.table_name, block); - } - else if (!state.need_receive_data_for_insert && !state.need_receive_data_for_input) - { - /// Data for external tables + state.read_all_data = true; + return false; + } - auto resolved = query_context->tryResolveStorageID(temporary_id, Context::ResolveExternal); - StoragePtr storage; - /// If such a table does not exist, create it. - if (resolved) - { - storage = DatabaseCatalog::instance().getTable(resolved, query_context); - } - else - { - NamesAndTypesList columns = block.getNamesAndTypesList(); - auto temporary_table = TemporaryTableHolder(query_context, ColumnsDescription{columns}, {}); - storage = temporary_table.getTable(); - query_context->addExternalTable(temporary_id.table_name, std::move(temporary_table)); - } - auto metadata_snapshot = storage->getInMemoryMetadataPtr(); - /// The data will be written directly to the table. - auto temporary_table_out = std::make_shared(storage->write(ASTPtr(), metadata_snapshot, query_context)); - temporary_table_out->write(block); - temporary_table_out->writeSuffix(); + if (scalar) + { + /// Scalar value + query_context->addScalar(temporary_id.table_name, block); + } + else if (!state.need_receive_data_for_insert && !state.need_receive_data_for_input) + { + /// Data for external tables - } - else if (state.need_receive_data_for_input) + auto resolved = query_context->tryResolveStorageID(temporary_id, Context::ResolveExternal); + StoragePtr storage; + /// If such a table does not exist, create it. + if (resolved) { - /// 'input' table function. - state.block_for_input = block; + storage = DatabaseCatalog::instance().getTable(resolved, query_context); } else { - /// INSERT query. - state.io.out->write(block); + NamesAndTypesList columns = block.getNamesAndTypesList(); + auto temporary_table = TemporaryTableHolder(query_context, ColumnsDescription{columns}, {}); + storage = temporary_table.getTable(); + query_context->addExternalTable(temporary_id.table_name, std::move(temporary_table)); } - return true; + auto metadata_snapshot = storage->getInMemoryMetadataPtr(); + /// The data will be written directly to the table. + auto temporary_table_out = std::make_shared(storage->write(ASTPtr(), metadata_snapshot, query_context)); + temporary_table_out->write(block); + temporary_table_out->writeSuffix(); + + } + else if (state.need_receive_data_for_input) + { + /// 'input' table function. + state.block_for_input = block; } else - return false; + { + /// INSERT query. + state.io.out->write(block); + } + return true; } -void TCPHandler::receiveUnexpectedData() + +bool TCPHandler::receiveUnexpectedData(bool throw_exception) { String skip_external_table_name; readStringBinary(skip_external_table_name, *in); std::shared_ptr maybe_compressed_in; - if (last_block_in.compression == Protocol::Compression::Enable) maybe_compressed_in = std::make_shared(*in, /* allow_different_codecs */ true); else maybe_compressed_in = in; - auto skip_block_in = std::make_shared( - *maybe_compressed_in, - last_block_in.header, - client_tcp_protocol_version); + auto skip_block_in = std::make_shared(*maybe_compressed_in, client_tcp_protocol_version); + bool read_ok = skip_block_in->read(); - skip_block_in->read(); - throw NetException("Unexpected packet Data received from client", ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT); + if (!read_ok) + state.read_all_data = true; + + if (throw_exception) + throw NetException("Unexpected packet Data received from client", ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT); + + return read_ok; } void TCPHandler::initBlockInput() @@ -1411,9 +1404,6 @@ void TCPHandler::initBlockInput() else if (state.need_receive_data_for_input) header = state.input_header; - last_block_in.header = header; - last_block_in.compression = state.compression; - state.block_in = std::make_shared( *state.maybe_compressed_in, header, @@ -1426,10 +1416,9 @@ void TCPHandler::initBlockOutput(const Block & block) { if (!state.block_out) { + const Settings & query_settings = query_context->getSettingsRef(); if (!state.maybe_compressed_out) { - const Settings & query_settings = query_context->getSettingsRef(); - std::string method = Poco::toUpper(query_settings.network_compression_method.toString()); std::optional level; if (method == "ZSTD") @@ -1450,7 +1439,7 @@ void TCPHandler::initBlockOutput(const Block & block) *state.maybe_compressed_out, client_tcp_protocol_version, block.cloneEmpty(), - !connection_context->getSettingsRef().low_cardinality_allow_in_native_format); + !query_settings.low_cardinality_allow_in_native_format); } } @@ -1459,11 +1448,12 @@ void TCPHandler::initLogsBlockOutput(const Block & block) if (!state.logs_block_out) { /// Use uncompressed stream since log blocks usually contain only one row + const Settings & query_settings = query_context->getSettingsRef(); state.logs_block_out = std::make_shared( *out, client_tcp_protocol_version, block.cloneEmpty(), - !connection_context->getSettingsRef().low_cardinality_allow_in_native_format); + !query_settings.low_cardinality_allow_in_native_format); } } @@ -1473,7 +1463,7 @@ bool TCPHandler::isQueryCancelled() if (state.is_cancelled || state.sent_all_data) return true; - if (after_check_cancelled.elapsed() / 1000 < query_context->getSettingsRef().interactive_delay) + if (after_check_cancelled.elapsed() / 1000 < interactive_delay) return false; after_check_cancelled.restart(); @@ -1501,10 +1491,9 @@ bool TCPHandler::isQueryCancelled() state.is_cancelled = true; /// For testing connection collector. { - const Settings & settings = query_context->getSettingsRef(); - if (settings.sleep_in_receive_cancel_ms.totalMilliseconds()) + if (sleep_in_receive_cancel.totalMilliseconds()) { - std::chrono::milliseconds ms(settings.sleep_in_receive_cancel_ms.totalMilliseconds()); + std::chrono::milliseconds ms(sleep_in_receive_cancel.totalMilliseconds()); std::this_thread::sleep_for(ms); } } @@ -1542,11 +1531,10 @@ void TCPHandler::sendData(const Block & block) writeStringBinary("", *out); /// For testing hedged requests - const Settings & settings = query_context->getSettingsRef(); - if (block.rows() > 0 && settings.sleep_in_send_data_ms.totalMilliseconds()) + if (block.rows() > 0 && query_context->getSettingsRef().sleep_in_send_data_ms.totalMilliseconds()) { out->next(); - std::chrono::milliseconds ms(settings.sleep_in_send_data_ms.totalMilliseconds()); + std::chrono::milliseconds ms(query_context->getSettingsRef().sleep_in_send_data_ms.totalMilliseconds()); std::this_thread::sleep_for(ms); } diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h index 086c1f7d5e5..7f75d0ac04b 100644 --- a/src/Server/TCPHandler.h +++ b/src/Server/TCPHandler.h @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include "IServer.h" @@ -26,7 +26,10 @@ namespace Poco { class Logger; } namespace DB { +class Session; +struct Settings; class ColumnsDescription; +struct BlockStreamProfileInfo; /// State of query processing. struct QueryState @@ -65,11 +68,11 @@ struct QueryState bool sent_all_data = false; /// Request requires data from the client (INSERT, but not INSERT SELECT). bool need_receive_data_for_insert = false; - /// Temporary tables read - bool temporary_tables_read = false; + /// Data was read. + bool read_all_data = false; /// A state got uuids to exclude from a query - bool part_uuids = false; + std::optional> part_uuids_to_ignore; /// Request requires data from client for function input() bool need_receive_data_for_input = false; @@ -78,6 +81,9 @@ struct QueryState /// sample block from StorageInput Block input_header; + /// If true, the data packets will be skipped instead of reading. Used to recover after errors. + bool skipping_data = false; + /// To output progress, the difference after the previous sending of progress. Progress progress; @@ -99,7 +105,6 @@ struct QueryState struct LastBlockInputParameters { Protocol::Compression compression = Protocol::Compression::Disable; - Block header; }; class TCPHandler : public Poco::Net::TCPServerConnection @@ -132,10 +137,19 @@ private: UInt64 client_version_patch = 0; UInt64 client_tcp_protocol_version = 0; - ContextMutablePtr connection_context; - ContextMutablePtr query_context; + /// Connection settings, which are extracted from a context. + bool send_exception_with_stack_trace = true; + Poco::Timespan send_timeout = DBMS_DEFAULT_SEND_TIMEOUT_SEC; + Poco::Timespan receive_timeout = DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC; + UInt64 poll_interval = DBMS_DEFAULT_POLL_INTERVAL; + UInt64 idle_connection_timeout = 3600; + UInt64 interactive_delay = 100000; + Poco::Timespan sleep_in_send_tables_status; + UInt64 unknown_packet_in_send_data = 0; + Poco::Timespan sleep_in_receive_cancel; - size_t unknown_packet_in_send_data = 0; + std::unique_ptr session; + ContextMutablePtr query_context; /// Streams for reading/writing from/to client connection socket. std::shared_ptr in; @@ -148,6 +162,7 @@ private: String default_database; /// For inter-server secret (remote_server.*.secret) + bool is_interserver_mode = false; String salt; String cluster; String cluster_secret; @@ -167,6 +182,8 @@ private: void runImpl(); + void extractConnectionSettingsFromContext(const ContextPtr & context); + bool receiveProxyHeader(); void receiveHello(); bool receivePacket(); @@ -174,18 +191,19 @@ private: void receiveIgnoredPartUUIDs(); String receiveReadTaskResponseAssumeLocked(); bool receiveData(bool scalar); - bool readDataNext(size_t poll_interval, time_t receive_timeout); - void readData(const Settings & connection_settings); + bool readDataNext(); + void readData(); + void skipData(); void receiveClusterNameAndSalt(); - std::tuple getReadTimeouts(const Settings & connection_settings); - [[noreturn]] void receiveUnexpectedData(); + bool receiveUnexpectedData(bool throw_exception = true); [[noreturn]] void receiveUnexpectedQuery(); + [[noreturn]] void receiveUnexpectedIgnoredPartUUIDs(); [[noreturn]] void receiveUnexpectedHello(); [[noreturn]] void receiveUnexpectedTablesStatusRequest(); /// Process INSERT query - void processInsertQuery(const Settings & connection_settings); + void processInsertQuery(); /// Process a request that does not require the receiving of data blocks from the client void processOrdinaryQuery(); diff --git a/src/Storages/CMakeLists.txt b/src/Storages/CMakeLists.txt index ff22e9fa9e1..6d4f18d34b1 100644 --- a/src/Storages/CMakeLists.txt +++ b/src/Storages/CMakeLists.txt @@ -1,6 +1,10 @@ add_subdirectory(MergeTree) add_subdirectory(System) -if(ENABLE_EXAMPLES) +if (ENABLE_EXAMPLES) add_subdirectory(examples) endif() + +if (ENABLE_FUZZING) + add_subdirectory(fuzzers) +endif() diff --git a/src/Storages/Kafka/KafkaBlockInputStream.cpp b/src/Storages/Kafka/KafkaBlockInputStream.cpp index 5d9b19b1972..95fa1459e76 100644 --- a/src/Storages/Kafka/KafkaBlockInputStream.cpp +++ b/src/Storages/Kafka/KafkaBlockInputStream.cpp @@ -252,7 +252,11 @@ Block KafkaBlockInputStream::readImpl() } else { - LOG_WARNING(log, "Parsing of message (topic: {}, partition: {}, offset: {}) return no rows.", buffer->currentTopic(), buffer->currentPartition(), buffer->currentOffset()); + // We came here in case of tombstone (or sometimes zero-length) messages, and it is not something abnormal + // TODO: it seems like in case of put_error_to_stream=true we may need to process those differently + // currently we just skip them with note in logs. + buffer->storeLastReadMessageOffset(); + LOG_DEBUG(log, "Parsing of message (topic: {}, partition: {}, offset: {}) return no rows.", buffer->currentTopic(), buffer->currentPartition(), buffer->currentOffset()); } if (!buffer->hasMorePolledMessages() diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index bd25607a5f3..86037276166 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -466,13 +466,19 @@ bool ReadBufferFromKafkaConsumer::nextImpl() if (!allowed || !hasMorePolledMessages()) return false; - // XXX: very fishy place with const casting. - auto * new_position = reinterpret_cast(const_cast(current->get_payload().get_data())); - BufferBase::set(new_position, current->get_payload().get_size(), 0); - allowed = false; + const auto * message_data = current->get_payload().get_data(); + size_t message_size = current->get_payload().get_size(); + allowed = false; ++current; + /// If message is empty, return end of stream. + if (message_data == nullptr) + return false; + + /// const_cast is needed, because ReadBuffer works with non-const char *. + auto * new_position = reinterpret_cast(const_cast(message_data)); + BufferBase::set(new_position, message_size, 0); return true; } diff --git a/src/Storages/Kafka/WriteBufferToKafkaProducer.cpp b/src/Storages/Kafka/WriteBufferToKafkaProducer.cpp index dbb18b56769..34ab48e501d 100644 --- a/src/Storages/Kafka/WriteBufferToKafkaProducer.cpp +++ b/src/Storages/Kafka/WriteBufferToKafkaProducer.cpp @@ -48,7 +48,7 @@ WriteBufferToKafkaProducer::WriteBufferToKafkaProducer( WriteBufferToKafkaProducer::~WriteBufferToKafkaProducer() { - assert(rows == 0 && chunks.empty()); + assert(rows == 0); } void WriteBufferToKafkaProducer::countRow(const Columns & columns, size_t current_row) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 37309cec5b6..7b07f4aba76 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -51,6 +51,7 @@ #include #include #include +#include #include #include @@ -2395,7 +2396,7 @@ MergeTreeData::DataPartsVector MergeTreeData::removePartsInRangeFromWorkingSet(c /// It's a DROP PART and it's already executed by fetching some covering part bool is_drop_part = !drop_range.isFakeDropRangePart() && drop_range.min_block; - if (is_drop_part && (part->info.min_block != drop_range.min_block || part->info.max_block != drop_range.max_block || part->info.getDataVersion() != drop_range.getDataVersion())) + if (is_drop_part && (part->info.min_block != drop_range.min_block || part->info.max_block != drop_range.max_block || part->info.getMutationVersion() != drop_range.getMutationVersion())) { /// Why we check only min and max blocks here without checking merge /// level? It's a tricky situation which can happen on a stale @@ -2412,7 +2413,7 @@ MergeTreeData::DataPartsVector MergeTreeData::removePartsInRangeFromWorkingSet(c /// So here we just check that all_1_3_1 covers blocks from drop /// all_2_2_2. /// - bool is_covered_by_min_max_block = part->info.min_block <= drop_range.min_block && part->info.max_block >= drop_range.max_block && part->info.getDataVersion() >= drop_range.getDataVersion(); + bool is_covered_by_min_max_block = part->info.min_block <= drop_range.min_block && part->info.max_block >= drop_range.max_block && part->info.getMutationVersion() >= drop_range.getMutationVersion(); if (is_covered_by_min_max_block) { LOG_INFO(log, "Skipping drop range for part {} because covering part {} already exists", drop_range.getPartName(), part->name); @@ -3942,7 +3943,7 @@ static void selectBestProjection( if (projection_parts.empty()) return; - auto sum_marks = reader.estimateNumMarksToRead( + auto projection_result_ptr = reader.estimateNumMarksToRead( projection_parts, candidate.required_columns, metadata_snapshot, @@ -3952,6 +3953,10 @@ static void selectBestProjection( settings.max_threads, max_added_blocks); + if (projection_result_ptr->error()) + return; + + auto sum_marks = projection_result_ptr->marks(); if (normal_parts.empty()) { // All parts are projection parts which allows us to use in_order_optimization. @@ -3960,7 +3965,7 @@ static void selectBestProjection( } else { - sum_marks += reader.estimateNumMarksToRead( + auto normal_result_ptr = reader.estimateNumMarksToRead( normal_parts, required_columns, metadata_snapshot, @@ -3969,7 +3974,14 @@ static void selectBestProjection( query_context, settings.max_threads, max_added_blocks); + + if (normal_result_ptr->error()) + return; + + sum_marks += normal_result_ptr->marks(); + candidate.merge_tree_normal_select_result_ptr = normal_result_ptr; } + candidate.merge_tree_projection_select_result_ptr = projection_result_ptr; // We choose the projection with least sum_marks to read. if (sum_marks < min_sum_marks) @@ -4190,10 +4202,25 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection( auto parts = getDataPartsVector(); MergeTreeDataSelectExecutor reader(*this); + query_info.merge_tree_select_result_ptr = reader.estimateNumMarksToRead( + parts, + analysis_result.required_columns, + metadata_snapshot, + metadata_snapshot, + query_info, + query_context, + settings.max_threads, + max_added_blocks); + + size_t min_sum_marks = std::numeric_limits::max(); + if (!query_info.merge_tree_select_result_ptr->error()) + { + // Add 1 to base sum_marks so that we prefer projections even when they have equal number of marks to read. + // NOTE: It is not clear if we need it. E.g. projections do not support skip index for now. + min_sum_marks = query_info.merge_tree_select_result_ptr->marks() + 1; + } ProjectionCandidate * selected_candidate = nullptr; - size_t min_sum_marks = std::numeric_limits::max(); - bool has_ordinary_projection = false; /// Favor aggregate projections for (auto & candidate : candidates) { @@ -4212,44 +4239,25 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection( selected_candidate, min_sum_marks); } - else - has_ordinary_projection = true; } - /// Select the best normal projection if no aggregate projection is available - if (!selected_candidate && has_ordinary_projection) + /// Select the best normal projection. + for (auto & candidate : candidates) { - min_sum_marks = reader.estimateNumMarksToRead( - parts, - analysis_result.required_columns, - metadata_snapshot, - metadata_snapshot, - query_info, - query_context, - settings.max_threads, - max_added_blocks); - - // Add 1 to base sum_marks so that we prefer projections even when they have equal number of marks to read. - // NOTE: It is not clear if we need it. E.g. projections do not support skip index for now. - min_sum_marks += 1; - - for (auto & candidate : candidates) + if (candidate.desc->type == ProjectionDescription::Type::Normal) { - if (candidate.desc->type == ProjectionDescription::Type::Normal) - { - selectBestProjection( - reader, - metadata_snapshot, - query_info, - analysis_result.required_columns, - candidate, - query_context, - max_added_blocks, - settings, - parts, - selected_candidate, - min_sum_marks); - } + selectBestProjection( + reader, + metadata_snapshot, + query_info, + analysis_result.required_columns, + candidate, + query_context, + max_added_blocks, + settings, + parts, + selected_candidate, + min_sum_marks); } } @@ -4263,7 +4271,6 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection( } query_info.projection = std::move(*selected_candidate); - return true; } return false; diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index c7eb8200957..f5c1890154a 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -133,11 +133,10 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( std::shared_ptr max_block_numbers_to_read) const { const auto & settings = context->getSettingsRef(); - auto parts = data.getDataPartsVector(); if (!query_info.projection) { auto plan = readFromParts( - parts, + query_info.merge_tree_select_result_ptr ? MergeTreeData::DataPartsVector{} : data.getDataPartsVector(), column_names_to_return, metadata_snapshot, metadata_snapshot, @@ -145,7 +144,8 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( context, max_block_size, num_streams, - max_block_numbers_to_read); + max_block_numbers_to_read, + query_info.merge_tree_select_result_ptr); if (plan->isInitialized() && settings.allow_experimental_projection_optimization && settings.force_optimize_projection && !metadata_snapshot->projections.empty()) @@ -162,27 +162,15 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( ProjectionDescription::typeToString(query_info.projection->desc->type), query_info.projection->desc->name); - MergeTreeData::DataPartsVector projection_parts; - MergeTreeData::DataPartsVector normal_parts; - for (const auto & part : parts) - { - const auto & projections = part->getProjectionParts(); - auto it = projections.find(query_info.projection->desc->name); - if (it != projections.end()) - projection_parts.push_back(it->second); - else - normal_parts.push_back(part); - } - Pipes pipes; Pipe projection_pipe; Pipe ordinary_pipe; - if (!projection_parts.empty()) + if (query_info.projection->merge_tree_projection_select_result_ptr) { LOG_DEBUG(log, "projection required columns: {}", fmt::join(query_info.projection->required_columns, ", ")); auto plan = readFromParts( - projection_parts, + {}, query_info.projection->required_columns, metadata_snapshot, query_info.projection->desc->metadata, @@ -190,7 +178,8 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( context, max_block_size, num_streams, - max_block_numbers_to_read); + max_block_numbers_to_read, + query_info.projection->merge_tree_projection_select_result_ptr); if (plan) { @@ -222,9 +211,10 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( } } - if (!normal_parts.empty()) + if (query_info.projection->merge_tree_normal_select_result_ptr) { - auto storage_from_base_parts_of_projection = StorageFromMergeTreeDataPart::create(std::move(normal_parts)); + auto storage_from_base_parts_of_projection + = StorageFromMergeTreeDataPart::create(data, query_info.projection->merge_tree_normal_select_result_ptr); auto interpreter = InterpreterSelectQuery( query_info.query, context, @@ -763,8 +753,7 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd Poco::Logger * log, size_t num_streams, ReadFromMergeTree::IndexStats & index_stats, - bool use_skip_indexes, - bool check_limits) + bool use_skip_indexes) { RangesInDataParts parts_with_ranges(parts.size()); const Settings & settings = context->getSettingsRef(); @@ -892,7 +881,7 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd if (!ranges.ranges.empty()) { - if (check_limits && (limits.max_rows || leaf_limits.max_rows)) + if (limits.max_rows || leaf_limits.max_rows) { /// Fail fast if estimated number of rows to read exceeds the limit auto current_rows_estimate = ranges.getRowsCount(); @@ -1082,7 +1071,7 @@ static void selectColumnNames( } } -size_t MergeTreeDataSelectExecutor::estimateNumMarksToRead( +MergeTreeDataSelectAnalysisResultPtr MergeTreeDataSelectExecutor::estimateNumMarksToRead( MergeTreeData::DataPartsVector parts, const Names & column_names_to_return, const StorageMetadataPtr & metadata_snapshot_base, @@ -1094,7 +1083,8 @@ size_t MergeTreeDataSelectExecutor::estimateNumMarksToRead( { size_t total_parts = parts.size(); if (total_parts == 0) - return 0; + return std::make_shared( + MergeTreeDataSelectAnalysisResult{.result = ReadFromMergeTree::AnalysisResult()}); Names real_column_names; Names virt_column_names; @@ -1104,63 +1094,18 @@ size_t MergeTreeDataSelectExecutor::estimateNumMarksToRead( selectColumnNames(column_names_to_return, data, real_column_names, virt_column_names, sample_factor_column_queried); - auto part_values = filterPartsByVirtualColumns(data, parts, query_info.query, context); - if (part_values && part_values->empty()) - return 0; - - /// If there are only virtual columns in the query, you must request at least one non-virtual one. - if (real_column_names.empty()) - { - NamesAndTypesList available_real_columns = metadata_snapshot->getColumns().getAllPhysical(); - real_column_names.push_back(ExpressionActions::getSmallestColumn(available_real_columns)); - } - - metadata_snapshot->check(real_column_names, data.getVirtuals(), data.getStorageID()); - - const auto & primary_key = metadata_snapshot->getPrimaryKey(); - Names primary_key_columns = primary_key.column_names; - KeyCondition key_condition(query_info, context, primary_key_columns, primary_key.expression); - - if (key_condition.alwaysUnknownOrTrue()) - { - size_t total_marks = 0; - for (const auto & part : parts) - total_marks += part->index_granularity.getMarksCountWithoutFinal(); - - return total_marks; - } - - const auto & select = query_info.query->as(); - ReadFromMergeTree::IndexStats index_stats; - - filterPartsByPartition( - parts, part_values, metadata_snapshot_base, data, query_info, - context, max_block_numbers_to_read.get(), log, index_stats); - - auto sampling = MergeTreeDataSelectExecutor::getSampling( - select, metadata_snapshot->getColumns().getAllPhysical(), parts, key_condition, - data, metadata_snapshot, context, sample_factor_column_queried, log); - - if (sampling.read_nothing) - return 0; - - /// Do not init. It is not used (cause skip index is ignored) - MergeTreeReaderSettings reader_settings; - - auto parts_with_ranges = filterPartsByPrimaryKeyAndSkipIndexes( + return ReadFromMergeTree::selectRangesToRead( std::move(parts), + metadata_snapshot_base, metadata_snapshot, query_info, context, - key_condition, - reader_settings, - log, num_streams, - index_stats, - true /* use_skip_indexes */, - false /* check_limits */); - - return index_stats.back().num_granules_after; + max_block_numbers_to_read, + data, + real_column_names, + sample_factor_column_queried, + log); } QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( @@ -1172,10 +1117,16 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( ContextPtr context, const UInt64 max_block_size, const unsigned num_streams, - std::shared_ptr max_block_numbers_to_read) const + std::shared_ptr max_block_numbers_to_read, + MergeTreeDataSelectAnalysisResultPtr merge_tree_select_result_ptr) const { - size_t total_parts = parts.size(); - if (total_parts == 0) + /// If merge_tree_select_result_ptr != nullptr, we use analyzed result so parts will always be empty. + if (merge_tree_select_result_ptr) + { + if (merge_tree_select_result_ptr->marks() == 0) + return std::make_unique(); + } + else if (parts.empty()) return std::make_unique(); Names real_column_names; @@ -1187,7 +1138,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( selectColumnNames(column_names_to_return, data, real_column_names, virt_column_names, sample_factor_column_queried); auto read_from_merge_tree = std::make_unique( - parts, + std::move(parts), real_column_names, virt_column_names, data, @@ -1199,7 +1150,8 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( num_streams, sample_factor_column_queried, max_block_numbers_to_read, - log + log, + merge_tree_select_result_ptr ); QueryPlanPtr plan = std::make_unique(); diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index de5ca1f0138..92c4382dc90 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -13,15 +13,6 @@ namespace DB class KeyCondition; -struct MergeTreeDataSelectSamplingData -{ - bool use_sampling = false; - bool read_nothing = false; - Float64 used_sample_factor = 1.0; - std::shared_ptr filter_function; - ActionsDAGPtr filter_expression; -}; - using PartitionIdToMaxBlock = std::unordered_map; /** Executes SELECT queries on data from the merge tree. @@ -55,12 +46,13 @@ public: ContextPtr context, UInt64 max_block_size, unsigned num_streams, - std::shared_ptr max_block_numbers_to_read = nullptr) const; + std::shared_ptr max_block_numbers_to_read = nullptr, + MergeTreeDataSelectAnalysisResultPtr merge_tree_select_result_ptr = nullptr) const; /// Get an estimation for the number of marks we are going to read. /// Reads nothing. Secondary indexes are not used. /// This method is used to select best projection for table. - size_t estimateNumMarksToRead( + MergeTreeDataSelectAnalysisResultPtr estimateNumMarksToRead( MergeTreeData::DataPartsVector parts, const Names & column_names, const StorageMetadataPtr & metadata_snapshot_base, @@ -185,8 +177,7 @@ public: Poco::Logger * log, size_t num_streams, ReadFromMergeTree::IndexStats & index_stats, - bool use_skip_indexes, - bool check_limits); + bool use_skip_indexes); /// Create expression for sampling. /// Also, calculate _sample_factor if needed. diff --git a/src/Storages/MergeTree/MergeTreePartInfo.h b/src/Storages/MergeTree/MergeTreePartInfo.h index be856c1f157..181fef7990c 100644 --- a/src/Storages/MergeTree/MergeTreePartInfo.h +++ b/src/Storages/MergeTree/MergeTreePartInfo.h @@ -65,6 +65,12 @@ struct MergeTreePartInfo && mutation >= rhs.mutation; } + /// Return part mutation version, if part wasn't mutated return zero + Int64 getMutationVersion() const + { + return mutation ? mutation : 0; + } + /// True if parts do not intersect in any way. bool isDisjoint(const MergeTreePartInfo & rhs) const { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index 35a011a4a58..797d0570fbc 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -145,6 +145,7 @@ ReplicatedMergeTreePartCheckThread::MissingPartSearchResult ReplicatedMergeTreeP if (found_part_with_the_same_min_block && found_part_with_the_same_max_block) { + /// FIXME It may never appear LOG_WARNING(log, "Found parts with the same min block and with the same max block as the missing part {}. Hoping that it will eventually appear as a result of a merge.", part_name); return MissingPartSearchResult::FoundAndDontNeedFetch; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index ef276a53df2..c71a79d2009 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -23,6 +23,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int UNEXPECTED_NODE_IN_ZOOKEEPER; extern const int ABORTED; + extern const int READONLY; } @@ -472,9 +473,15 @@ bool ReplicatedMergeTreeQueue::removeFailedQuorumPart(const MergeTreePartInfo & return virtual_parts.remove(part_info); } -int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, Coordination::WatchCallback watch_callback) +int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, Coordination::WatchCallback watch_callback, PullLogsReason reason) { std::lock_guard lock(pull_logs_to_queue_mutex); + if (storage.is_readonly && reason == SYNC) + { + throw Exception(ErrorCodes::READONLY, "Cannot SYNC REPLICA, because replica is readonly"); + /// TODO throw logical error for other reasons (except LOAD) + } + if (pull_log_blocker.isCancelled()) throw Exception("Log pulling is cancelled", ErrorCodes::ABORTED); @@ -714,13 +721,22 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C std::vector> futures; for (const String & entry : entries_to_load) - futures.emplace_back(zookeeper->asyncGet(fs::path(zookeeper_path) / "mutations" / entry)); + futures.emplace_back(zookeeper->asyncTryGet(fs::path(zookeeper_path) / "mutations" / entry)); std::vector new_mutations; for (size_t i = 0; i < entries_to_load.size(); ++i) { + auto maybe_response = futures[i].get(); + if (maybe_response.error != Coordination::Error::ZOK) + { + assert(maybe_response.error == Coordination::Error::ZNONODE); + /// It's ok if it happened on server startup or table creation and replica loads all mutation entries. + /// It's also ok if mutation was killed. + LOG_WARNING(log, "Cannot get mutation node {} ({}), probably it was concurrently removed", entries_to_load[i], maybe_response.error); + continue; + } new_mutations.push_back(std::make_shared( - ReplicatedMergeTreeMutationEntry::parse(futures[i].get().data, entries_to_load[i]))); + ReplicatedMergeTreeMutationEntry::parse(maybe_response.data, entries_to_load[i]))); } bool some_mutations_are_probably_done = false; @@ -1504,6 +1520,9 @@ MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( /// to allow recovering from a mutation that cannot be executed. This way you can delete the mutation entry /// from /mutations in ZK and the replicas will simply skip the mutation. + /// NOTE: However, it's quite dangerous to skip MUTATE_PART. Replicas may diverge if one of them have executed part mutation, + /// and then mutation was killed before execution of MUTATE_PART on remaining replicas. + if (part->info.getDataVersion() > desired_mutation_version) { LOG_WARNING(log, "Data version of part {} is already greater than desired mutation version {}", part->name, desired_mutation_version); @@ -1831,7 +1850,7 @@ ReplicatedMergeTreeMergePredicate::ReplicatedMergeTreeMergePredicate( } } - merges_version = queue_.pullLogsToQueue(zookeeper); + merges_version = queue_.pullLogsToQueue(zookeeper, {}, ReplicatedMergeTreeQueue::MERGE_PREDICATE); { /// We avoid returning here a version to be used in a lightweight transaction. diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index e49d80fc832..57e1e658665 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -294,13 +294,22 @@ public: bool removeFailedQuorumPart(const MergeTreePartInfo & part_info); + enum PullLogsReason + { + LOAD, + UPDATE, + MERGE_PREDICATE, + SYNC, + OTHER, + }; + /** Copy the new entries from the shared log to the queue of this replica. Set the log_pointer to the appropriate value. * If watch_callback is not empty, will call it when new entries appear in the log. * If there were new entries, notifies storage.queue_task_handle. * Additionally loads mutations (so that the set of mutations is always more recent than the queue). * Return the version of "logs" node (that is updated for every merge/mutation/... added to the log) */ - int32_t pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, Coordination::WatchCallback watch_callback = {}); + int32_t pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, Coordination::WatchCallback watch_callback = {}, PullLogsReason reason = OTHER); /// Load new mutation entries. If something new is loaded, schedule storage.merge_selecting_task. /// If watch_callback is not empty, will call it when new mutations appear in ZK. diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 25f25480549..5c19f4166fa 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -25,6 +25,8 @@ namespace DB namespace ErrorCodes { extern const int REPLICA_IS_ALREADY_ACTIVE; + extern const int REPLICA_STATUS_CHANGED; + } namespace @@ -55,6 +57,7 @@ void ReplicatedMergeTreeRestartingThread::run() if (need_stop) return; + bool reschedule_now = false; try { if (first_time || readonly_mode_was_set || storage.getZooKeeper()->expired()) @@ -131,15 +134,29 @@ void ReplicatedMergeTreeRestartingThread::run() first_time = false; } } - catch (...) + catch (const Exception & e) { /// We couldn't activate table let's set it into readonly mode setReadonly(); + partialShutdown(); + storage.startup_event.set(); + tryLogCurrentException(log, __PRETTY_FUNCTION__); + + if (e.code() == ErrorCodes::REPLICA_STATUS_CHANGED) + reschedule_now = true; + } + catch (...) + { + setReadonly(); + partialShutdown(); storage.startup_event.set(); tryLogCurrentException(log, __PRETTY_FUNCTION__); } - task->scheduleAfter(check_period_ms); + if (reschedule_now) + task->schedule(); + else + task->scheduleAfter(check_period_ms); } @@ -155,11 +172,21 @@ bool ReplicatedMergeTreeRestartingThread::tryStartup() storage.cloneReplicaIfNeeded(zookeeper); - storage.queue.load(zookeeper); + try + { + storage.queue.load(zookeeper); + + /// pullLogsToQueue() after we mark replica 'is_active' (and after we repair if it was lost); + /// because cleanup_thread doesn't delete log_pointer of active replicas. + storage.queue.pullLogsToQueue(zookeeper, {}, ReplicatedMergeTreeQueue::LOAD); + } + catch (...) + { + std::unique_lock lock(storage.last_queue_update_exception_lock); + storage.last_queue_update_exception = getCurrentExceptionMessage(false); + throw; + } - /// pullLogsToQueue() after we mark replica 'is_active' (and after we repair if it was lost); - /// because cleanup_thread doesn't delete log_pointer of active replicas. - storage.queue.pullLogsToQueue(zookeeper); storage.queue.removeCurrentPartsFromMutations(); storage.last_queue_update_finish_time.store(time(nullptr)); diff --git a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 15beb94404b..bcce2d990ca 100644 --- a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -31,8 +31,7 @@ public: size_t max_block_size, unsigned num_streams) override { - // NOTE: It's used to read normal parts only - QueryPlan query_plan = std::move(*MergeTreeDataSelectExecutor(parts.front()->storage) + QueryPlan query_plan = std::move(*MergeTreeDataSelectExecutor(storage) .readFromParts( parts, column_names, @@ -41,7 +40,9 @@ public: query_info, context, max_block_size, - num_streams)); + num_streams, + nullptr, + analysis_result_ptr)); return query_plan.convertToPipe( QueryPlanOptimizationSettings::fromContext(context), BuildQueryPipelineSettings::fromContext(context)); @@ -54,54 +55,53 @@ public: bool mayBenefitFromIndexForIn( const ASTPtr & left_in_operand, ContextPtr query_context, const StorageMetadataPtr & metadata_snapshot) const override { - return parts.front()->storage.mayBenefitFromIndexForIn(left_in_operand, query_context, metadata_snapshot); + return storage.mayBenefitFromIndexForIn(left_in_operand, query_context, metadata_snapshot); } NamesAndTypesList getVirtuals() const override { - return parts.front()->storage.getVirtuals(); + return storage.getVirtuals(); } String getPartitionId() const { - return parts.front()->info.partition_id; + return partition_id; } String getPartitionIDFromQuery(const ASTPtr & ast, ContextPtr context) const { - return parts.front()->storage.getPartitionIDFromQuery(ast, context); + return storage.getPartitionIDFromQuery(ast, context); } protected: + /// Used in part mutation. StorageFromMergeTreeDataPart(const MergeTreeData::DataPartPtr & part_) : IStorage(getIDFromPart(part_)) , parts({part_}) + , storage(part_->storage) + , partition_id(part_->info.partition_id) { - setInMemoryMetadata(part_->storage.getInMemoryMetadata()); + setInMemoryMetadata(storage.getInMemoryMetadata()); } - StorageFromMergeTreeDataPart(MergeTreeData::DataPartsVector && parts_) - : IStorage(getIDFromParts(parts_)) - , parts(std::move(parts_)) + /// Used in queries with projection. + StorageFromMergeTreeDataPart(const MergeTreeData & storage_, MergeTreeDataSelectAnalysisResultPtr analysis_result_ptr_) + : IStorage(storage_.getStorageID()), storage(storage_), analysis_result_ptr(analysis_result_ptr_) { - setInMemoryMetadata(parts.front()->storage.getInMemoryMetadata()); + setInMemoryMetadata(storage.getInMemoryMetadata()); } private: MergeTreeData::DataPartsVector parts; + const MergeTreeData & storage; + String partition_id; + MergeTreeDataSelectAnalysisResultPtr analysis_result_ptr; static StorageID getIDFromPart(const MergeTreeData::DataPartPtr & part_) { auto table_id = part_->storage.getStorageID(); return StorageID(table_id.database_name, table_id.table_name + " (part " + part_->name + ")"); } - - static StorageID getIDFromParts(const MergeTreeData::DataPartsVector & parts_) - { - assert(!parts_.empty()); - auto table_id = parts_.front()->storage.getStorageID(); - return StorageID(table_id.database_name, table_id.table_name + " (parts)"); - } }; } diff --git a/src/Storages/RabbitMQ/WriteBufferToRabbitMQProducer.cpp b/src/Storages/RabbitMQ/WriteBufferToRabbitMQProducer.cpp index b9af60eb66f..be7f1fe508a 100644 --- a/src/Storages/RabbitMQ/WriteBufferToRabbitMQProducer.cpp +++ b/src/Storages/RabbitMQ/WriteBufferToRabbitMQProducer.cpp @@ -101,7 +101,7 @@ WriteBufferToRabbitMQProducer::~WriteBufferToRabbitMQProducer() std::this_thread::sleep_for(std::chrono::milliseconds(CONNECT_SLEEP)); } - assert(rows == 0 && chunks.empty()); + assert(rows == 0); } diff --git a/src/Storages/ReadInOrderOptimizer.cpp b/src/Storages/ReadInOrderOptimizer.cpp index 912d284bfc0..bae24f97b28 100644 --- a/src/Storages/ReadInOrderOptimizer.cpp +++ b/src/Storages/ReadInOrderOptimizer.cpp @@ -6,8 +6,6 @@ #include #include #include -#include -#include namespace DB { diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index 3b3c0fa1258..a4536e1ff58 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -39,6 +39,9 @@ using ReadInOrderOptimizerPtr = std::shared_ptr; class Cluster; using ClusterPtr = std::shared_ptr; +struct MergeTreeDataSelectAnalysisResult; +using MergeTreeDataSelectAnalysisResultPtr = std::shared_ptr; + struct PrewhereInfo { /// Actions which are executed in order to alias columns are used for prewhere actions. @@ -118,6 +121,8 @@ struct ProjectionCandidate ReadInOrderOptimizerPtr order_optimizer; InputOrderInfoPtr input_order_info; ManyExpressionActions group_by_elements_actions; + MergeTreeDataSelectAnalysisResultPtr merge_tree_projection_select_result_ptr; + MergeTreeDataSelectAnalysisResultPtr merge_tree_normal_select_result_ptr; }; /** Query along with some additional data, @@ -158,6 +163,7 @@ struct SelectQueryInfo std::optional projection; bool ignore_projections = false; bool is_projection_query = false; + MergeTreeDataSelectAnalysisResultPtr merge_tree_select_result_ptr; }; } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index fcd0e255e5c..df7d568deb9 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -327,11 +327,13 @@ StorageDistributed::StorageDistributed( const String & relative_data_path_, const DistributedSettings & distributed_settings_, bool attach_, - ClusterPtr owned_cluster_) + ClusterPtr owned_cluster_, + ASTPtr remote_table_function_ptr_) : IStorage(id_) , WithContext(context_->getGlobalContext()) , remote_database(remote_database_) , remote_table(remote_table_) + , remote_table_function_ptr(remote_table_function_ptr_) , log(&Poco::Logger::get("StorageDistributed (" + id_.table_name + ")")) , owned_cluster(std::move(owned_cluster_)) , cluster_name(getContext()->getMacros()->expand(cluster_name_)) @@ -363,10 +365,13 @@ StorageDistributed::StorageDistributed( } /// Sanity check. Skip check if the table is already created to allow the server to start. - if (!attach_ && !cluster_name.empty()) + if (!attach_) { - size_t num_local_shards = getContext()->getCluster(cluster_name)->getLocalShardCount(); - if (num_local_shards && remote_database == id_.database_name && remote_table == id_.table_name) + if (remote_database.empty() && !remote_table_function_ptr && !getCluster()->maybeCrossReplication()) + LOG_WARNING(log, "Name of remote database is empty. Default database will be used implicitly."); + + size_t num_local_shards = getCluster()->getLocalShardCount(); + if (num_local_shards && (remote_database.empty() || remote_database == id_.database_name) && remote_table == id_.table_name) throw Exception("Distributed table " + id_.table_name + " looks at itself", ErrorCodes::INFINITE_LOOP); } } @@ -399,9 +404,9 @@ StorageDistributed::StorageDistributed( relative_data_path_, distributed_settings_, attach, - std::move(owned_cluster_)) + std::move(owned_cluster_), + remote_table_function_ptr_) { - remote_table_function_ptr = std::move(remote_table_function_ptr_); } QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage( @@ -810,9 +815,6 @@ void StorageDistributed::alter(const AlterCommands & params, ContextPtr local_co void StorageDistributed::startup() { - if (remote_database.empty() && !remote_table_function_ptr && !getCluster()->maybeCrossReplication()) - LOG_WARNING(log, "Name of remote database is empty. Default database will be used implicitly."); - if (!storage_policy) return; diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index f8b16dec7be..b003f8c6486 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -136,7 +136,8 @@ private: const String & relative_data_path_, const DistributedSettings & distributed_settings_, bool attach_, - ClusterPtr owned_cluster_ = {}); + ClusterPtr owned_cluster_ = {}, + ASTPtr remote_table_function_ptr_ = {}); StorageDistributed( const StorageID & id_, @@ -174,8 +175,9 @@ private: /// - optimize_skip_unused_shards /// - force_optimize_skip_unused_shards ClusterPtr getOptimizedCluster(ContextPtr, const StorageMetadataPtr & metadata_snapshot, const ASTPtr & query_ptr) const; - ClusterPtr - skipUnusedShards(ClusterPtr cluster, const ASTPtr & query_ptr, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) const; + + ClusterPtr skipUnusedShards( + ClusterPtr cluster, const ASTPtr & query_ptr, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) const; /// This method returns optimal query processing stage. /// diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 243294351f3..ed203db1a35 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -435,11 +435,17 @@ Pipe StorageMerge::createSources( if (!pipe.empty()) { if (concat_streams && pipe.numOutputPorts() > 1) + { // It's possible to have many tables read from merge, resize(1) might open too many files at the same time. // Using concat instead. pipe.addTransform(std::make_shared(pipe.getHeader(), pipe.numOutputPorts())); + } - if (has_database_virtual_column) + /// Add virtual columns if we don't already have them. + + Block pipe_header = pipe.getHeader(); + + if (has_database_virtual_column && !pipe_header.has("_database")) { ColumnWithTypeAndName column; column.name = "_database"; @@ -457,7 +463,7 @@ Pipe StorageMerge::createSources( }); } - if (has_table_virtual_column) + if (has_table_virtual_column && !pipe_header.has("_table")) { ColumnWithTypeAndName column; column.name = "_table"; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 150a71a09e5..3bdba5d1b35 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -141,6 +141,7 @@ namespace ErrorCodes extern const int DUPLICATE_DATA_PART; extern const int BAD_ARGUMENTS; extern const int CONCURRENT_ACCESS_NOT_SUPPORTED; + extern const int CHECKSUM_DOESNT_MATCH; } namespace ActionLocks @@ -1314,32 +1315,35 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: } ReplicatedMergeTreePartHeader replica_part_header; - if (!part_zk_str.empty()) - replica_part_header = ReplicatedMergeTreePartHeader::fromString(part_zk_str); - else + if (part_zk_str.empty()) { - Coordination::Stat columns_stat_before, columns_stat_after; String columns_str; String checksums_str; - /// Let's check that the node's version with the columns did not change while we were reading the checksums. - /// This ensures that the columns and the checksum refer to the same - if (!zookeeper->tryGet(fs::path(current_part_path) / "columns", columns_str, &columns_stat_before) || - !zookeeper->tryGet(fs::path(current_part_path) / "checksums", checksums_str) || - !zookeeper->exists(fs::path(current_part_path) / "columns", &columns_stat_after) || - columns_stat_before.version != columns_stat_after.version) + if (zookeeper->tryGet(fs::path(current_part_path) / "columns", columns_str) && + zookeeper->tryGet(fs::path(current_part_path) / "checksums", checksums_str)) { - LOG_INFO(log, "Not checking checksums of part {} with replica {} because part changed while we were reading its checksums", part_name, replica); + replica_part_header = ReplicatedMergeTreePartHeader::fromColumnsAndChecksumsZNodes(columns_str, checksums_str); + } + else + { + if (zookeeper->exists(current_part_path)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Part {} has empty header and does not have columns and checksums. " + "Looks like a bug.", current_part_path); + LOG_INFO(log, "Not checking checksums of part {} with replica {} because part was removed from ZooKeeper", part_name, replica); continue; } - - replica_part_header = ReplicatedMergeTreePartHeader::fromColumnsAndChecksumsZNodes( - columns_str, checksums_str); + } + else + { + replica_part_header = ReplicatedMergeTreePartHeader::fromString(part_zk_str); } if (replica_part_header.getColumnsHash() != local_part_header.getColumnsHash()) { - LOG_INFO(log, "Not checking checksums of part {} with replica {} because columns are different", part_name, replica); - continue; + /// Either it's a bug or ZooKeeper contains broken data. + /// TODO Fix KILL MUTATION and replace CHECKSUM_DOESNT_MATCH with LOGICAL_ERROR + /// (some replicas may skip killed mutation even if it was executed on other replicas) + throw Exception(ErrorCodes::CHECKSUM_DOESNT_MATCH, "Part {} from {} has different columns hash", part_name, replica); } replica_part_header.getChecksums().checkEqual(local_part_header.getChecksums(), true); @@ -2137,6 +2141,8 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) if (!parts_for_merge.empty() && replica.empty()) { LOG_INFO(log, "No active replica has part {}. Will fetch merged part instead.", entry.new_part_name); + /// We should enqueue it for check, because merged part may never appear if source part is lost + enqueuePartForCheck(entry.new_part_name); return false; } @@ -3073,6 +3079,12 @@ void StorageReplicatedMergeTree::cloneReplicaIfNeeded(zkutil::ZooKeeperPtr zooke zookeeper->set(fs::path(replica_path) / "is_lost", "0"); } +String StorageReplicatedMergeTree::getLastQueueUpdateException() const +{ + std::unique_lock lock(last_queue_update_exception_lock); + return last_queue_update_exception; +} + void StorageReplicatedMergeTree::queueUpdatingTask() { @@ -3083,7 +3095,7 @@ void StorageReplicatedMergeTree::queueUpdatingTask() } try { - queue.pullLogsToQueue(getZooKeeper(), queue_updating_task->getWatchCallback()); + queue.pullLogsToQueue(getZooKeeper(), queue_updating_task->getWatchCallback(), ReplicatedMergeTreeQueue::UPDATE); last_queue_update_finish_time.store(time(nullptr)); queue_update_in_progress = false; } @@ -3091,6 +3103,9 @@ void StorageReplicatedMergeTree::queueUpdatingTask() { tryLogCurrentException(log, __PRETTY_FUNCTION__); + std::unique_lock lock(last_queue_update_exception_lock); + last_queue_update_exception = getCurrentExceptionMessage(false); + if (e.code == Coordination::Error::ZSESSIONEXPIRED) { restarting_thread.wakeup(); @@ -3102,6 +3117,10 @@ void StorageReplicatedMergeTree::queueUpdatingTask() catch (...) { tryLogCurrentException(log, __PRETTY_FUNCTION__); + + std::unique_lock lock(last_queue_update_exception_lock); + last_queue_update_exception = getCurrentExceptionMessage(false); + queue_updating_task->scheduleAfter(QUEUE_UPDATE_ERROR_SLEEP_MS); } } @@ -4321,9 +4340,6 @@ void StorageReplicatedMergeTree::startup() /// Wait while restarting_thread initializes LeaderElection (and so on) or makes first attempt to do it startup_event.wait(); - /// If we don't separate create/start steps, race condition will happen - /// between the assignment of queue_task_handle and queueTask that use the queue_task_handle. - background_executor.start(); startBackgroundMovesIfNeeded(); part_moves_between_shards_orchestrator.start(); @@ -5460,9 +5476,9 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( const auto & stop_waiting = [&]() { - bool stop_waiting_itself = waiting_itself && (partial_shutdown_called || is_dropped); + bool stop_waiting_itself = waiting_itself && partial_shutdown_called; bool stop_waiting_non_active = !wait_for_non_active && !getZooKeeper()->exists(fs::path(table_zookeeper_path) / "replicas" / replica / "is_active"); - return stop_waiting_itself || stop_waiting_non_active; + return is_dropped || stop_waiting_itself || stop_waiting_non_active; }; /// Don't recheck ZooKeeper too often @@ -5562,6 +5578,7 @@ void StorageReplicatedMergeTree::getStatus(Status & res, bool with_zk_fields) res.log_pointer = 0; res.total_replicas = 0; res.active_replicas = 0; + res.last_queue_update_exception = getLastQueueUpdateException(); if (with_zk_fields && !res.is_session_expired) { @@ -6058,7 +6075,7 @@ CancellationCode StorageReplicatedMergeTree::killMutation(const String & mutatio zkutil::ZooKeeperPtr zookeeper = getZooKeeper(); - LOG_TRACE(log, "Killing mutation {}", mutation_id); + LOG_INFO(log, "Killing mutation {}", mutation_id); auto mutation_entry = queue.removeMutation(zookeeper, mutation_id); if (!mutation_entry) @@ -6964,7 +6981,7 @@ bool StorageReplicatedMergeTree::waitForShrinkingQueueSize(size_t queue_size, UI Stopwatch watch; /// Let's fetch new log entries firstly - queue.pullLogsToQueue(getZooKeeper()); + queue.pullLogsToQueue(getZooKeeper(), {}, ReplicatedMergeTreeQueue::SYNC); /// This is significant, because the execution of this task could be delayed at BackgroundPool. /// And we force it to be executed. diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 3d2727d7bb9..2fd10f88895 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -174,6 +174,7 @@ public: UInt64 absolute_delay; UInt8 total_replicas; UInt8 active_replicas; + String last_queue_update_exception; /// If the error has happened fetching the info from ZooKeeper, this field will be set. String zookeeper_exception; std::unordered_map replica_is_active; @@ -331,6 +332,10 @@ private: std::atomic last_queue_update_start_time{0}; std::atomic last_queue_update_finish_time{0}; + mutable std::mutex last_queue_update_exception_lock; + String last_queue_update_exception; + String getLastQueueUpdateException() const; + DataPartsExchange::Fetcher fetcher; /// When activated, replica is initialized and startup() method could exit diff --git a/src/Storages/StorageSet.cpp b/src/Storages/StorageSet.cpp index 67fd89f5098..c16b60af45f 100644 --- a/src/Storages/StorageSet.cpp +++ b/src/Storages/StorageSet.cpp @@ -146,13 +146,13 @@ StorageSet::StorageSet( Block header = getInMemoryMetadataPtr()->getSampleBlock(); header = header.sortColumns(); - set->setHeader(header); + set->setHeader(header.getColumnsWithTypeAndName()); restore(); } -void StorageSet::insertBlock(const Block & block) { set->insertFromBlock(block); } +void StorageSet::insertBlock(const Block & block) { set->insertFromBlock(block.getColumnsWithTypeAndName()); } void StorageSet::finishInsert() { set->finishInsert(); } size_t StorageSet::getSize() const { return set->getTotalRowCount(); } @@ -170,7 +170,7 @@ void StorageSet::truncate(const ASTPtr &, const StorageMetadataPtr & metadata_sn increment = 0; set = std::make_shared(SizeLimits(), false, true); - set->setHeader(header); + set->setHeader(header.getColumnsWithTypeAndName()); } diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp index 5c22d3c2fae..467226c3b7a 100644 --- a/src/Storages/System/StorageSystemReplicas.cpp +++ b/src/Storages/System/StorageSystemReplicas.cpp @@ -51,6 +51,7 @@ StorageSystemReplicas::StorageSystemReplicas(const StorageID & table_id_) { "absolute_delay", std::make_shared() }, { "total_replicas", std::make_shared() }, { "active_replicas", std::make_shared() }, + { "last_queue_update_exception", std::make_shared() }, { "zookeeper_exception", std::make_shared() }, { "replica_is_active", std::make_shared(std::make_shared(), std::make_shared()) } })); @@ -186,6 +187,7 @@ Pipe StorageSystemReplicas::read( res_columns[col_num++]->insert(status.absolute_delay); res_columns[col_num++]->insert(status.total_replicas); res_columns[col_num++]->insert(status.active_replicas); + res_columns[col_num++]->insert(status.last_queue_update_exception); res_columns[col_num++]->insert(status.zookeeper_exception); Map replica_is_active_values; diff --git a/src/Storages/System/StorageSystemZooKeeper.cpp b/src/Storages/System/StorageSystemZooKeeper.cpp index d19aef47616..cba10548852 100644 --- a/src/Storages/System/StorageSystemZooKeeper.cpp +++ b/src/Storages/System/StorageSystemZooKeeper.cpp @@ -97,12 +97,12 @@ static bool extractPathImpl(const IAST & elem, Paths & res, ContextPtr context) auto stream = interpreter_subquery->execute().getInputStream(); SizeLimits limites(context->getSettingsRef().max_rows_in_set, context->getSettingsRef().max_bytes_in_set, OverflowMode::THROW); Set set(limites, true, context->getSettingsRef().transform_null_in); - set.setHeader(stream->getHeader()); + set.setHeader(stream->getHeader().getColumnsWithTypeAndName()); stream->readPrefix(); while (Block block = stream->read()) { - set.insertFromBlock(block); + set.insertFromBlock(block.getColumnsWithTypeAndName()); } set.finishInsert(); stream->readSuffix(); diff --git a/src/Storages/examples/CMakeLists.txt b/src/Storages/examples/CMakeLists.txt index 59d44829363..cb03ae751e3 100644 --- a/src/Storages/examples/CMakeLists.txt +++ b/src/Storages/examples/CMakeLists.txt @@ -23,10 +23,3 @@ target_link_libraries (transform_part_zk_nodes string_utils ) -if (ENABLE_FUZZING) - add_executable (mergetree_checksum_fuzzer mergetree_checksum_fuzzer.cpp) - target_link_libraries (mergetree_checksum_fuzzer PRIVATE dbms ${LIB_FUZZING_ENGINE}) - - add_executable (columns_description_fuzzer columns_description_fuzzer.cpp) - target_link_libraries (columns_description_fuzzer PRIVATE dbms ${LIB_FUZZING_ENGINE}) -endif () diff --git a/src/Storages/fuzzers/CMakeLists.txt b/src/Storages/fuzzers/CMakeLists.txt new file mode 100644 index 00000000000..93d3d2926bd --- /dev/null +++ b/src/Storages/fuzzers/CMakeLists.txt @@ -0,0 +1,11 @@ + +add_executable (mergetree_checksum_fuzzer + mergetree_checksum_fuzzer.cpp + "${ClickHouse_SOURCE_DIR}/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp" + "${ClickHouse_SOURCE_DIR}/src/Compression/CompressedReadBuffer.cpp" + "${ClickHouse_SOURCE_DIR}/src/Compression/CompressedWriteBuffer.cpp" +) +target_link_libraries (mergetree_checksum_fuzzer PRIVATE clickhouse_common_io fuzz_compression ${LIB_FUZZING_ENGINE}) + +add_executable (columns_description_fuzzer columns_description_fuzzer.cpp) +target_link_libraries (columns_description_fuzzer PRIVATE dbms ${LIB_FUZZING_ENGINE}) diff --git a/src/Storages/examples/columns_description_fuzzer.cpp b/src/Storages/fuzzers/columns_description_fuzzer.cpp similarity index 100% rename from src/Storages/examples/columns_description_fuzzer.cpp rename to src/Storages/fuzzers/columns_description_fuzzer.cpp diff --git a/src/Storages/examples/mergetree_checksum_fuzzer.cpp b/src/Storages/fuzzers/mergetree_checksum_fuzzer.cpp similarity index 100% rename from src/Storages/examples/mergetree_checksum_fuzzer.cpp rename to src/Storages/fuzzers/mergetree_checksum_fuzzer.cpp diff --git a/src/Storages/tests/gtest_storage_log.cpp b/src/Storages/tests/gtest_storage_log.cpp index 16902eafc98..b3ceef7e697 100644 --- a/src/Storages/tests/gtest_storage_log.cpp +++ b/src/Storages/tests/gtest_storage_log.cpp @@ -128,6 +128,7 @@ std::string readData(DB::StoragePtr & table, const DB::ContextPtr context) { ColumnWithTypeAndName col; col.type = std::make_shared(); + col.name = "a"; sample.insert(std::move(col)); } diff --git a/tests/clickhouse-test b/tests/clickhouse-test index c627810a550..0d833e5fbe6 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -44,15 +44,17 @@ DISTRIBUTED_DDL_TIMEOUT_MSG = "is executing longer than distributed_ddl_task_tim MESSAGES_TO_RETRY = [ "DB::Exception: ZooKeeper session has been expired", - "DB::Exception: Connection loss", "Coordination::Exception: Session expired", "Coordination::Exception: Connection loss", "Coordination::Exception: Operation timeout", + "DB::Exception: Session expired", + "DB::Exception: Connection loss", "DB::Exception: Operation timeout", "Operation timed out", "ConnectionPoolWithFailover: Connection failed at try", "DB::Exception: New table appeared in database being dropped or detached. Try again", "is already started to be removing by another replica right now", + "DB::Exception: Cannot enqueue query", "Shutdown is called for table", # It happens in SYSTEM SYNC REPLICA query if session with ZooKeeper is being reinitialized. DISTRIBUTED_DDL_TIMEOUT_MSG # FIXME ] diff --git a/tests/config/config.d/database_replicated.xml b/tests/config/config.d/database_replicated.xml index 9a3b4d68ea6..e51d90dd4d4 100644 --- a/tests/config/config.d/database_replicated.xml +++ b/tests/config/config.d/database_replicated.xml @@ -22,9 +22,9 @@ 10000 30000 1000 - 2000 - 4000 - trace + 4000 + 5000 + information false 1000000000000000 diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 6fe01b5df03..9db4c85e36d 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -474,6 +474,11 @@ class ClickHouseCluster: cmd += " client" return cmd + def copy_file_from_container_to_container(self, src_node, src_path, dst_node, dst_path): + fname = os.path.basename(src_path) + run_and_check([f"docker cp {src_node.docker_id}:{src_path} {self.instances_dir}"], shell=True) + run_and_check([f"docker cp {self.instances_dir}/{fname} {dst_node.docker_id}:{dst_path}"], shell=True) + def setup_zookeeper_secure_cmd(self, instance, env_variables, docker_compose_yml_dir): logging.debug('Setup ZooKeeper Secure') zookeeper_docker_compose_path = p.join(docker_compose_yml_dir, 'docker_compose_zookeeper_secure.yml') @@ -1836,6 +1841,10 @@ class ClickHouseInstance: build_opts = self.query("SELECT value FROM system.build_options WHERE name = 'CXX_FLAGS'") return "-fsanitize={}".format(sanitizer_name) in build_opts + def is_debug_build(self): + build_opts = self.query("SELECT value FROM system.build_options WHERE name = 'CXX_FLAGS'") + return 'NDEBUG' not in build_opts + def is_built_with_thread_sanitizer(self): return self.is_built_with_sanitizer('thread') @@ -2024,6 +2033,37 @@ class ClickHouseInstance: return None return None + def restart_with_original_version(self, stop_start_wait_sec=300, callback_onstop=None, signal=15): + if not self.stay_alive: + raise Exception("Cannot restart not stay alive container") + self.exec_in_container(["bash", "-c", "pkill -{} clickhouse".format(signal)], user='root') + retries = int(stop_start_wait_sec / 0.5) + local_counter = 0 + # wait stop + while local_counter < retries: + if not self.get_process_pid("clickhouse server"): + break + time.sleep(0.5) + local_counter += 1 + + # force kill if server hangs + if self.get_process_pid("clickhouse server"): + # server can die before kill, so don't throw exception, it's expected + self.exec_in_container(["bash", "-c", "pkill -{} clickhouse".format(9)], nothrow=True, user='root') + + if callback_onstop: + callback_onstop(self) + self.exec_in_container( + ["bash", "-c", "cp /usr/share/clickhouse_original /usr/bin/clickhouse && chmod 777 /usr/bin/clickhouse"], + user='root') + self.exec_in_container(["bash", "-c", + "cp /usr/share/clickhouse-odbc-bridge_fresh /usr/bin/clickhouse-odbc-bridge && chmod 777 /usr/bin/clickhouse"], + user='root') + self.exec_in_container(["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], user=str(os.getuid())) + + # wait start + assert_eq_with_retry(self, "select 1", "1", retry_count=retries) + def restart_with_latest_version(self, stop_start_wait_sec=300, callback_onstop=None, signal=15): if not self.stay_alive: raise Exception("Cannot restart not stay alive container") @@ -2044,6 +2084,9 @@ class ClickHouseInstance: if callback_onstop: callback_onstop(self) + self.exec_in_container( + ["bash", "-c", "cp /usr/bin/clickhouse /usr/share/clickhouse_original"], + user='root') self.exec_in_container( ["bash", "-c", "cp /usr/share/clickhouse_fresh /usr/bin/clickhouse && chmod 777 /usr/bin/clickhouse"], user='root') diff --git a/tests/integration/test_backward_compatibility/test_aggregate_function_state_avg.py b/tests/integration/test_backward_compatibility/test_aggregate_function_state_avg.py index 5ed97e7a9a5..feaf96c439d 100644 --- a/tests/integration/test_backward_compatibility/test_aggregate_function_state_avg.py +++ b/tests/integration/test_backward_compatibility/test_aggregate_function_state_avg.py @@ -53,3 +53,9 @@ def test_backward_compatability(start_cluster): node1.restart_with_latest_version() assert (node1.query("SELECT avgMerge(x) FROM state") == '2.5\n') + + node1.query("drop table tab") + node1.query("drop table state") + node2.query("drop table tab") + node3.query("drop table tab") + node4.query("drop table tab") \ No newline at end of file diff --git a/tests/integration/test_backward_compatibility/test_data_skipping_indices.py b/tests/integration/test_backward_compatibility/test_data_skipping_indices.py index 45b85897798..db6a3eb7a08 100644 --- a/tests/integration/test_backward_compatibility/test_data_skipping_indices.py +++ b/tests/integration/test_backward_compatibility/test_data_skipping_indices.py @@ -5,7 +5,7 @@ import pytest from helpers.cluster import ClickHouseCluster -cluster = ClickHouseCluster(__file__) +cluster = ClickHouseCluster(__file__, name="skipping_indices") node = cluster.add_instance('node', image='yandex/clickhouse-server', tag='21.6', stay_alive=True, with_installed_binary=True) @@ -41,4 +41,4 @@ def test_index(start_cluster): node.query(""" SELECT * FROM data WHERE value = 20000 SETTINGS force_data_skipping_indices = 'value_index' SETTINGS force_data_skipping_indices = 'value_index', max_rows_to_read=1; DROP TABLE data; - """) + """) \ No newline at end of file diff --git a/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py b/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py index 7c20b3c2476..abebaaea8b8 100644 --- a/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py +++ b/tests/integration/test_backward_compatibility/test_detach_part_wrong_partition_id.py @@ -30,3 +30,7 @@ def test_detach_part_wrong_partition_id(start_cluster): num_detached = node_21_6.query("select count() from system.detached_parts") assert num_detached == '1\n' + + node_21_6.restart_with_original_version() + + node_21_6.query("drop table tab SYNC") diff --git a/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py b/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py index b3f5c68cf68..9a7c7f73eb5 100644 --- a/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py +++ b/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py @@ -27,3 +27,6 @@ def test_select_aggregate_alias_column(start_cluster): node1.query("select sum(x_alias) from remote('node{1,2}', default, tab)") node2.query("select sum(x_alias) from remote('node{1,2}', default, tab)") + + node1.query("drop table tab") + node2.query("drop table tab") \ No newline at end of file diff --git a/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py b/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py index 463fadc36e8..54dd53c344e 100644 --- a/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py +++ b/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py @@ -29,3 +29,5 @@ def test_backward_compatability(start_cluster): "select s, count() from remote('node{1,2}', default, tab) group by s order by toUInt64(s) limit 50") print(res) assert res == ''.join('{}\t2\n'.format(i) for i in range(50)) + node1.query("drop table tab") + node2.query("drop table tab") diff --git a/tests/integration/test_dictionaries_postgresql/configs/dictionaries/postgres_dict.xml b/tests/integration/test_dictionaries_postgresql/configs/dictionaries/postgres_dict.xml index 4ee07d0972a..734da0cff70 100644 --- a/tests/integration/test_dictionaries_postgresql/configs/dictionaries/postgres_dict.xml +++ b/tests/integration/test_dictionaries_postgresql/configs/dictionaries/postgres_dict.xml @@ -19,10 +19,10 @@ id - UInt32 + - id + key UInt32 @@ -65,10 +65,10 @@ id - UInt32 + - id + key UInt32 diff --git a/tests/integration/test_dictionaries_postgresql/test.py b/tests/integration/test_dictionaries_postgresql/test.py index 6eb4a04ed2c..58a503bd571 100644 --- a/tests/integration/test_dictionaries_postgresql/test.py +++ b/tests/integration/test_dictionaries_postgresql/test.py @@ -13,11 +13,11 @@ node1 = cluster.add_instance('node1', postgres_dict_table_template = """ CREATE TABLE IF NOT EXISTS {} ( - id Integer NOT NULL, value Integer NOT NULL, PRIMARY KEY (id)) + id Integer NOT NULL, key Integer NOT NULL, value Integer NOT NULL, PRIMARY KEY (id)) """ click_dict_table_template = """ CREATE TABLE IF NOT EXISTS `test`.`dict_table_{}` ( - `id` UInt64, `value` UInt32 + `key` UInt32, `value` UInt32 ) ENGINE = Dictionary({}) """ @@ -43,7 +43,7 @@ def create_and_fill_postgres_table(cursor, table_name, port, host): create_postgres_table(cursor, table_name) # Fill postgres table using clickhouse postgres table function and check table_func = '''postgresql('{}:{}', 'clickhouse', '{}', 'postgres', 'mysecretpassword')'''.format(host, port, table_name) - node1.query('''INSERT INTO TABLE FUNCTION {} SELECT number, number from numbers(10000) + node1.query('''INSERT INTO TABLE FUNCTION {} SELECT number, number, number from numbers(10000) '''.format(table_func, table_name)) result = node1.query("SELECT count() FROM {}".format(table_func)) assert result.rstrip() == '10000' @@ -82,7 +82,7 @@ def test_load_dictionaries(started_cluster): node1.query("SYSTEM RELOAD DICTIONARY {}".format(dict_name)) assert node1.query("SELECT count() FROM `test`.`dict_table_{}`".format(table_name)).rstrip() == '10000' - assert node1.query("SELECT dictGetUInt32('{}', 'id', toUInt64(0))".format(dict_name)) == '0\n' + assert node1.query("SELECT dictGetUInt32('{}', 'key', toUInt64(0))".format(dict_name)) == '0\n' assert node1.query("SELECT dictGetUInt32('{}', 'value', toUInt64(9999))".format(dict_name)) == '9999\n' cursor.execute("DROP TABLE IF EXISTS {}".format(table_name)) @@ -252,11 +252,11 @@ def test_dictionary_with_replicas(started_cluster): create_postgres_table(cursor1, 'test1') create_postgres_table(cursor2, 'test1') - cursor1.execute('INSERT INTO test1 select i, i from generate_series(0, 99) as t(i);'); - cursor2.execute('INSERT INTO test1 select i, i from generate_series(100, 199) as t(i);'); + cursor1.execute('INSERT INTO test1 select i, i, i from generate_series(0, 99) as t(i);') + cursor2.execute('INSERT INTO test1 select i, i, i from generate_series(100, 199) as t(i);') create_dict('test1', 1) - result = node1.query("SELECT * FROM `test`.`dict_table_test1` ORDER BY id") + result = node1.query("SELECT * FROM `test`.`dict_table_test1` ORDER BY key") # priority 0 - non running port assert node1.contains_in_log('PostgreSQLConnectionPool: Connection error*') diff --git a/tests/integration/test_distributed_respect_user_timeouts/test.py b/tests/integration/test_distributed_respect_user_timeouts/test.py index d8eb92d96b5..03c85a82c90 100644 --- a/tests/integration/test_distributed_respect_user_timeouts/test.py +++ b/tests/integration/test_distributed_respect_user_timeouts/test.py @@ -3,6 +3,7 @@ import os.path import timeit import pytest +import logging from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager from helpers.test_tools import TSV @@ -11,6 +12,8 @@ cluster = ClickHouseCluster(__file__) NODES = {'node' + str(i): None for i in (1, 2)} +IS_DEBUG = False + CREATE_TABLES_SQL = ''' CREATE DATABASE test; @@ -104,6 +107,11 @@ def started_cluster(request): try: cluster.start() + if cluster.instances["node1"].is_debug_build(): + global IS_DEBUG + IS_DEBUG = True + logging.warning("Debug build is too slow to show difference in timings. We disable checks.") + for node_id, node in list(NODES.items()): node.query(CREATE_TABLES_SQL) node.query(INSERT_SQL_TEMPLATE.format(node_id=node_id)) @@ -133,8 +141,9 @@ def _check_timeout_and_exception(node, user, query_base, query): # And it should timeout no faster than: measured_timeout = timeit.default_timer() - start - assert expected_timeout - measured_timeout <= TIMEOUT_MEASUREMENT_EPS - assert measured_timeout - expected_timeout <= TIMEOUT_DIFF_UPPER_BOUND[user][query_base] + if not IS_DEBUG: + assert expected_timeout - measured_timeout <= TIMEOUT_MEASUREMENT_EPS + assert measured_timeout - expected_timeout <= TIMEOUT_DIFF_UPPER_BOUND[user][query_base] # And exception should reflect connection attempts: _check_exception(exception, repeats) diff --git a/tests/integration/test_keeper_back_to_back/test.py b/tests/integration/test_keeper_back_to_back/test.py index 41c270e05e8..64f2f42d71e 100644 --- a/tests/integration/test_keeper_back_to_back/test.py +++ b/tests/integration/test_keeper_back_to_back/test.py @@ -90,6 +90,46 @@ def test_sequential_nodes(started_cluster): genuine_childs = list(sorted(genuine_zk.get_children("/test_sequential_nodes"))) fake_childs = list(sorted(fake_zk.get_children("/test_sequential_nodes"))) assert genuine_childs == fake_childs + + genuine_zk.create("/test_sequential_nodes_1") + fake_zk.create("/test_sequential_nodes_1") + + genuine_zk.create("/test_sequential_nodes_1/a", sequence=True) + fake_zk.create("/test_sequential_nodes_1/a", sequence=True) + + genuine_zk.create("/test_sequential_nodes_1/a0000000002") + fake_zk.create("/test_sequential_nodes_1/a0000000002") + + genuine_throw = False + fake_throw = False + try: + genuine_zk.create("/test_sequential_nodes_1/a", sequence=True) + except Exception as ex: + genuine_throw = True + + try: + fake_zk.create("/test_sequential_nodes_1/a", sequence=True) + except Exception as ex: + fake_throw = True + + assert genuine_throw == True + assert fake_throw == True + + genuine_childs_1 = list(sorted(genuine_zk.get_children("/test_sequential_nodes_1"))) + fake_childs_1 = list(sorted(fake_zk.get_children("/test_sequential_nodes_1"))) + assert genuine_childs_1 == fake_childs_1 + + genuine_zk.create("/test_sequential_nodes_2") + fake_zk.create("/test_sequential_nodes_2") + + genuine_zk.create("/test_sequential_nodes_2/node") + fake_zk.create("/test_sequential_nodes_2/node") + genuine_zk.create("/test_sequential_nodes_2/node", sequence=True) + fake_zk.create("/test_sequential_nodes_2/node", sequence=True) + + genuine_childs_2 = list(sorted(genuine_zk.get_children("/test_sequential_nodes_2"))) + fake_childs_2 = list(sorted(fake_zk.get_children("/test_sequential_nodes_2"))) + assert genuine_childs_2 == fake_childs_2 finally: for zk in [genuine_zk, fake_zk]: stop_zk(zk) diff --git a/tests/queries/0_stateless/01516_drop_table_stress.reference b/tests/integration/test_keeper_snapshot_small_distance/__init__.py similarity index 100% rename from tests/queries/0_stateless/01516_drop_table_stress.reference rename to tests/integration/test_keeper_snapshot_small_distance/__init__.py diff --git a/tests/integration/test_keeper_snapshot_small_distance/configs/keeper_config1.xml b/tests/integration/test_keeper_snapshot_small_distance/configs/keeper_config1.xml new file mode 100644 index 00000000000..b3c010fed24 --- /dev/null +++ b/tests/integration/test_keeper_snapshot_small_distance/configs/keeper_config1.xml @@ -0,0 +1,37 @@ + + + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 75 + 5 + 5000 + 10000 + trace + + + + + 1 + node1 + 44444 + 3 + + + 2 + node2 + 44444 + 2 + + + 3 + node3 + 44444 + 1 + + + + diff --git a/tests/integration/test_keeper_snapshot_small_distance/configs/keeper_config2.xml b/tests/integration/test_keeper_snapshot_small_distance/configs/keeper_config2.xml new file mode 100644 index 00000000000..f2de2c96f06 --- /dev/null +++ b/tests/integration/test_keeper_snapshot_small_distance/configs/keeper_config2.xml @@ -0,0 +1,37 @@ + + + 9181 + 2 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 75 + 5 + 5000 + 10000 + trace + + + + + 1 + node1 + 44444 + 3 + + + 2 + node2 + 44444 + 2 + + + 3 + node3 + 44444 + 1 + + + + diff --git a/tests/integration/test_keeper_snapshot_small_distance/configs/keeper_config3.xml b/tests/integration/test_keeper_snapshot_small_distance/configs/keeper_config3.xml new file mode 100644 index 00000000000..609c9045259 --- /dev/null +++ b/tests/integration/test_keeper_snapshot_small_distance/configs/keeper_config3.xml @@ -0,0 +1,37 @@ + + + 9181 + 3 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 75 + 5 + 5000 + 10000 + trace + + + + + 1 + node1 + 44444 + 3 + + + 2 + node2 + 44444 + 2 + + + 3 + node3 + 44444 + 1 + + + + diff --git a/tests/integration/test_keeper_snapshot_small_distance/configs/logs_conf.xml b/tests/integration/test_keeper_snapshot_small_distance/configs/logs_conf.xml new file mode 100644 index 00000000000..318a6bca95d --- /dev/null +++ b/tests/integration/test_keeper_snapshot_small_distance/configs/logs_conf.xml @@ -0,0 +1,12 @@ + + 3 + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + diff --git a/tests/integration/test_keeper_snapshot_small_distance/test.py b/tests/integration/test_keeper_snapshot_small_distance/test.py new file mode 100644 index 00000000000..4acd76806b4 --- /dev/null +++ b/tests/integration/test_keeper_snapshot_small_distance/test.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +##!/usr/bin/env python3 +import pytest +from helpers.cluster import ClickHouseCluster +from multiprocessing.dummy import Pool +from kazoo.client import KazooClient, KazooState +import random +import string +import os +import time + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance('node1', main_configs=['configs/keeper_config1.xml'], stay_alive=True) +node2 = cluster.add_instance('node2', main_configs=['configs/keeper_config2.xml'], stay_alive=True) +node3 = cluster.add_instance('node3', main_configs=['configs/keeper_config3.xml'], stay_alive=True) + +def start_zookeeper(node): + node1.exec_in_container(['bash', '-c', '/opt/zookeeper/bin/zkServer.sh start']) + +def stop_zookeeper(node): + node.exec_in_container(['bash', '-c', '/opt/zookeeper/bin/zkServer.sh stop']) + +def clear_zookeeper(node): + node.exec_in_container(['bash', '-c', 'rm -fr /zookeeper/*']) + +def restart_and_clear_zookeeper(node): + stop_zookeeper(node) + clear_zookeeper(node) + start_zookeeper(node) + +def clear_clickhouse_data(node): + node.exec_in_container(['bash', '-c', 'rm -fr /var/lib/clickhouse/coordination/logs/* /var/lib/clickhouse/coordination/snapshots/*']) + +def convert_zookeeper_data(node): + cmd = '/usr/bin/clickhouse keeper-converter --zookeeper-logs-dir /zookeeper/version-2/ --zookeeper-snapshots-dir /zookeeper/version-2/ --output-dir /var/lib/clickhouse/coordination/snapshots' + node.exec_in_container(['bash', '-c', cmd]) + return os.path.join('/var/lib/clickhouse/coordination/snapshots', node.exec_in_container(['bash', '-c', 'ls /var/lib/clickhouse/coordination/snapshots']).strip()) + +def stop_clickhouse(node): + node.stop_clickhouse() + +def start_clickhouse(node): + node.start_clickhouse() + +def copy_zookeeper_data(make_zk_snapshots, node): + stop_zookeeper(node) + + if make_zk_snapshots: # force zookeeper to create snapshot + start_zookeeper(node) + stop_zookeeper(node) + + stop_clickhouse(node) + clear_clickhouse_data(node) + convert_zookeeper_data(node) + start_zookeeper(node) + start_clickhouse(node) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + yield cluster + + finally: + cluster.shutdown() + +def get_fake_zk(node, timeout=30.0): + _fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(node.name) + ":9181", timeout=timeout) + _fake_zk_instance.start() + return _fake_zk_instance + +def get_genuine_zk(node, timeout=30.0): + _genuine_zk_instance = KazooClient(hosts=cluster.get_instance_ip(node.name) + ":2181", timeout=timeout) + _genuine_zk_instance.start() + return _genuine_zk_instance + + +def test_snapshot_and_load(started_cluster): + restart_and_clear_zookeeper(node1) + genuine_connection = get_genuine_zk(node1) + for node in [node1, node2, node3]: + print("Stop and clear", node.name, "with dockerid", node.docker_id) + stop_clickhouse(node) + clear_clickhouse_data(node) + + for i in range(1000): + genuine_connection.create("/test" + str(i), b"data") + + print("Data loaded to zookeeper") + + stop_zookeeper(node1) + start_zookeeper(node1) + stop_zookeeper(node1) + + print("Data copied to node1") + resulted_path = convert_zookeeper_data(node1) + print("Resulted path", resulted_path) + for node in [node2, node3]: + print("Copy snapshot from", node1.name, "to", node.name) + cluster.copy_file_from_container_to_container(node1, resulted_path, node, '/var/lib/clickhouse/coordination/snapshots') + + print("Starting clickhouses") + + p = Pool(3) + result = p.map_async(start_clickhouse, [node1, node2, node3]) + result.wait() + + print("Loading additional data") + fake_zks = [get_fake_zk(node) for node in [node1, node2, node3]] + for i in range(1000): + fake_zk = random.choice(fake_zks) + try: + fake_zk.create("/test" + str(i + 1000), b"data") + except Exception as ex: + print("Got exception:" + str(ex)) + + print("Final") + fake_zks[0].create("/test10000", b"data") diff --git a/tests/integration/test_library_bridge/test.py b/tests/integration/test_library_bridge/test.py index 97b2ccfbdbe..12a967ebaa4 100644 --- a/tests/integration/test_library_bridge/test.py +++ b/tests/integration/test_library_bridge/test.py @@ -44,6 +44,11 @@ def ch_cluster(): '/usr/bin/g++ -shared -o /etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.so -fPIC /etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.cpp'], user='root') + instance.exec_in_container( + ['bash', '-c', + '/usr/bin/g++ -shared -o /dict_lib_copy.so -fPIC /etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.cpp'], user='root') + instance.exec_in_container(['bash', '-c', 'ln -s /dict_lib_copy.so /etc/clickhouse-server/config.d/dictionaries_lib/dict_lib_symlink.so']) + yield cluster finally: @@ -59,6 +64,7 @@ def test_load_all(ch_cluster): if instance.is_built_with_memory_sanitizer(): pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + instance.query('DROP DICTIONARY IF EXISTS lib_dict') instance.query(''' CREATE DICTIONARY lib_dict (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64) PRIMARY KEY key @@ -128,6 +134,7 @@ def test_load_keys(ch_cluster): if instance.is_built_with_memory_sanitizer(): pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + instance.query('DROP DICTIONARY IF EXISTS lib_dict_ckc') instance.query(''' CREATE DICTIONARY lib_dict_ckc (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64) PRIMARY KEY key @@ -148,6 +155,7 @@ def test_load_all_many_rows(ch_cluster): pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") num_rows = [1000, 10000, 100000, 1000000] + instance.query('DROP DICTIONARY IF EXISTS lib_dict') for num in num_rows: instance.query(''' CREATE DICTIONARY lib_dict (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64) @@ -267,6 +275,42 @@ def test_bridge_dies_with_parent(ch_cluster): instance.query('DROP DICTIONARY lib_dict_c') +def test_path_validation(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + instance.query('DROP DICTIONARY IF EXISTS lib_dict_c') + instance.query(''' + CREATE DICTIONARY lib_dict_c (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64) + PRIMARY KEY key SOURCE(library(PATH '/etc/clickhouse-server/config.d/dictionaries_lib/dict_lib_symlink.so')) + LAYOUT(CACHE( + SIZE_IN_CELLS 10000000 + BLOCK_SIZE 4096 + FILE_SIZE 16777216 + READ_BUFFER_SIZE 1048576 + MAX_STORED_KEYS 1048576)) + LIFETIME(2) ; + ''') + + result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(1));''') + assert(result.strip() == '101') + + instance.query('DROP DICTIONARY IF EXISTS lib_dict_c') + instance.query(''' + CREATE DICTIONARY lib_dict_c (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64) + PRIMARY KEY key SOURCE(library(PATH '/etc/clickhouse-server/config.d/dictionaries_lib/../../../../dict_lib_copy.so')) + LAYOUT(CACHE( + SIZE_IN_CELLS 10000000 + BLOCK_SIZE 4096 + FILE_SIZE 16777216 + READ_BUFFER_SIZE 1048576 + MAX_STORED_KEYS 1048576)) + LIFETIME(2) ; + ''') + result = instance.query_and_get_error('''select dictGet(lib_dict_c, 'value1', toUInt64(1));''') + assert('DB::Exception: File path /etc/clickhouse-server/config.d/dictionaries_lib/../../../../dict_lib_copy.so is not inside /etc/clickhouse-server/config.d/dictionaries_lib' in result) + + if __name__ == '__main__': cluster.start() input("Cluster created, press any key to destroy...") diff --git a/tests/integration/test_odbc_interaction/configs/dictionaries/postgres_odbc_hashed_dictionary.xml b/tests/integration/test_odbc_interaction/configs/dictionaries/postgres_odbc_hashed_dictionary.xml index 6aad3ad9917..a65360b0e26 100644 --- a/tests/integration/test_odbc_interaction/configs/dictionaries/postgres_odbc_hashed_dictionary.xml +++ b/tests/integration/test_odbc_interaction/configs/dictionaries/postgres_odbc_hashed_dictionary.xml @@ -18,7 +18,7 @@ - column1 + id diff --git a/tests/integration/test_odbc_interaction/configs/dictionaries/sqlite3_odbc_cached_dictionary.xml b/tests/integration/test_odbc_interaction/configs/dictionaries/sqlite3_odbc_cached_dictionary.xml index 45f3966ee8a..3a505b79304 100644 --- a/tests/integration/test_odbc_interaction/configs/dictionaries/sqlite3_odbc_cached_dictionary.xml +++ b/tests/integration/test_odbc_interaction/configs/dictionaries/sqlite3_odbc_cached_dictionary.xml @@ -20,7 +20,7 @@ - X + id diff --git a/tests/integration/test_odbc_interaction/configs/dictionaries/sqlite3_odbc_hashed_dictionary.xml b/tests/integration/test_odbc_interaction/configs/dictionaries/sqlite3_odbc_hashed_dictionary.xml index 18a14b896bd..5b53818cf13 100644 --- a/tests/integration/test_odbc_interaction/configs/dictionaries/sqlite3_odbc_hashed_dictionary.xml +++ b/tests/integration/test_odbc_interaction/configs/dictionaries/sqlite3_odbc_hashed_dictionary.xml @@ -20,7 +20,7 @@ - X + id diff --git a/tests/integration/test_odbc_interaction/test.py b/tests/integration/test_odbc_interaction/test.py index 39a283448f5..4d2f70ad08c 100644 --- a/tests/integration/test_odbc_interaction/test.py +++ b/tests/integration/test_odbc_interaction/test.py @@ -99,19 +99,19 @@ def started_cluster(): logging.debug(f"sqlite data received: {sqlite_db}") node1.exec_in_container( - ["sqlite3", sqlite_db, "CREATE TABLE t1(x INTEGER PRIMARY KEY ASC, y, z);"], + ["sqlite3", sqlite_db, "CREATE TABLE t1(id INTEGER PRIMARY KEY ASC, x INTEGER, y, z);"], privileged=True, user='root') node1.exec_in_container( - ["sqlite3", sqlite_db, "CREATE TABLE t2(X INTEGER PRIMARY KEY ASC, Y, Z);"], + ["sqlite3", sqlite_db, "CREATE TABLE t2(id INTEGER PRIMARY KEY ASC, X INTEGER, Y, Z);"], privileged=True, user='root') node1.exec_in_container( - ["sqlite3", sqlite_db, "CREATE TABLE t3(X INTEGER PRIMARY KEY ASC, Y, Z);"], + ["sqlite3", sqlite_db, "CREATE TABLE t3(id INTEGER PRIMARY KEY ASC, X INTEGER, Y, Z);"], privileged=True, user='root') node1.exec_in_container( - ["sqlite3", sqlite_db, "CREATE TABLE t4(X INTEGER PRIMARY KEY ASC, Y, Z);"], + ["sqlite3", sqlite_db, "CREATE TABLE t4(id INTEGER PRIMARY KEY ASC, X INTEGER, Y, Z);"], privileged=True, user='root') node1.exec_in_container( - ["sqlite3", sqlite_db, "CREATE TABLE tf1(x INTEGER PRIMARY KEY ASC, y, z);"], + ["sqlite3", sqlite_db, "CREATE TABLE tf1(id INTEGER PRIMARY KEY ASC, x INTEGER, y, z);"], privileged=True, user='root') logging.debug("sqlite tables created") mysql_conn = get_mysql_conn() @@ -128,7 +128,7 @@ def started_cluster(): cursor = postgres_conn.cursor() cursor.execute( - "create table if not exists clickhouse.test_table (column1 int primary key, column2 varchar(40) not null)") + "create table if not exists clickhouse.test_table (id int primary key, column1 int not null, column2 varchar(40) not null)") yield cluster @@ -210,9 +210,9 @@ def test_sqlite_simple_select_function_works(started_cluster): sqlite_setup = node1.odbc_drivers["SQLite3"] sqlite_db = sqlite_setup["Database"] - node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t1 values(1, 2, 3);"], + node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t1 values(1, 1, 2, 3);"], privileged=True, user='root') - assert node1.query("select * from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], 't1')) == "1\t2\t3\n" + assert node1.query("select * from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], 't1')) == "1\t1\t2\t3\n" assert node1.query("select y from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], 't1')) == "2\n" assert node1.query("select z from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], 't1')) == "3\n" @@ -228,10 +228,10 @@ def test_sqlite_table_function(started_cluster): sqlite_setup = node1.odbc_drivers["SQLite3"] sqlite_db = sqlite_setup["Database"] - node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO tf1 values(1, 2, 3);"], + node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO tf1 values(1, 1, 2, 3);"], privileged=True, user='root') node1.query("create table odbc_tf as odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], 'tf1')) - assert node1.query("select * from odbc_tf") == "1\t2\t3\n" + assert node1.query("select * from odbc_tf") == "1\t1\t2\t3\n" assert node1.query("select y from odbc_tf") == "2\n" assert node1.query("select z from odbc_tf") == "3\n" @@ -246,7 +246,7 @@ def test_sqlite_simple_select_storage_works(started_cluster): sqlite_setup = node1.odbc_drivers["SQLite3"] sqlite_db = sqlite_setup["Database"] - node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t4 values(1, 2, 3);"], + node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t4 values(1, 1, 2, 3);"], privileged=True, user='root') node1.query("create table SqliteODBC (x Int32, y String, z String) engine = ODBC('DSN={}', '', 't4')".format( sqlite_setup["DSN"])) @@ -264,7 +264,7 @@ def test_sqlite_odbc_hashed_dictionary(started_cluster): skip_test_msan(node1) sqlite_db = node1.odbc_drivers["SQLite3"]["Database"] - node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t2 values(1, 2, 3);"], + node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t2 values(1, 1, 2, 3);"], privileged=True, user='root') node1.query("SYSTEM RELOAD DICTIONARY sqlite3_odbc_hashed") @@ -282,7 +282,7 @@ def test_sqlite_odbc_hashed_dictionary(started_cluster): logging.debug("Waiting dictionary to update for the second time") time.sleep(0.1) - node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t2 values(200, 2, 7);"], + node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t2 values(200, 200, 2, 7);"], privileged=True, user='root') # No reload because of invalidate query @@ -299,7 +299,7 @@ def test_sqlite_odbc_hashed_dictionary(started_cluster): assert_eq_with_retry(node1, "select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(1))", "3") assert_eq_with_retry(node1, "select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(200))", "1") # still default - node1.exec_in_container(["sqlite3", sqlite_db, "REPLACE INTO t2 values(1, 2, 5);"], + node1.exec_in_container(["sqlite3", sqlite_db, "REPLACE INTO t2 values(1, 1, 2, 5);"], privileged=True, user='root') assert_eq_with_retry(node1, "select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(1))", "5") @@ -310,7 +310,7 @@ def test_sqlite_odbc_cached_dictionary(started_cluster): skip_test_msan(node1) sqlite_db = node1.odbc_drivers["SQLite3"]["Database"] - node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t3 values(1, 2, 3);"], + node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t3 values(1, 1, 2, 3);"], privileged=True, user='root') assert node1.query("select dictGetUInt8('sqlite3_odbc_cached', 'Z', toUInt64(1))") == "3\n" @@ -319,12 +319,12 @@ def test_sqlite_odbc_cached_dictionary(started_cluster): node1.exec_in_container(["chmod", "a+rw", "/tmp"], privileged=True, user='root') node1.exec_in_container(["chmod", "a+rw", sqlite_db], privileged=True, user='root') - node1.query("insert into table function odbc('DSN={};ReadOnly=0', '', 't3') values (200, 2, 7)".format( + node1.query("insert into table function odbc('DSN={};ReadOnly=0', '', 't3') values (200, 200, 2, 7)".format( node1.odbc_drivers["SQLite3"]["DSN"])) assert node1.query("select dictGetUInt8('sqlite3_odbc_cached', 'Z', toUInt64(200))") == "7\n" # new value - node1.exec_in_container(["sqlite3", sqlite_db, "REPLACE INTO t3 values(1, 2, 12);"], + node1.exec_in_container(["sqlite3", sqlite_db, "REPLACE INTO t3 values(1, 1, 2, 12);"], privileged=True, user='root') assert_eq_with_retry(node1, "select dictGetUInt8('sqlite3_odbc_cached', 'Z', toUInt64(1))", "12") @@ -336,7 +336,7 @@ def test_postgres_odbc_hashed_dictionary_with_schema(started_cluster): conn = get_postgres_conn(started_cluster) cursor = conn.cursor() cursor.execute("truncate table clickhouse.test_table") - cursor.execute("insert into clickhouse.test_table values(1, 'hello'),(2, 'world')") + cursor.execute("insert into clickhouse.test_table values(1, 1, 'hello'),(2, 2, 'world')") node1.query("SYSTEM RELOAD DICTIONARY postgres_odbc_hashed") assert_eq_with_retry(node1, "select dictGetString('postgres_odbc_hashed', 'column2', toUInt64(1))", "hello") assert_eq_with_retry(node1, "select dictGetString('postgres_odbc_hashed', 'column2', toUInt64(2))", "world") @@ -348,7 +348,7 @@ def test_postgres_odbc_hashed_dictionary_no_tty_pipe_overflow(started_cluster): conn = get_postgres_conn(started_cluster) cursor = conn.cursor() cursor.execute("truncate table clickhouse.test_table") - cursor.execute("insert into clickhouse.test_table values(3, 'xxx')") + cursor.execute("insert into clickhouse.test_table values(3, 3, 'xxx')") for i in range(100): try: node1.query("system reload dictionary postgres_odbc_hashed", timeout=15) @@ -369,13 +369,13 @@ def test_postgres_insert(started_cluster): # reconstruction of connection string. node1.query( - "create table pg_insert (column1 UInt8, column2 String) engine=ODBC('DSN=postgresql_odbc;Servername=postgre-sql.local', 'clickhouse', 'test_table')") - node1.query("insert into pg_insert values (1, 'hello'), (2, 'world')") - assert node1.query("select * from pg_insert") == '1\thello\n2\tworld\n' - node1.query("insert into table function odbc('DSN=postgresql_odbc', 'clickhouse', 'test_table') format CSV 3,test") + "create table pg_insert (id UInt64, column1 UInt8, column2 String) engine=ODBC('DSN=postgresql_odbc;Servername=postgre-sql.local', 'clickhouse', 'test_table')") + node1.query("insert into pg_insert values (1, 1, 'hello'), (2, 2, 'world')") + assert node1.query("select * from pg_insert") == '1\t1\thello\n2\t2\tworld\n' + node1.query("insert into table function odbc('DSN=postgresql_odbc', 'clickhouse', 'test_table') format CSV 3,3,test") node1.query( "insert into table function odbc('DSN=postgresql_odbc;Servername=postgre-sql.local', 'clickhouse', 'test_table')" \ - " select number, 's' || toString(number) from numbers (4, 7)") + " select number, number, 's' || toString(number) from numbers (4, 7)") assert node1.query("select sum(column1), count(column1) from pg_insert") == "55\t10\n" assert node1.query( "select sum(n), count(n) from (select (*,).1 as n from (select * from odbc('DSN=postgresql_odbc', 'clickhouse', 'test_table')))") == "55\t10\n" @@ -426,19 +426,19 @@ def test_odbc_postgres_date_data_type(started_cluster): conn = get_postgres_conn(started_cluster); cursor = conn.cursor() - cursor.execute("CREATE TABLE IF NOT EXISTS clickhouse.test_date (column1 integer, column2 date)") + cursor.execute("CREATE TABLE IF NOT EXISTS clickhouse.test_date (id integer, column1 integer, column2 date)") - cursor.execute("INSERT INTO clickhouse.test_date VALUES (1, '2020-12-01')") - cursor.execute("INSERT INTO clickhouse.test_date VALUES (2, '2020-12-02')") - cursor.execute("INSERT INTO clickhouse.test_date VALUES (3, '2020-12-03')") + cursor.execute("INSERT INTO clickhouse.test_date VALUES (1, 1, '2020-12-01')") + cursor.execute("INSERT INTO clickhouse.test_date VALUES (2, 2, '2020-12-02')") + cursor.execute("INSERT INTO clickhouse.test_date VALUES (3, 3, '2020-12-03')") conn.commit() node1.query( ''' - CREATE TABLE test_date (column1 UInt64, column2 Date) + CREATE TABLE test_date (id UInt64, column1 UInt64, column2 Date) ENGINE=ODBC('DSN=postgresql_odbc; Servername=postgre-sql.local', 'clickhouse', 'test_date')''') - expected = '1\t2020-12-01\n2\t2020-12-02\n3\t2020-12-03\n' + expected = '1\t1\t2020-12-01\n2\t2\t2020-12-02\n3\t3\t2020-12-03\n' result = node1.query('SELECT * FROM test_date'); assert(result == expected) cursor.execute("DROP TABLE IF EXISTS clickhouse.test_date") diff --git a/tests/integration/test_read_temporary_tables_on_failure/test.py b/tests/integration/test_read_temporary_tables_on_failure/test.py index e62c7c9eaec..ae59fb31641 100644 --- a/tests/integration/test_read_temporary_tables_on_failure/test.py +++ b/tests/integration/test_read_temporary_tables_on_failure/test.py @@ -24,3 +24,4 @@ def test_different_versions(start_cluster): node.query("SELECT 1", settings={'max_concurrent_queries_for_user': 1}) assert node.contains_in_log('Too many simultaneous queries for user') assert not node.contains_in_log('Unknown packet') + assert not node.contains_in_log('Unexpected packet') diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index 4bcad7de16f..450caafb9ba 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -305,3 +305,12 @@ def test_startup_without_zk(started_cluster): main_node.query("EXCHANGE TABLES startup.rmt AND startup.m") assert main_node.query("SELECT (*,).1 FROM startup.m") == "42\n" + + +def test_server_uuid(started_cluster): + uuid1 = main_node.query("select serverUUID()") + uuid2 = dummy_node.query("select serverUUID()") + assert uuid1 != uuid2 + main_node.restart_clickhouse() + uuid1_after_restart = main_node.query("select serverUUID()") + assert uuid1 == uuid1_after_restart diff --git a/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/test.py b/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/test.py index f426c3619a4..f557a69569a 100644 --- a/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/test.py +++ b/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/test.py @@ -37,6 +37,9 @@ def cluster(): with_hdfs=True) logging.info("Starting cluster...") cluster.start() + if cluster.instances["node1"].is_debug_build(): + # https://github.com/ClickHouse/ClickHouse/issues/27814 + pytest.skip("libhdfs3 calls rand function which does not pass harmful check in debug build") logging.info("Cluster started") fs = HdfsClient(hosts=cluster.hdfs_ip) diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index cff2b972983..8a6bf10b3c5 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -180,28 +180,6 @@ def avro_confluent_message(schema_registry_client, value): }) return serializer.encode_record_with_schema('test_subject', schema, value) -# Fixtures - -@pytest.fixture(scope="module") -def kafka_cluster(): - try: - global kafka_id - cluster.start() - kafka_id = instance.cluster.kafka_docker_id - print(("kafka_id is {}".format(kafka_id))) - yield cluster - - finally: - cluster.shutdown() - -@pytest.fixture(autouse=True) -def kafka_setup_teardown(): - instance.query('DROP DATABASE IF EXISTS test; CREATE DATABASE test;') - wait_kafka_is_available() # ensure kafka is alive - kafka_producer_send_heartbeat_msg() # ensure python kafka client is ok - # print("kafka is available - running test") - yield # run test - # Tests def test_kafka_settings_old_syntax(kafka_cluster): @@ -283,6 +261,11 @@ def test_kafka_json_as_string(kafka_cluster): kafka_produce(kafka_cluster, 'kafka_json_as_string', ['{"t": 123, "e": {"x": "woof"} }', '', '{"t": 124, "e": {"x": "test"} }', '{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"}']) + # 'tombstone' record (null value) = marker of deleted record + producer = KafkaProducer(bootstrap_servers="localhost:{}".format(cluster.kafka_port), value_serializer=producer_serializer, key_serializer=producer_serializer) + producer.send(topic='kafka_json_as_string', key='xxx') + producer.flush() + instance.query(''' CREATE TABLE test.kafka (field String) ENGINE = Kafka @@ -694,6 +677,8 @@ def describe_consumer_group(kafka_cluster, name): def kafka_cluster(): try: cluster.start() + kafka_id = instance.cluster.kafka_docker_id + print(("kafka_id is {}".format(kafka_id))) yield cluster finally: cluster.shutdown() @@ -1124,6 +1109,7 @@ def test_kafka_protobuf_no_delimiter(kafka_cluster): def test_kafka_materialized_view(kafka_cluster): + instance.query(''' DROP TABLE IF EXISTS test.view; DROP TABLE IF EXISTS test.consumer; diff --git a/tests/integration/test_storage_kerberized_kafka/test.py b/tests/integration/test_storage_kerberized_kafka/test.py index 126c52bb1d9..f5fba594022 100644 --- a/tests/integration/test_storage_kerberized_kafka/test.py +++ b/tests/integration/test_storage_kerberized_kafka/test.py @@ -55,6 +55,9 @@ def kafka_produce(kafka_cluster, topic, messages, timestamp=None): def kafka_cluster(): try: cluster.start() + if instance.is_debug_build(): + # https://github.com/ClickHouse/ClickHouse/issues/27651 + pytest.skip("librdkafka calls system function for kinit which does not pass harmful check in debug build") yield cluster finally: cluster.shutdown() diff --git a/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh b/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh index 724630057d9..399a4677a44 100755 --- a/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh +++ b/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh @@ -42,10 +42,10 @@ popd > /dev/null #SCRIPTDIR=`dirname "$SCRIPTPATH"` SCRIPTDIR=$SCRIPTPATH -cat "$SCRIPTDIR"/00282_merging.sql | $CLICKHOUSE_CLIENT --preferred_block_size_bytes=10 -n > "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout 2>&1 +cat "$SCRIPTDIR"/00282_merging.sql | $CLICKHOUSE_CLIENT --preferred_block_size_bytes=10 -n > "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout cmp "$SCRIPTDIR"/00282_merging.reference "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout && echo PASSED || echo FAILED -cat "$SCRIPTDIR"/00282_merging.sql | $CLICKHOUSE_CLIENT --preferred_block_size_bytes=20 -n > "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout 2>&1 +cat "$SCRIPTDIR"/00282_merging.sql | $CLICKHOUSE_CLIENT --preferred_block_size_bytes=20 -n > "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout cmp "$SCRIPTDIR"/00282_merging.reference "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout && echo PASSED || echo FAILED rm "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout diff --git a/tests/queries/0_stateless/00818_alias_bug_4110.reference b/tests/queries/0_stateless/00818_alias_bug_4110.reference index e6013d269c2..210fc67db66 100644 --- a/tests/queries/0_stateless/00818_alias_bug_4110.reference +++ b/tests/queries/0_stateless/00818_alias_bug_4110.reference @@ -4,7 +4,6 @@ 11 12 12 11 10 10 -10 11 11 12 11 10 12 11 12 diff --git a/tests/queries/0_stateless/00818_alias_bug_4110.sql b/tests/queries/0_stateless/00818_alias_bug_4110.sql index 7b2fd5d3864..df7e70cb275 100644 --- a/tests/queries/0_stateless/00818_alias_bug_4110.sql +++ b/tests/queries/0_stateless/00818_alias_bug_4110.sql @@ -5,7 +5,7 @@ select s.a + 1 as b, s.a + 2 as a from (select 10 as a) s; select s.a + 2 as b, s.a + 1 as a from (select 10 as a) s; select a, a as a from (select 10 as a); -select s.a, a, a + 1 as a from (select 10 as a) as s; +select s.a, a, a + 1 as a from (select 10 as a) as s; -- { serverError 352 } select s.a + 2 as b, b - 1 as a from (select 10 as a) s; select s.a as a, s.a + 2 as b from (select 10 as a) s; select s.a + 1 as a, s.a + 2 as b from (select 10 as a) s; diff --git a/tests/queries/0_stateless/00987_distributed_stack_overflow.sql b/tests/queries/0_stateless/00987_distributed_stack_overflow.sql index d2e2b8f37ef..1ef7c543252 100644 --- a/tests/queries/0_stateless/00987_distributed_stack_overflow.sql +++ b/tests/queries/0_stateless/00987_distributed_stack_overflow.sql @@ -4,8 +4,7 @@ DROP TABLE IF EXISTS distr2; CREATE TABLE distr (x UInt8) ENGINE = Distributed(test_shard_localhost, currentDatabase(), distr); -- { serverError 269 } -CREATE TABLE distr0 (x UInt8) ENGINE = Distributed(test_shard_localhost, '', distr0); -SELECT * FROM distr0; -- { serverError 581 } +CREATE TABLE distr0 (x UInt8) ENGINE = Distributed(test_shard_localhost, '', distr0); -- { serverError 269 } CREATE TABLE distr1 (x UInt8) ENGINE = Distributed(test_shard_localhost, currentDatabase(), distr2); CREATE TABLE distr2 (x UInt8) ENGINE = Distributed(test_shard_localhost, currentDatabase(), distr1); @@ -13,6 +12,5 @@ CREATE TABLE distr2 (x UInt8) ENGINE = Distributed(test_shard_localhost, current SELECT * FROM distr1; -- { serverError 581 } SELECT * FROM distr2; -- { serverError 581 } -DROP TABLE distr0; DROP TABLE distr1; DROP TABLE distr2; diff --git a/tests/queries/0_stateless/01054_cache_dictionary_overflow_cell.sql b/tests/queries/0_stateless/01054_cache_dictionary_overflow_cell.sql index d8d1d61be63..1b317e2165e 100644 --- a/tests/queries/0_stateless/01054_cache_dictionary_overflow_cell.sql +++ b/tests/queries/0_stateless/01054_cache_dictionary_overflow_cell.sql @@ -54,3 +54,4 @@ FROM ); DROP TABLE if exists test_01054.ints; +DROP DATABASE test_01054_overflow; diff --git a/tests/queries/0_stateless/01101_literal_column_clash.reference b/tests/queries/0_stateless/01101_literal_column_clash.reference index b89f59abb18..8f76d98575c 100644 --- a/tests/queries/0_stateless/01101_literal_column_clash.reference +++ b/tests/queries/0_stateless/01101_literal_column_clash.reference @@ -3,7 +3,6 @@ 7 0 7 1 xyzabc 2 -1 3 1 2 0 0 1 0 0 3 \N 1 2 \N 0 diff --git a/tests/queries/0_stateless/01101_literal_column_clash.sql b/tests/queries/0_stateless/01101_literal_column_clash.sql index 4a6064141ea..b9645e3609e 100644 --- a/tests/queries/0_stateless/01101_literal_column_clash.sql +++ b/tests/queries/0_stateless/01101_literal_column_clash.sql @@ -7,7 +7,7 @@ join (select '1' as sid) as t2 on t2.sid = cast(t1.iid as String); select cast(7 as String), * from (select 3 "'String'"); select cast(7 as String), * from (select number "'String'" FROM numbers(2)); SELECT concat('xyz', 'abc'), * FROM (SELECT 2 AS "'xyz'"); -with 3 as "1" select 1, "1"; +with 3 as "1" select 1, "1"; -- { serverError 352 } -- https://github.com/ClickHouse/ClickHouse/issues/9953 select 1, * from (select 2 x) a left join (select 1, 3 y) b on y = x; diff --git a/tests/queries/0_stateless/01114_mysql_database_engine_segfault.sql b/tests/queries/0_stateless/01114_mysql_database_engine_segfault.sql index af88c5af53a..5893365e11c 100644 --- a/tests/queries/0_stateless/01114_mysql_database_engine_segfault.sql +++ b/tests/queries/0_stateless/01114_mysql_database_engine_segfault.sql @@ -1 +1,2 @@ +DROP DATABASE IF EXISTS conv_main; CREATE DATABASE conv_main ENGINE = MySQL('127.0.0.1:3456', conv_main, 'metrika', 'password'); -- { serverError 501 } diff --git a/tests/queries/0_stateless/01236_graphite_mt.sql b/tests/queries/0_stateless/01236_graphite_mt.sql index ccf7c066e75..a6dd4b8b6fb 100644 --- a/tests/queries/0_stateless/01236_graphite_mt.sql +++ b/tests/queries/0_stateless/01236_graphite_mt.sql @@ -13,7 +13,7 @@ WITH dates AS today - INTERVAL 3 day as older_date ) -- Newer than 2 days are kept in windows of 600 seconds - select 1, 'sum_1', today - number * 60 - 30, number, 1, number from dates, numbers(300) union all + select 1 AS key, 'sum_1' AS s, today - number * 60 - 30, number, 1, number from dates, numbers(300) union all select 2, 'sum_1', today - number * 60 - 30, number, 1, number from dates, numbers(300) union all select 1, 'sum_2', today - number * 60 - 30, number, 1, number from dates, numbers(300) union all select 2, 'sum_2', today - number * 60 - 30, number, 1, number from dates, numbers(300) union all @@ -23,7 +23,7 @@ WITH dates AS select 2, 'max_2', today - number * 60 - 30, number, 1, number from dates, numbers(300) union all -- Older than 2 days use 6000 second windows - select 1, 'sum_1', older_date - number * 60 - 30, number, 1, number from dates, numbers(1200) union all + select 1 AS key, 'sum_1' AS s, older_date - number * 60 - 30, number, 1, number from dates, numbers(1200) union all select 2, 'sum_1', older_date - number * 60 - 30, number, 1, number from dates, numbers(1200) union all select 1, 'sum_2', older_date - number * 60 - 30, number, 1, number from dates, numbers(1200) union all select 2, 'sum_2', older_date - number * 60 - 30, number, 1, number from dates, numbers(1200) union all diff --git a/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh b/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh index ff62b70c184..e1e0018a1dd 100755 --- a/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh +++ b/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh @@ -122,3 +122,5 @@ $CLICKHOUSE_CLIENT -n --query="DROP DICTIONARY 01280_db.ssd_dict; SELECT arrayJoin([('1', toInt32(3)), ('2', toInt32(-1)), ('', toInt32(0)), ('', toInt32(0)), ('2', toInt32(-1)), ('1', toInt32(3))]) AS keys, dictGetInt32('01280_db.ssd_dict', 'b', keys); DROP DICTIONARY IF EXISTS database_for_dict.ssd_dict; DROP TABLE IF EXISTS database_for_dict.keys_table;" + +$CLICKHOUSE_CLIENT -n --query="DROP DATABASE IF EXISTS 01280_db;" diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference index b40e4f87c13..f45f1ab6104 100644 --- a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference @@ -1,8 +1,20 @@ ===http=== +{"query":"select 1 from remote('127.0.0.2', system, one) format Null\n","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} +{"query":"DESC TABLE system.one","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} +{"query":"DESC TABLE system.one","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} +{"query":"SELECT 1 FROM system.one","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} +{"query":"DESC TABLE system.one","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} +{"query":"DESC TABLE system.one","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} +{"query":"SELECT 1 FROM system.one","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} +{"query":"select 1 from remote('127.0.0.2', system, one) format Null\n","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} {"total spans":"4","unique spans":"4","unique non-zero parent spans":"3"} {"initial query spans with proper parent":"1"} {"unique non-empty tracestate values":"1"} ===native=== +{"query":"select * from url('http:\/\/127.0.0.2:8123\/?query=select%201%20format%20Null', CSV, 'a int')","status":"QueryFinish","tracestate":"another custom state","sorted_by_start_time":1} +{"query":"select 1 format Null\n","status":"QueryFinish","tracestate":"another custom state","sorted_by_start_time":1} +{"query":"select 1 format Null\n","query_status":"QueryFinish","tracestate":"another custom state","sorted_by_finish_time":1} +{"query":"select * from url('http:\/\/127.0.0.2:8123\/?query=select%201%20format%20Null', CSV, 'a int')","query_status":"QueryFinish","tracestate":"another custom state","sorted_by_finish_time":1} {"total spans":"2","unique spans":"2","unique non-zero parent spans":"2"} {"initial query spans with proper parent":"1"} {"unique non-empty tracestate values":"1"} diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh index 8f034b0bf61..59cd1b57d1e 100755 --- a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh @@ -12,6 +12,28 @@ function check_log ${CLICKHOUSE_CLIENT} --format=JSONEachRow -nq " system flush logs; +-- Show queries sorted by start time. +select attribute['db.statement'] as query, + attribute['clickhouse.query_status'] as status, + attribute['clickhouse.tracestate'] as tracestate, + 1 as sorted_by_start_time + from system.opentelemetry_span_log + where trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + and operation_name = 'query' + order by start_time_us + ; + +-- Show queries sorted by finish time. +select attribute['db.statement'] as query, + attribute['clickhouse.query_status'] as query_status, + attribute['clickhouse.tracestate'] as tracestate, + 1 as sorted_by_finish_time + from system.opentelemetry_span_log + where trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + and operation_name = 'query' + order by finish_time_us + ; + -- Check the number of query spans with given trace id, to verify it was -- propagated. select count(*) "'"'"total spans"'"'", @@ -89,10 +111,10 @@ check_log echo "===sampled===" query_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4()))))") -for i in {1..200} +for i in {1..20} do ${CLICKHOUSE_CLIENT} \ - --opentelemetry_start_trace_probability=0.1 \ + --opentelemetry_start_trace_probability=0.5 \ --query_id "$query_id-$i" \ --query "select 1 from remote('127.0.0.2', system, one) format Null" \ & @@ -108,8 +130,8 @@ wait ${CLICKHOUSE_CLIENT} -q "system flush logs" ${CLICKHOUSE_CLIENT} -q " - -- expect 200 * 0.1 = 20 sampled events on average - select if(count() > 1 and count() < 50, 'OK', 'Fail') + -- expect 20 * 0.5 = 10 sampled events on average + select if(2 <= count() and count() <= 18, 'OK', 'Fail') from system.opentelemetry_span_log where operation_name = 'query' and parent_span_id = 0 -- only account for the initial queries diff --git a/tests/queries/0_stateless/01470_columns_transformers2.reference b/tests/queries/0_stateless/01470_columns_transformers2.reference index 18c0f5c7e89..6d196530135 100644 --- a/tests/queries/0_stateless/01470_columns_transformers2.reference +++ b/tests/queries/0_stateless/01470_columns_transformers2.reference @@ -1 +1,4 @@ 100 10 324 120.00 B 8.00 B 23.00 B +0 +SELECT argMax(number, number) +FROM numbers(1) diff --git a/tests/queries/0_stateless/01470_columns_transformers2.sql b/tests/queries/0_stateless/01470_columns_transformers2.sql index 3691ef1e65d..88513d023c4 100644 --- a/tests/queries/0_stateless/01470_columns_transformers2.sql +++ b/tests/queries/0_stateless/01470_columns_transformers2.sql @@ -5,3 +5,6 @@ INSERT INTO columns_transformers VALUES (100, 10, 324, 120, 8, 23); SELECT * EXCEPT 'bytes', COLUMNS('bytes') APPLY formatReadableSize FROM columns_transformers; DROP TABLE IF EXISTS columns_transformers; + +SELECT * APPLY x->argMax(x, number) FROM numbers(1); +EXPLAIN SYNTAX SELECT * APPLY x->argMax(x, number) FROM numbers(1); diff --git a/tests/queries/0_stateless/01516_drop_table_stress.sh b/tests/queries/0_stateless/01516_drop_table_stress.sh deleted file mode 100755 index d72104c8c7f..00000000000 --- a/tests/queries/0_stateless/01516_drop_table_stress.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CURDIR"/../shell_config.sh - -function drop_database() -{ - # redirect stderr since it is racy with DROP TABLE - # and tries to remove db_01516.data too. - ${CLICKHOUSE_CLIENT} -q "DROP DATABASE IF EXISTS db_01516" 2>/dev/null -} - -function drop_table() -{ - ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS db_01516.data3;" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" - ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS db_01516.data1;" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" - ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS db_01516.data2;" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" -} - -function create() -{ - ${CLICKHOUSE_CLIENT} -q "CREATE DATABASE IF NOT EXISTS db_01516;" - ${CLICKHOUSE_CLIENT} -q "CREATE TABLE IF NOT EXISTS db_01516.data1 Engine=MergeTree() ORDER BY number AS SELECT * FROM numbers(1);" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" - ${CLICKHOUSE_CLIENT} -q "CREATE TABLE IF NOT EXISTS db_01516.data2 Engine=MergeTree() ORDER BY number AS SELECT * FROM numbers(1);" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" - ${CLICKHOUSE_CLIENT} -q "CREATE TABLE IF NOT EXISTS db_01516.data3 Engine=MergeTree() ORDER BY number AS SELECT * FROM numbers(1);" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" -} - -for _ in {1..100}; do - create - drop_table & - drop_database & - wait -done diff --git a/tests/queries/0_stateless/01516_drop_table_stress_long.reference b/tests/queries/0_stateless/01516_drop_table_stress_long.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01516_drop_table_stress_long.sh b/tests/queries/0_stateless/01516_drop_table_stress_long.sh new file mode 100755 index 00000000000..20e76ce49c7 --- /dev/null +++ b/tests/queries/0_stateless/01516_drop_table_stress_long.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +export CURR_DATABASE="test_01516_${CLICKHOUSE_DATABASE}" + +function drop_database() +{ + # redirect stderr since it is racy with DROP TABLE + # and tries to remove ${CURR_DATABASE}.data too. + ${CLICKHOUSE_CLIENT} -q "DROP DATABASE IF EXISTS ${CURR_DATABASE}" 2>/dev/null +} +trap drop_database EXIT + +function drop_table() +{ + ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${CURR_DATABASE}.data3;" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${CURR_DATABASE}.data1;" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${CURR_DATABASE}.data2;" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" +} + +function create() +{ + ${CLICKHOUSE_CLIENT} -q "CREATE DATABASE IF NOT EXISTS ${CURR_DATABASE};" + ${CLICKHOUSE_CLIENT} -q "CREATE TABLE IF NOT EXISTS ${CURR_DATABASE}.data1 Engine=MergeTree() ORDER BY number AS SELECT * FROM numbers(1);" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "CREATE TABLE IF NOT EXISTS ${CURR_DATABASE}.data2 Engine=MergeTree() ORDER BY number AS SELECT * FROM numbers(1);" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "CREATE TABLE IF NOT EXISTS ${CURR_DATABASE}.data3 Engine=MergeTree() ORDER BY number AS SELECT * FROM numbers(1);" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" +} + +for _ in {1..100}; do + create + drop_table & + drop_database & + wait +done diff --git a/tests/queries/0_stateless/01763_max_distributed_depth.sql b/tests/queries/0_stateless/01763_max_distributed_depth.sql index d1bb9e4be90..89909a3bd8d 100644 --- a/tests/queries/0_stateless/01763_max_distributed_depth.sql +++ b/tests/queries/0_stateless/01763_max_distributed_depth.sql @@ -9,7 +9,9 @@ CREATE TABLE tt6 `status` String ) -ENGINE = Distributed('test_shard_localhost', '', 'tt6', rand()); +ENGINE = Distributed('test_shard_localhost', '', 'tt7', rand()); + +CREATE TABLE tt7 as tt6 ENGINE = Distributed('test_shard_localhost', '', 'tt6', rand()); INSERT INTO tt6 VALUES (1, 1, 1, 1, 'ok'); -- { serverError 581 } diff --git a/tests/queries/0_stateless/01766_todatetime64_no_timezone_arg.reference b/tests/queries/0_stateless/01766_todatetime64_no_timezone_arg.reference index 52eea094ae4..b0e96ac9e54 100644 --- a/tests/queries/0_stateless/01766_todatetime64_no_timezone_arg.reference +++ b/tests/queries/0_stateless/01766_todatetime64_no_timezone_arg.reference @@ -1 +1 @@ -2021-03-22 00:00:00.000 +2021-03-23 00:00:00.000 diff --git a/tests/queries/0_stateless/01766_todatetime64_no_timezone_arg.sql b/tests/queries/0_stateless/01766_todatetime64_no_timezone_arg.sql index 99141a694c1..2aac922487e 100644 --- a/tests/queries/0_stateless/01766_todatetime64_no_timezone_arg.sql +++ b/tests/queries/0_stateless/01766_todatetime64_no_timezone_arg.sql @@ -1 +1 @@ -SELECT toDateTime64('2021-03-22', 3); +SELECT toDateTime64('2021-03-23', 3); diff --git a/tests/queries/0_stateless/01822_short_circuit.reference b/tests/queries/0_stateless/01822_short_circuit.reference index 0cebb9d4f75..96c4e161244 100644 --- a/tests/queries/0_stateless/01822_short_circuit.reference +++ b/tests/queries/0_stateless/01822_short_circuit.reference @@ -100,26 +100,6 @@ 18 19 0 -2 -3 -4 -5 -5 -7 -8 -9 -10 -10 -12 -13 -14 -15 -15 -17 -18 -19 -20 -0 \N \N \N @@ -139,26 +119,6 @@ \N \N \N -\N -1 -2 -3 -4 -\N -6 -7 -8 -9 -\N -11 -12 -13 -14 -\N -16 -17 -18 -19 0 2 3 @@ -220,26 +180,6 @@ 18 19 0 -2 -3 -4 -5 -5 -7 -8 -9 -10 -10 -12 -13 -14 -15 -15 -17 -18 -19 -20 -0 \N \N \N @@ -259,26 +199,6 @@ \N \N \N -\N -1 -2 -3 -4 -\N -6 -7 -8 -9 -\N -11 -12 -13 -14 -\N -16 -17 -18 -19 0 2 3 @@ -340,26 +260,6 @@ 18 19 0 -2 -3 -4 -5 -5 -7 -8 -9 -10 -10 -12 -13 -14 -15 -15 -17 -18 -19 -20 -0 \N \N \N @@ -379,26 +279,6 @@ \N \N \N -\N -1 -2 -3 -4 -\N -6 -7 -8 -9 -\N -11 -12 -13 -14 -\N -16 -17 -18 -19 0 2 3 @@ -460,26 +340,6 @@ 18 19 0 -2 -3 -4 -5 -5 -7 -8 -9 -10 -10 -12 -13 -14 -15 -15 -17 -18 -19 -20 -0 \N \N \N @@ -499,26 +359,6 @@ \N \N \N -\N -1 -2 -3 -4 -\N -6 -7 -8 -9 -\N -11 -12 -13 -14 -\N -16 -17 -18 -19 0 2 3 @@ -580,26 +420,6 @@ 18 19 0 -2 -3 -4 -5 -5 -7 -8 -9 -10 -10 -12 -13 -14 -15 -15 -17 -18 -19 -20 -0 \N \N \N @@ -619,26 +439,6 @@ \N \N \N -\N -1 -2 -3 -4 -\N -6 -7 -8 -9 -\N -11 -12 -13 -14 -\N -16 -17 -18 -19 0 2 3 @@ -700,6 +500,26 @@ 18 19 0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +0 2 3 4 @@ -759,6 +579,426 @@ 17 18 19 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +0 +2 +3 +4 +5 +5 +7 +8 +9 +10 +10 +12 +13 +14 +15 +15 +17 +18 +19 +20 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +\N +1 +2 +3 +4 +\N +6 +7 +8 +9 +\N +11 +12 +13 +14 +\N +16 +17 +18 +19 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +0 +2 +3 +4 +5 +5 +7 +8 +9 +10 +10 +12 +13 +14 +15 +15 +17 +18 +19 +20 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +\N +1 +2 +3 +4 +\N +6 +7 +8 +9 +\N +11 +12 +13 +14 +\N +16 +17 +18 +19 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +0 +2 +3 +4 +5 +5 +7 +8 +9 +10 +10 +12 +13 +14 +15 +15 +17 +18 +19 +20 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +\N +1 +2 +3 +4 +\N +6 +7 +8 +9 +\N +11 +12 +13 +14 +\N +16 +17 +18 +19 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +0 +2 +3 +4 +5 +5 +7 +8 +9 +10 +10 +12 +13 +14 +15 +15 +17 +18 +19 +20 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +\N +1 +2 +3 +4 +\N +6 +7 +8 +9 +\N +11 +12 +13 +14 +\N +16 +17 +18 +19 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +0 +2 +3 +4 +5 +5 +7 +8 +9 +10 +10 +12 +13 +14 +15 +15 +17 +18 +19 +20 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +\N +1 +2 +3 +4 +\N +6 +7 +8 +9 +\N +11 +12 +13 +14 +\N +16 +17 +18 +19 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N 00 22 33 @@ -819,6 +1059,26 @@ 1717 1818 1919 +00 +\N +\N +\N +\N +55 +\N +\N +\N +\N +1010 +\N +\N +\N +\N +1515 +\N +\N +\N +\N 10 12 13 @@ -879,6 +1139,26 @@ 27 28 29 +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +20 +\N +\N +\N +\N +25 +\N +\N +\N +\N 1970-01-01 1970-01-01 1970-01-01 @@ -939,6 +1219,26 @@ 1970-01-01 1970-01-01 1970-01-01 +1970-01-01 +\N +\N +\N +\N +1970-01-01 +\N +\N +\N +\N +1970-01-01 +\N +\N +\N +\N +1970-01-01 +\N +\N +\N +\N 1970-01-01 00:00:00 1970-01-01 05:33:20 1970-01-01 08:20:00 @@ -999,6 +1299,26 @@ 1970-01-02 23:13:20 1970-01-03 02:00:00 1970-01-03 04:46:40 +1970-01-01 00:00:00 +\N +\N +\N +\N +1970-01-01 13:53:20 +\N +\N +\N +\N +1970-01-02 03:46:40 +\N +\N +\N +\N +1970-01-02 17:40:00 +\N +\N +\N +\N 0 2 3 @@ -1060,26 +1380,6 @@ 18 19 0 -2 -3 -4 -5 -5 -7 -8 -9 -10 -10 -12 -13 -14 -15 -15 -17 -18 -19 -20 -0 \N \N \N @@ -1099,26 +1399,6 @@ \N \N \N -\N -1 -2 -3 -4 -\N -6 -7 -8 -9 -\N -11 -12 -13 -14 -\N -16 -17 -18 -19 0 2 3 @@ -1180,6 +1460,26 @@ 18 19 0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +0 2 3 4 @@ -1239,6 +1539,106 @@ 17 18 19 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +0 +2 +3 +4 +5 +5 +7 +8 +9 +10 +10 +12 +13 +14 +15 +15 +17 +18 +19 +20 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +\N +1 +2 +3 +4 +\N +6 +7 +8 +9 +\N +11 +12 +13 +14 +\N +16 +17 +18 +19 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N [] [0,1] [0,1,2] @@ -1395,3 +1795,11 @@ Decimal32 21 14 10.5 +0 +42 +21 +14 +10.5 +\N +\N +\N diff --git a/tests/queries/0_stateless/01822_short_circuit.sql b/tests/queries/0_stateless/01822_short_circuit.sql index 16908642c52..fe8a0315d4a 100644 --- a/tests/queries/0_stateless/01822_short_circuit.sql +++ b/tests/queries/0_stateless/01822_short_circuit.sql @@ -11,82 +11,102 @@ select count() from (select if(number >= 0, number, sleep(1)) from numbers(10000 select if(number % 5 == 0, toInt8OrZero(toString(number)), toInt8OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toInt8OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toInt8OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toInt8OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toUInt8OrZero(toString(number)), toUInt8OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toUInt8OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toUInt8OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toUInt8OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toInt32OrZero(toString(number)), toInt32OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toInt32OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toInt32OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toInt32OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toUInt32OrZero(toString(number)), toUInt32OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toUInt32OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toUInt32OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toUInt32OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toInt64OrZero(toString(number)), toInt64OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toInt64OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toInt64OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toInt64OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toUInt64OrZero(toString(number)), toUInt64OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toUInt64OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toUInt64OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toUInt64OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toInt128OrZero(toString(number)), toInt128OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toInt128OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toInt128OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toInt128OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toUInt128OrZero(toString(number)), toUInt128OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toUInt128OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toUInt128OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toUInt128OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toInt256OrZero(toString(number)), toInt256OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toInt256OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toInt256OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toInt256OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toUInt256OrZero(toString(number)), toUInt256OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toUInt256OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toUInt256OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toUInt256OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toFloat32OrZero(toString(number)), toFloat32OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toFloat32OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toFloat32OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toFloat32OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toFloat64OrZero(toString(number)), toFloat64OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toFloat64OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toFloat64OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toFloat64OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, repeat(toString(number), 2), repeat(toString(number + 1), 2)) from numbers(20); select if(number % 5 == 0, repeat(toString(number), 2), Null) from numbers(20); select if(number % 5 == 0, Null, repeat(toString(number), 2)) from numbers(20); +select if(number % 5, Null, repeat(toString(number), 2)) from numbers(20); select if(number % 5 == 0, toFixedString(toString(number + 10), 2), toFixedString(toString(number + 11), 2)) from numbers(20); select if(number % 5 == 0, toFixedString(toString(number + 10), 2), Null) from numbers(20); select if(number % 5 == 0, Null, toFixedString(toString(number + 10), 2)) from numbers(20); +select if(number % 5, Null, toFixedString(toString(number + 10), 2)) from numbers(20); select if(number % 5 == 0, toDateOrZero(toString(number)), toDateOrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toDateOrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toDateOrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toDateOrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toDateTimeOrZero(toString(number * 10000), 'UTC'), toDateTimeOrZero(toString((number + 1) * 10000), 'UTC')) from numbers(20); select if(number % 5 == 0, toDateTimeOrZero(toString(number * 10000), 'UTC'), Null) from numbers(20); select if(number % 5 == 0, Null, toDateTimeOrZero(toString(number * 10000), 'UTC')) from numbers(20); +select if(number % 5, Null, toDateTimeOrZero(toString(number * 10000), 'UTC')) from numbers(20); select if(number % 5 == 0, toDecimal32OrZero(toString(number), 5), toDecimal32OrZero(toString(number + 1), 5)) from numbers(20); select if(number % 5 == 0, toDecimal32OrZero(toString(number), 5), Null) from numbers(20); select if(number % 5 == 0, Null, toDecimal32OrZero(toString(number), 5)) from numbers(20); +select if(number % 5, Null, toDecimal32OrZero(toString(number), 5)) from numbers(20); select if(number % 5 == 0, toDecimal64OrZero(toString(number), 5), toDecimal64OrZero(toString(number + 1), 5)) from numbers(20); select if(number % 5 == 0, toDecimal64OrZero(toString(number), 5), Null) from numbers(20); select if(number % 5 == 0, Null, toDecimal64OrZero(toString(number), 5)) from numbers(20); +select if(number % 5, Null, toDecimal64OrZero(toString(number), 5)) from numbers(20); select if(number % 5 == 0, toDecimal128OrZero(toString(number), 5), toDecimal128OrZero(toString(number + 1), 5)) from numbers(20); select if(number % 5 == 0, toDecimal128OrZero(toString(number), 5), Null) from numbers(20); select if(number % 5 == 0, Null, toDecimal128OrZero(toString(number), 5)) from numbers(20); +select if(number % 5, Null, toDecimal128OrZero(toString(number), 5)) from numbers(20); select if(number % 5 == 0, toDecimal256OrZero(toString(number), 5), toDecimal256OrZero(toString(number + 1), 5)) from numbers(20); select if(number % 5 == 0, toDecimal256OrZero(toString(number), 5), Null) from numbers(20); select if(number % 5 == 0, Null, toDecimal256OrZero(toString(number), 5)) from numbers(20); +select if(number % 5, Null, toDecimal256OrZero(toString(number), 5)) from numbers(20); select if(number % 5 == 0, range(number), range(number + 1)) from numbers(20); select if(number % 5 == 0, replicate(toString(number), range(number)), replicate(toString(number), range(number + 1))) from numbers(20); @@ -123,3 +143,8 @@ select if(number > 0, intDiv(42, number), 1) from numbers(5); select if(number > 0, intDiv(42, number), 1) from numbers(5); select if(number > 0, 42 / toDecimal32(number, 2), 0) from numbers(5); +select if(number = 0, 0, toDecimal32(42, 2) / number) from numbers(5); +select if(isNull(x), Null, 42 / x) from (select CAST(materialize(Null), 'Nullable(Decimal32(2))') as x); +select if(isNull(x), Null, x / 0) from (select CAST(materialize(Null), 'Nullable(Decimal32(2))') as x); + +select if(isNull(x), Null, intDiv(42, x)) from (select CAST(materialize(Null), 'Nullable(Int64)') as x); diff --git a/tests/queries/0_stateless/01852_cast_operator_3.reference b/tests/queries/0_stateless/01852_cast_operator_3.reference new file mode 100644 index 00000000000..a1e54797d60 --- /dev/null +++ b/tests/queries/0_stateless/01852_cast_operator_3.reference @@ -0,0 +1,10 @@ +-1 +SELECT CAST(\'-1\', \'Int32\') +-0.1 +SELECT CAST(\'-0.1\', \'Decimal(38, 38)\') +-0.111 +SELECT CAST(\'-0.111\', \'Float64\') +[-1,2,-3] +SELECT CAST(\'[-1, 2, -3]\', \'Array(Int32)\') +[-1.1,2,-3] +SELECT CAST(\'[-1.1, 2, -3]\', \'Array(Float64)\') diff --git a/tests/queries/0_stateless/01852_cast_operator_3.sql b/tests/queries/0_stateless/01852_cast_operator_3.sql new file mode 100644 index 00000000000..1ad015a8dc4 --- /dev/null +++ b/tests/queries/0_stateless/01852_cast_operator_3.sql @@ -0,0 +1,14 @@ +SELECT -1::Int32; +EXPLAIN SYNTAX SELECT -1::Int32; + +SELECT -0.1::Decimal(38, 38); +EXPLAIN SYNTAX SELECT -0.1::Decimal(38, 38); + +SELECT -0.111::Float64; +EXPLAIN SYNTAX SELECT -0.111::Float64; + +SELECT [-1, 2, -3]::Array(Int32); +EXPLAIN SYNTAX SELECT [-1, 2, -3]::Array(Int32); + +SELECT [-1.1, 2, -3]::Array(Float64); +EXPLAIN SYNTAX SELECT [-1.1, 2, -3]::Array(Float64); diff --git a/tests/queries/0_stateless/01852_cast_operator_bad_cases.reference b/tests/queries/0_stateless/01852_cast_operator_bad_cases.reference index 2c4517e0eda..b179e5e927a 100644 --- a/tests/queries/0_stateless/01852_cast_operator_bad_cases.reference +++ b/tests/queries/0_stateless/01852_cast_operator_bad_cases.reference @@ -8,3 +8,11 @@ Syntax error Syntax error Syntax error Code: 6 +Syntax error +Syntax error +Syntax error +Syntax error +Syntax error +Syntax error +Syntax error +Syntax error diff --git a/tests/queries/0_stateless/01852_cast_operator_bad_cases.sh b/tests/queries/0_stateless/01852_cast_operator_bad_cases.sh index f2f566b78c4..6c578a0996c 100755 --- a/tests/queries/0_stateless/01852_cast_operator_bad_cases.sh +++ b/tests/queries/0_stateless/01852_cast_operator_bad_cases.sh @@ -15,3 +15,13 @@ $CLICKHOUSE_CLIENT --query="SELECT [1 2]::Array(UInt8)" 2>&1 | grep -o -m1 'Syn $CLICKHOUSE_CLIENT --query="SELECT 1 4::UInt32" 2>&1 | grep -o 'Syntax error' $CLICKHOUSE_CLIENT --query="SELECT '1' '4'::UInt32" 2>&1 | grep -o -m1 'Syntax error' $CLICKHOUSE_CLIENT --query="SELECT '1''4'::UInt32" 2>&1 | grep -o -m1 'Code: 6' + +$CLICKHOUSE_CLIENT --query="SELECT ::UInt32" 2>&1 | grep -o 'Syntax error' +$CLICKHOUSE_CLIENT --query="SELECT ::String" 2>&1 | grep -o 'Syntax error' +$CLICKHOUSE_CLIENT --query="SELECT -::Int32" 2>&1 | grep -o 'Syntax error' + +$CLICKHOUSE_CLIENT --query="SELECT [1, -]::Array(Int32)" 2>&1 | grep -o 'Syntax error' +$CLICKHOUSE_CLIENT --query="SELECT [1, 3-]::Array(Int32)" 2>&1 | grep -o 'Syntax error' +$CLICKHOUSE_CLIENT --query="SELECT [-, 2]::Array(Int32)" 2>&1 | grep -o 'Syntax error' +$CLICKHOUSE_CLIENT --query="SELECT [--, 2]::Array(Int32)" 2>&1 | grep -o 'Syntax error' +$CLICKHOUSE_CLIENT --query="SELECT [1, 2]-::Array(Int32)" 2>&1 | grep -o 'Syntax error' diff --git a/tests/queries/0_stateless/01889_sqlite_read_write.sh b/tests/queries/0_stateless/01889_sqlite_read_write.sh index 73b106e9eb4..3e7e15d2830 100755 --- a/tests/queries/0_stateless/01889_sqlite_read_write.sh +++ b/tests/queries/0_stateless/01889_sqlite_read_write.sh @@ -7,60 +7,68 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # See 01658_read_file_to_string_column.sh user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') -mkdir -p ${user_files_path}/ -chmod 777 ${user_files_path} -DB_PATH=${user_files_path}/db1 +mkdir -p "${user_files_path}/" +chmod 777 "${user_files_path}" +export CURR_DATABASE="test_01889_sqllite_${CLICKHOUSE_DATABASE}" -sqlite3 ${DB_PATH} 'DROP TABLE IF EXISTS table1' -sqlite3 ${DB_PATH} 'DROP TABLE IF EXISTS table2' -sqlite3 ${DB_PATH} 'DROP TABLE IF EXISTS table3' -sqlite3 ${DB_PATH} 'DROP TABLE IF EXISTS table4' -sqlite3 ${DB_PATH} 'DROP TABLE IF EXISTS table5' +DB_PATH=${user_files_path}/${CURR_DATABASE}_db1 +DB_PATH2=$CUR_DIR/${CURR_DATABASE}_db2 -sqlite3 ${DB_PATH} 'CREATE TABLE table1 (col1 text, col2 smallint);' -sqlite3 ${DB_PATH} 'CREATE TABLE table2 (col1 int, col2 text);' +function cleanup() +{ + ${CLICKHOUSE_CLIENT} --query="DROP DATABASE IF EXISTS ${CURR_DATABASE}" + rm -r "${DB_PATH}" "${DB_PATH2}" +} +trap cleanup EXIT -chmod ugo+w ${DB_PATH} +sqlite3 "${DB_PATH}" 'DROP TABLE IF EXISTS table1' +sqlite3 "${DB_PATH}" 'DROP TABLE IF EXISTS table2' +sqlite3 "${DB_PATH}" 'DROP TABLE IF EXISTS table3' +sqlite3 "${DB_PATH}" 'DROP TABLE IF EXISTS table4' +sqlite3 "${DB_PATH}" 'DROP TABLE IF EXISTS table5' -sqlite3 ${DB_PATH} "INSERT INTO table1 VALUES ('line1', 1), ('line2', 2), ('line3', 3)" -sqlite3 ${DB_PATH} "INSERT INTO table2 VALUES (1, 'text1'), (2, 'text2'), (3, 'text3')" +sqlite3 "${DB_PATH}" 'CREATE TABLE table1 (col1 text, col2 smallint);' +sqlite3 "${DB_PATH}" 'CREATE TABLE table2 (col1 int, col2 text);' -sqlite3 ${DB_PATH} 'CREATE TABLE table3 (col1 text, col2 int);' -sqlite3 ${DB_PATH} 'INSERT INTO table3 VALUES (NULL, 1)' -sqlite3 ${DB_PATH} "INSERT INTO table3 VALUES ('not a null', 2)" -sqlite3 ${DB_PATH} 'INSERT INTO table3 VALUES (NULL, 3)' -sqlite3 ${DB_PATH} "INSERT INTO table3 VALUES ('', 4)" +chmod ugo+w "${DB_PATH}" -sqlite3 ${DB_PATH} 'CREATE TABLE table4 (a int, b integer, c tinyint, d smallint, e mediumint, bigint, int2, int8)' -sqlite3 ${DB_PATH} 'CREATE TABLE table5 (a character(20), b varchar(10), c real, d double, e double precision, f float)' +sqlite3 "${DB_PATH}" "INSERT INTO table1 VALUES ('line1', 1), ('line2', 2), ('line3', 3)" +sqlite3 "${DB_PATH}" "INSERT INTO table2 VALUES (1, 'text1'), (2, 'text2'), (3, 'text3')" +sqlite3 "${DB_PATH}" 'CREATE TABLE table3 (col1 text, col2 int);' +sqlite3 "${DB_PATH}" 'INSERT INTO table3 VALUES (NULL, 1)' +sqlite3 "${DB_PATH}" "INSERT INTO table3 VALUES ('not a null', 2)" +sqlite3 "${DB_PATH}" 'INSERT INTO table3 VALUES (NULL, 3)' +sqlite3 "${DB_PATH}" "INSERT INTO table3 VALUES ('', 4)" + +sqlite3 "${DB_PATH}" 'CREATE TABLE table4 (a int, b integer, c tinyint, d smallint, e mediumint, bigint, int2, int8)' +sqlite3 "${DB_PATH}" 'CREATE TABLE table5 (a character(20), b varchar(10), c real, d double, e double precision, f float)' -${CLICKHOUSE_CLIENT} --query='DROP DATABASE IF EXISTS sqlite_database' ${CLICKHOUSE_CLIENT} --query="select 'create database engine'"; -${CLICKHOUSE_CLIENT} --query="CREATE DATABASE sqlite_database ENGINE = SQLite('${DB_PATH}')" +${CLICKHOUSE_CLIENT} --query="CREATE DATABASE ${CURR_DATABASE} ENGINE = SQLite('${DB_PATH}')" ${CLICKHOUSE_CLIENT} --query="select 'show database tables:'"; -${CLICKHOUSE_CLIENT} --query='SHOW TABLES FROM sqlite_database;' +${CLICKHOUSE_CLIENT} --query='SHOW TABLES FROM '"${CURR_DATABASE}"';' ${CLICKHOUSE_CLIENT} --query="select 'show creare table:'"; -${CLICKHOUSE_CLIENT} --query='SHOW CREATE TABLE sqlite_database.table1;' | sed -r 's/(.*SQLite)(.*)/\1/' -${CLICKHOUSE_CLIENT} --query='SHOW CREATE TABLE sqlite_database.table2;' | sed -r 's/(.*SQLite)(.*)/\1/' +${CLICKHOUSE_CLIENT} --query="SHOW CREATE TABLE ${CURR_DATABASE}.table1;" | sed -r 's/(.*SQLite)(.*)/\1/' +${CLICKHOUSE_CLIENT} --query="SHOW CREATE TABLE ${CURR_DATABASE}.table2;" | sed -r 's/(.*SQLite)(.*)/\1/' ${CLICKHOUSE_CLIENT} --query="select 'describe table:'"; -${CLICKHOUSE_CLIENT} --query='DESCRIBE TABLE sqlite_database.table1;' -${CLICKHOUSE_CLIENT} --query='DESCRIBE TABLE sqlite_database.table2;' +${CLICKHOUSE_CLIENT} --query="DESCRIBE TABLE ${CURR_DATABASE}.table1;" +${CLICKHOUSE_CLIENT} --query="DESCRIBE TABLE ${CURR_DATABASE}.table2;" ${CLICKHOUSE_CLIENT} --query="select 'select *:'"; -${CLICKHOUSE_CLIENT} --query='SELECT * FROM sqlite_database.table1 ORDER BY col2' -${CLICKHOUSE_CLIENT} --query='SELECT * FROM sqlite_database.table2 ORDER BY col1;' +${CLICKHOUSE_CLIENT} --query="SELECT * FROM ${CURR_DATABASE}.table1 ORDER BY col2" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM ${CURR_DATABASE}.table2 ORDER BY col1" ${CLICKHOUSE_CLIENT} --query="select 'test types'"; -${CLICKHOUSE_CLIENT} --query='SHOW CREATE TABLE sqlite_database.table4;' | sed -r 's/(.*SQLite)(.*)/\1/' -${CLICKHOUSE_CLIENT} --query='SHOW CREATE TABLE sqlite_database.table5;' | sed -r 's/(.*SQLite)(.*)/\1/' +${CLICKHOUSE_CLIENT} --query="SHOW CREATE TABLE ${CURR_DATABASE}.table4;" | sed -r 's/(.*SQLite)(.*)/\1/' +${CLICKHOUSE_CLIENT} --query="SHOW CREATE TABLE ${CURR_DATABASE}.table5;" | sed -r 's/(.*SQLite)(.*)/\1/' -${CLICKHOUSE_CLIENT} --query='DROP DATABASE IF EXISTS sqlite_database' +${CLICKHOUSE_CLIENT} --query="DROP DATABASE IF EXISTS ${CURR_DATABASE}" ${CLICKHOUSE_CLIENT} --query="select 'create table engine with table3'"; @@ -79,11 +87,9 @@ ${CLICKHOUSE_CLIENT} --query="INSERT INTO TABLE FUNCTION sqlite('${DB_PATH}', 't ${CLICKHOUSE_CLIENT} --query="SELECT * FROM sqlite('${DB_PATH}', 'table1') ORDER BY col2" -sqlite3 $CUR_DIR/db2 'DROP TABLE IF EXISTS table1' -sqlite3 $CUR_DIR/db2 'CREATE TABLE table1 (col1 text, col2 smallint);' -sqlite3 $CUR_DIR/db2 "INSERT INTO table1 VALUES ('line1', 1), ('line2', 2), ('line3', 3)" +sqlite3 "${DB_PATH2}" 'DROP TABLE IF EXISTS table1' +sqlite3 "${DB_PATH2}" 'CREATE TABLE table1 (col1 text, col2 smallint);' +sqlite3 "${DB_PATH2}" "INSERT INTO table1 VALUES ('line1', 1), ('line2', 2), ('line3', 3)" ${CLICKHOUSE_CLIENT} --query="select 'test path in clickhouse-local'"; -${CLICKHOUSE_LOCAL} --query="SELECT * FROM sqlite('$CUR_DIR/db2', 'table1') ORDER BY col2" - -rm -r ${DB_PATH} +${CLICKHOUSE_LOCAL} --query="SELECT * FROM sqlite('${DB_PATH2}', 'table1') ORDER BY col2" diff --git a/tests/queries/0_stateless/01925_test_group_by_const_consistency.reference b/tests/queries/0_stateless/01925_test_group_by_const_consistency.reference index 573541ac970..93f9e3d10db 100644 --- a/tests/queries/0_stateless/01925_test_group_by_const_consistency.reference +++ b/tests/queries/0_stateless/01925_test_group_by_const_consistency.reference @@ -1 +1,2 @@ 0 +1 0 diff --git a/tests/queries/0_stateless/01925_test_group_by_const_consistency.sql b/tests/queries/0_stateless/01925_test_group_by_const_consistency.sql index 8a5de0e7c4f..a73c06bbe49 100644 --- a/tests/queries/0_stateless/01925_test_group_by_const_consistency.sql +++ b/tests/queries/0_stateless/01925_test_group_by_const_consistency.sql @@ -1,2 +1,4 @@ -SELECT 1 as a, count() FROM numbers(10)　WHERE 0 GROUP BY a; -SELECT count() FROM numbers(10) WHERE 0 +SELECT 1 as a, count() FROM numbers(10) WHERE 0 GROUP BY a; +SELECT count() FROM numbers(10) WHERE 0; + +SELECT 1 as a, count() FROM numbers(10) WHERE 0 GROUP BY a SETTINGS empty_result_for_aggregation_by_constant_keys_on_empty_set = 0; diff --git a/tests/queries/0_stateless/01942_dateTimeToSnowflake.reference b/tests/queries/0_stateless/01942_dateTimeToSnowflake.reference new file mode 100644 index 00000000000..dfca3a10eeb --- /dev/null +++ b/tests/queries/0_stateless/01942_dateTimeToSnowflake.reference @@ -0,0 +1,6 @@ +const column +2021-08-15 18:57:56 1426860702823350272 +2021-08-15 18:57:56.492 1426860704886947840 +non-const column +2021-08-15 18:57:56 1426860702823350272 +2021-08-15 18:57:56.492 1426860704886947840 diff --git a/tests/queries/0_stateless/01942_dateTimeToSnowflake.sql b/tests/queries/0_stateless/01942_dateTimeToSnowflake.sql new file mode 100644 index 00000000000..047d8be7be5 --- /dev/null +++ b/tests/queries/0_stateless/01942_dateTimeToSnowflake.sql @@ -0,0 +1,23 @@ +-- Error cases +SELECT dateTimeToSnowflake(); -- {serverError 42} +SELECT dateTime64ToSnowflake(); -- {serverError 42} + +SELECT dateTimeToSnowflake('abc'); -- {serverError 43} +SELECT dateTime64ToSnowflake('abc'); -- {serverError 43} + +SELECT dateTimeToSnowflake('abc', 123); -- {serverError 42} +SELECT dateTime64ToSnowflake('abc', 123); -- {serverError 42} + +SELECT 'const column'; +WITH toDateTime('2021-08-15 18:57:56', 'Asia/Shanghai') AS dt +SELECT dt, dateTimeToSnowflake(dt); + +WITH toDateTime64('2021-08-15 18:57:56.492', 3, 'Asia/Shanghai') AS dt64 +SELECT dt64, dateTime64ToSnowflake(dt64); + +SELECT 'non-const column'; +WITH toDateTime('2021-08-15 18:57:56', 'Asia/Shanghai') AS x +SELECT materialize(x) as dt, dateTimeToSnowflake(dt);; + +WITH toDateTime64('2021-08-15 18:57:56.492', 3, 'Asia/Shanghai') AS x +SELECT materialize(x) as dt64, dateTime64ToSnowflake(dt64); diff --git a/tests/queries/0_stateless/01942_snowflakeToDateTime.reference b/tests/queries/0_stateless/01942_snowflakeToDateTime.reference new file mode 100644 index 00000000000..bed18023f6a --- /dev/null +++ b/tests/queries/0_stateless/01942_snowflakeToDateTime.reference @@ -0,0 +1,3 @@ +const column +UTC 1426860704886947840 2021-08-15 10:57:56 DateTime(\'UTC\') 2021-08-15 10:57:56.492 DateTime64(3, \'UTC\') +Asia/Shanghai 1426860704886947840 2021-08-15 18:57:56 DateTime(\'Asia/Shanghai\') 2021-08-15 18:57:56.492 DateTime64(3, \'Asia/Shanghai\') diff --git a/tests/queries/0_stateless/01942_snowflakeToDateTime.sql b/tests/queries/0_stateless/01942_snowflakeToDateTime.sql new file mode 100644 index 00000000000..f6f171afabf --- /dev/null +++ b/tests/queries/0_stateless/01942_snowflakeToDateTime.sql @@ -0,0 +1,32 @@ +-- -- Error cases +SELECT snowflakeToDateTime(); -- {serverError 42} +SELECT snowflakeToDateTime64(); -- {serverError 42} + +SELECT snowflakeToDateTime('abc'); -- {serverError 43} +SELECT snowflakeToDateTime64('abc'); -- {serverError 43} + +SELECT snowflakeToDateTime('abc', 123); -- {serverError 43} +SELECT snowflakeToDateTime64('abc', 123); -- {serverError 43} + +SELECT 'const column'; +WITH + CAST(1426860704886947840 AS Int64) AS i64, + 'UTC' AS tz +SELECT + tz, + i64, + snowflakeToDateTime(i64, tz) as dt, + toTypeName(dt), + snowflakeToDateTime64(i64, tz) as dt64, + toTypeName(dt64); + +WITH + CAST(1426860704886947840 AS Int64) AS i64, + 'Asia/Shanghai' AS tz +SELECT + tz, + i64, + snowflakeToDateTime(i64, tz) as dt, + toTypeName(dt), + snowflakeToDateTime64(i64, tz) as dt64, + toTypeName(dt64); \ No newline at end of file diff --git a/tests/queries/0_stateless/01950_aliases_bad_cast.reference b/tests/queries/0_stateless/01950_aliases_bad_cast.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01950_aliases_bad_cast.sql b/tests/queries/0_stateless/01950_aliases_bad_cast.sql new file mode 100644 index 00000000000..bdd2339f855 --- /dev/null +++ b/tests/queries/0_stateless/01950_aliases_bad_cast.sql @@ -0,0 +1,2 @@ +SELECT 1, * FROM (SELECT NULL AS `1`); -- { serverError 352 } +SELECT '7', 'xyz', * FROM (SELECT NULL AS `'xyz'`); -- { serverError 352 } diff --git a/tests/queries/0_stateless/01950_kill_large_group_by_query.reference b/tests/queries/0_stateless/01950_kill_large_group_by_query.reference index 1602d6587ad..f1df2658897 100644 --- a/tests/queries/0_stateless/01950_kill_large_group_by_query.reference +++ b/tests/queries/0_stateless/01950_kill_large_group_by_query.reference @@ -1,2 +1,2 @@ -finished test_01948_tcp_default default SELECT * FROM\n (\n SELECT a.name as n\n FROM\n (\n SELECT \'Name\' as name, number FROM system.numbers LIMIT 2000000\n ) AS a,\n (\n SELECT \'Name\' as name, number FROM system.numbers LIMIT 2000000\n ) as b\n GROUP BY n\n )\n LIMIT 20\n FORMAT Null -finished test_01948_http_default default SELECT * FROM\n (\n SELECT a.name as n\n FROM\n (\n SELECT \'Name\' as name, number FROM system.numbers LIMIT 2000000\n ) AS a,\n (\n SELECT \'Name\' as name, number FROM system.numbers LIMIT 2000000\n ) as b\n GROUP BY n\n )\n LIMIT 20\n FORMAT Null +finished test_01948_tcp_default default SELECT * FROM\n (\n SELECT a.name as n\n FROM\n (\n SELECT \'Name\' as name, number FROM system.numbers LIMIT 2000000\n ) AS a,\n (\n SELECT \'Name\' as name2, number FROM system.numbers LIMIT 2000000\n ) as b\n GROUP BY n\n )\n LIMIT 20\n FORMAT Null +finished test_01948_http_default default SELECT * FROM\n (\n SELECT a.name as n\n FROM\n (\n SELECT \'Name\' as name, number FROM system.numbers LIMIT 2000000\n ) AS a,\n (\n SELECT \'Name\' as name2, number FROM system.numbers LIMIT 2000000\n ) as b\n GROUP BY n\n )\n LIMIT 20\n FORMAT Null diff --git a/tests/queries/0_stateless/01950_kill_large_group_by_query.sh b/tests/queries/0_stateless/01950_kill_large_group_by_query.sh index 465b923187e..0b369c7257e 100755 --- a/tests/queries/0_stateless/01950_kill_large_group_by_query.sh +++ b/tests/queries/0_stateless/01950_kill_large_group_by_query.sh @@ -23,7 +23,7 @@ $CLICKHOUSE_CLIENT --max_execution_time 10 --query_id "test_01948_tcp_$CLICKHOUS SELECT 'Name' as name, number FROM system.numbers LIMIT 2000000 ) AS a, ( - SELECT 'Name' as name, number FROM system.numbers LIMIT 2000000 + SELECT 'Name' as name2, number FROM system.numbers LIMIT 2000000 ) as b GROUP BY n ) @@ -44,7 +44,7 @@ ${CLICKHOUSE_CURL_COMMAND} -q --max-time 10 -sS "$CLICKHOUSE_URL&query_id=test_0 SELECT 'Name' as name, number FROM system.numbers LIMIT 2000000 ) AS a, ( - SELECT 'Name' as name, number FROM system.numbers LIMIT 2000000 + SELECT 'Name' as name2, number FROM system.numbers LIMIT 2000000 ) as b GROUP BY n ) diff --git a/tests/queries/0_stateless/02006_test_positional_arguments.reference b/tests/queries/0_stateless/02006_test_positional_arguments.reference new file mode 100644 index 00000000000..a8e8ccec100 --- /dev/null +++ b/tests/queries/0_stateless/02006_test_positional_arguments.reference @@ -0,0 +1,50 @@ +-- { echo } +select x3, x2, x1 from test order by 1; +1 100 100 +10 1 10 +100 10 1 +select x3, x2, x1 from test order by x3; +1 100 100 +10 1 10 +100 10 1 +select x3, x2, x1 from test order by 1 desc; +100 10 1 +10 1 10 +1 100 100 +select x3, x2, x1 from test order by x3 desc; +100 10 1 +10 1 10 +1 100 100 +insert into test values (1, 10, 200), (10, 1, 200), (100, 100, 1); +select x3, x2 from test group by x3, x2; +200 1 +10 1 +200 10 +1 100 +100 10 +select x3, x2 from test group by 1, 2; +200 1 +10 1 +200 10 +1 100 +100 10 +select x1, x2, x3 from test order by x3 limit 1 by x3; +100 100 1 +10 1 10 +1 10 100 +1 10 200 +select x1, x2, x3 from test order by 3 limit 1 by 3; +100 100 1 +10 1 10 +1 10 100 +1 10 200 +select x1, x2, x3 from test order by x3 limit 1 by x1; +100 100 1 +10 1 10 +1 10 100 +select x1, x2, x3 from test order by 3 limit 1 by 1; +100 100 1 +10 1 10 +1 10 100 +select max(x3), max(x2), max(x1) from test group by 1; -- { serverError 43 } +select max(x1) from test order by 1; -- { serverError 43 } diff --git a/tests/queries/0_stateless/02006_test_positional_arguments.sql b/tests/queries/0_stateless/02006_test_positional_arguments.sql new file mode 100644 index 00000000000..dc45b288016 --- /dev/null +++ b/tests/queries/0_stateless/02006_test_positional_arguments.sql @@ -0,0 +1,26 @@ +set enable_positional_arguments = 1; + +drop table if exists test; +create table test(x1 Int, x2 Int, x3 Int) engine=Memory(); +insert into test values (1, 10, 100), (10, 1, 10), (100, 100, 1); + +-- { echo } +select x3, x2, x1 from test order by 1; +select x3, x2, x1 from test order by x3; + +select x3, x2, x1 from test order by 1 desc; +select x3, x2, x1 from test order by x3 desc; + +insert into test values (1, 10, 200), (10, 1, 200), (100, 100, 1); +select x3, x2 from test group by x3, x2; +select x3, x2 from test group by 1, 2; + +select x1, x2, x3 from test order by x3 limit 1 by x3; +select x1, x2, x3 from test order by 3 limit 1 by 3; +select x1, x2, x3 from test order by x3 limit 1 by x1; +select x1, x2, x3 from test order by 3 limit 1 by 1; + +select max(x3), max(x2), max(x1) from test group by 1; -- { serverError 43 } +select max(x1) from test order by 1; -- { serverError 43 } + + diff --git a/tests/queries/0_stateless/02009_from_infile.reference b/tests/queries/0_stateless/02009_from_infile.reference new file mode 100644 index 00000000000..48483fe50c1 --- /dev/null +++ b/tests/queries/0_stateless/02009_from_infile.reference @@ -0,0 +1,3 @@ +Hello +Hello +Correct URL diff --git a/tests/queries/0_stateless/02009_from_infile.sh b/tests/queries/0_stateless/02009_from_infile.sh new file mode 100755 index 00000000000..578ac14f558 --- /dev/null +++ b/tests/queries/0_stateless/02009_from_infile.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +set -e + +[ -e "${CLICKHOUSE_TMP}"/test_infile.gz ] && rm "${CLICKHOUSE_TMP}"/test_infile.gz +[ -e "${CLICKHOUSE_TMP}"/test_infile ] && rm "${CLICKHOUSE_TMP}"/test_infile + +echo "Hello" > "${CLICKHOUSE_TMP}"/test_infile + +gzip "${CLICKHOUSE_TMP}"/test_infile + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test_infile;" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE test_infile (word String) ENGINE=Memory();" +${CLICKHOUSE_CLIENT} --query "INSERT INTO test_infile FROM INFILE '${CLICKHOUSE_TMP}/test_infile.gz' FORMAT CSV;" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_infile;" + +# if it not fails, select will print information +${CLICKHOUSE_LOCAL} --query "CREATE TABLE test_infile (word String) ENGINE=Memory(); INSERT INTO test_infile FROM INFILE '${CLICKHOUSE_TMP}/test_infile.gz' FORMAT CSV; SELECT * from test_infile;" + +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=DROP+TABLE" -d 'IF EXISTS test_infile_url' +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=CREATE" -d 'TABLE test_infile_url (x String) ENGINE = Memory' +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "INSERT INTO test_infile_url FROM INFILE '${CLICKHOUSE_TMP}/test_infile.gz' FORMAT CSV" 2>&1 | grep -q "UNKNOWN_TYPE_OF_QUERY" && echo "Correct URL" || echo 'Fail' +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d 'SELECT x FROM test_infile_url' +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=DROP+TABLE" -d 'test_infile_url' diff --git a/tests/queries/0_stateless/02011_dictionary_empty_attribute_list.reference b/tests/queries/0_stateless/02011_dictionary_empty_attribute_list.reference new file mode 100644 index 00000000000..7938dcdde86 --- /dev/null +++ b/tests/queries/0_stateless/02011_dictionary_empty_attribute_list.reference @@ -0,0 +1,3 @@ +0 +1 +0 diff --git a/tests/queries/0_stateless/02011_dictionary_empty_attribute_list.sql b/tests/queries/0_stateless/02011_dictionary_empty_attribute_list.sql new file mode 100644 index 00000000000..5c0668cb839 --- /dev/null +++ b/tests/queries/0_stateless/02011_dictionary_empty_attribute_list.sql @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS test_table; +CREATE TABLE test_table (id UInt64) ENGINE=TinyLog; +INSERT INTO test_table VALUES (0); + +DROP DICTIONARY IF EXISTS test_dictionary; +CREATE DICTIONARY test_dictionary (id UInt64) PRIMARY KEY id LAYOUT(DIRECT()) SOURCE(CLICKHOUSE(TABLE 'test_table')); +SELECT * FROM test_dictionary; +SELECT dictHas('test_dictionary', toUInt64(0)); +SELECT dictHas('test_dictionary', toUInt64(1)); + +DROP DICTIONARY test_dictionary; +DROP TABLE test_table; diff --git a/tests/queries/0_stateless/02012_low_cardinality_uuid_with_extremes.reference b/tests/queries/0_stateless/02012_low_cardinality_uuid_with_extremes.reference new file mode 100644 index 00000000000..af2447df807 --- /dev/null +++ b/tests/queries/0_stateless/02012_low_cardinality_uuid_with_extremes.reference @@ -0,0 +1,4 @@ +0562380c-d1f3-4091-83d5-8c972f534317 + +0562380c-d1f3-4091-83d5-8c972f534317 +0562380c-d1f3-4091-83d5-8c972f534317 diff --git a/tests/queries/0_stateless/02012_low_cardinality_uuid_with_extremes.sql b/tests/queries/0_stateless/02012_low_cardinality_uuid_with_extremes.sql new file mode 100644 index 00000000000..191383cc978 --- /dev/null +++ b/tests/queries/0_stateless/02012_low_cardinality_uuid_with_extremes.sql @@ -0,0 +1,11 @@ +DROP TABLE IF EXISTS tbl; + +SET allow_suspicious_low_cardinality_types = 1; +CREATE TABLE tbl (`lc` LowCardinality(UUID)) ENGINE = Memory; + +INSERT INTO tbl VALUES ('0562380c-d1f3-4091-83d5-8c972f534317'); + +SET extremes = 1; +SELECT * FROM tbl; + +DROP TABLE tbl; diff --git a/tests/queries/0_stateless/test_abxi8n/parquet_decimal0.parquet b/tests/queries/0_stateless/test_abxi8n/parquet_decimal0.parquet deleted file mode 100644 index cfa6cc2e0b6..00000000000 Binary files a/tests/queries/0_stateless/test_abxi8n/parquet_decimal0.parquet and /dev/null differ diff --git a/tests/queries/0_stateless/test_abxi8n/parquet_decimal1.parquet b/tests/queries/0_stateless/test_abxi8n/parquet_decimal1.parquet deleted file mode 100644 index 19d5e387f8d..00000000000 Binary files a/tests/queries/0_stateless/test_abxi8n/parquet_decimal1.parquet and /dev/null differ diff --git a/tests/queries/0_stateless/test_abxi8n/parquet_decimal2.parquet b/tests/queries/0_stateless/test_abxi8n/parquet_decimal2.parquet deleted file mode 100644 index c722f9701c8..00000000000 Binary files a/tests/queries/0_stateless/test_abxi8n/parquet_decimal2.parquet and /dev/null differ diff --git a/tests/queries/0_stateless/test_abxi8n/parquet_decimal3_1.parquet b/tests/queries/0_stateless/test_abxi8n/parquet_decimal3_1.parquet deleted file mode 100644 index 39acd4a8c14..00000000000 Binary files a/tests/queries/0_stateless/test_abxi8n/parquet_decimal3_1.parquet and /dev/null differ diff --git a/tests/queries/0_stateless/test_abxi8n/parquet_decimal3_2.parquet b/tests/queries/0_stateless/test_abxi8n/parquet_decimal3_2.parquet deleted file mode 100644 index 0de83a7eeb6..00000000000 Binary files a/tests/queries/0_stateless/test_abxi8n/parquet_decimal3_2.parquet and /dev/null differ diff --git a/tests/queries/0_stateless/test_abxi8n/parquet_decimal3_3.parquet b/tests/queries/0_stateless/test_abxi8n/parquet_decimal3_3.parquet deleted file mode 100644 index ad2e746c4fd..00000000000 Binary files a/tests/queries/0_stateless/test_abxi8n/parquet_decimal3_3.parquet and /dev/null differ diff --git a/tests/queries/0_stateless/test_dozlem/arrays.arrow b/tests/queries/0_stateless/test_dozlem/arrays.arrow deleted file mode 100644 index 9ab37ff5890..00000000000 Binary files a/tests/queries/0_stateless/test_dozlem/arrays.arrow and /dev/null differ diff --git a/tests/queries/0_stateless/test_dozlem/arrow_all_types_1.arrow b/tests/queries/0_stateless/test_dozlem/arrow_all_types_1.arrow deleted file mode 100644 index e52e997d12f..00000000000 Binary files a/tests/queries/0_stateless/test_dozlem/arrow_all_types_1.arrow and /dev/null differ diff --git a/tests/queries/0_stateless/test_dozlem/arrow_all_types_2.arrow b/tests/queries/0_stateless/test_dozlem/arrow_all_types_2.arrow deleted file mode 100644 index 04e1214ef84..00000000000 Binary files a/tests/queries/0_stateless/test_dozlem/arrow_all_types_2.arrow and /dev/null differ diff --git a/tests/queries/0_stateless/test_dozlem/arrow_all_types_5.arrow b/tests/queries/0_stateless/test_dozlem/arrow_all_types_5.arrow deleted file mode 100644 index 29de25ee984..00000000000 Binary files a/tests/queries/0_stateless/test_dozlem/arrow_all_types_5.arrow and /dev/null differ diff --git a/tests/queries/0_stateless/test_dozlem/dicts.arrow b/tests/queries/0_stateless/test_dozlem/dicts.arrow deleted file mode 100644 index c2af6dfedee..00000000000 Binary files a/tests/queries/0_stateless/test_dozlem/dicts.arrow and /dev/null differ diff --git a/tests/queries/0_stateless/test_dozlem/nullable_arrays.arrow b/tests/queries/0_stateless/test_dozlem/nullable_arrays.arrow deleted file mode 100644 index 2e497d358c6..00000000000 Binary files a/tests/queries/0_stateless/test_dozlem/nullable_arrays.arrow and /dev/null differ diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 83ad14c44dc..c2b5782e766 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -416,7 +416,6 @@ "01507_clickhouse_server_start_with_embedded_config", "01509_dictionary_preallocate", "01516_create_table_primary_key", - "01516_drop_table_stress", "01517_drop_mv_with_inner_table", "01526_complex_key_dict_direct_layout", "01527_clickhouse_local_optimize", @@ -487,7 +486,6 @@ "01824_prefer_global_in_and_join", "01870_modulo_partition_key", "01870_buffer_flush", // creates database - "01889_sqlite_read_write", "01889_postgresql_protocol_null_fields", "01889_check_row_policy_defined_using_user_function", "01921_concurrent_ttl_and_normal_merges_zookeeper_long", // heavy test, better to run sequentially diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 67238318e25..799eaaf1c05 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,4 +1,6 @@ +v21.8.4.51-lts 2021-08-17 v21.8.3.44-lts 2021-08-12 +v21.7.8.58-stable 2021-08-17 v21.7.7.47-stable 2021-08-09 v21.7.6.39-stable 2021-08-06 v21.7.5.29-stable 2021-07-28 diff --git a/website/blog/en/2021/performance-test-1.md b/website/blog/en/2021/performance-test-1.md new file mode 100644 index 00000000000..3d15a7ea3ec --- /dev/null +++ b/website/blog/en/2021/performance-test-1.md @@ -0,0 +1,83 @@ +--- +title: 'Testing the Performance of ClickHouse' +image: 'https://blog-images.clickhouse.tech/en/2021/performance-testing-1/chebu-crop.jpg' +date: '2021-08-19' +author: '[Alexander Kuzmenkov](https://github.com/akuzm)' +tags: ['testing', 'performance'] +--- + +One of the main selling points of ClickHouse is that it's very fast, in many cases utilizing the hardware up to the theoretical limits. This was noted by many independent benchmark such as [this one](http://brandonharris.io/redshift-clickhouse-time-series/). This speed boils down to a right combination of architectural choices and algorithmic optimizations, sprinkled with a dash of pixie dust. There is an [overview of these factors](https://clickhouse.tech/docs/en/faq/general/why-clickhouse-is-so-fast) on our website, or a talk by the ClickHouse lead developer Alexey Milovidov ["The secrets of ClickHouse performance optimizations"](https://www.youtube.com/watch?v=ZOZQCQEtrz8). But this is a static picture of "how the things are". Software is a living and changing organism, and ClickHouse is changing very fast — to give you a scale, in July 2021 we merged 319 pull requests made by 60 different authors ([live statistics here](https://gh-api.clickhouse.tech/play?user=play#c2VsZWN0IGRhdGVfdHJ1bmMoJ21vbnRoJywgY3JlYXRlZF9hdCkgbW9udGgsIHVuaXEoY3JlYXRvcl91c2VyX2xvZ2luKSBhdXRob3JzLCB1bmlxKG51bWJlcikgcHJzIGZyb20gZ2l0aHViX2V2ZW50cyB3aGVyZSByZXBvX25hbWUgPSAnQ2xpY2tIb3VzZS9DbGlja0hvdXNlJyBhbmQgbm90IGhhc0FueShsYWJlbHMsIFsncHItYmFja3BvcnQnLCAncHItZG9jdW1lbnRhdGlvbicsICdwci1jaGVycnlwaWNrJ10pIGFuZCBtZXJnZWQgYW5kIGNyZWF0ZWRfYXQgYmV0d2VlbiAnMjAyMC0wOS0wMScgYW5kICcyMDIxLTA5LTAxJyBncm91cCBieSBtb250aA==)). Any quality that is not actively selected for is going to be lost in this endless stream of changes, and the performance is no exception. For this reason, we have to have some process that allows us to ensure than ClickHouse always stays fast. + +# Measuring and Comparing the Performance + +How do we know it is fast, in the first place? We do a lot of benchmarks, many kinds of them. The most basic kind of a benchmark is a micro-benchmark, that doesn't use the full code of the server and tests a particular algorithm in isolation. We use them to choose a better inner loop for some aggregate function, or to test various layouts of hash tables, and so on. For example, when we discovered that a competing database engine completes a query with `sum` aggregate function twice as fast, we tested a couple of dozen implementations of `sum` to ultimately find the one that gives the best performance (see [a talk](https://www.youtube.com/watch?v=MJJfWoWJq0o) about this, in Russian). But testing a particular algorithm by itself is not enough to say how the entire query is going to work. We have to also make end-to-end measurements of entire queries, often using the real production data, because the particulars of the data (e.g. the cardinality and the distribution of values) heavily influence the performance. Currently we have about 3000 end-to-end test queries organized into about 200 [tests](https://github.com/ClickHouse/ClickHouse/tree/6c4c3df96e41425185beb0c471a8dde0ce6f25a7/tests/performance). Many of them use real data sets, such as the [production data of Yandex.Metrica](https://clickhouse.tech/docs/en/getting-started/example-datasets/metrica/), obfuscated with `clickhouse-obfuscator` as described [here](https://habr.com/ru/company/yandex/blog/485096/). + +Micro-benchmarks are normally ran by a developer while working on the code, but it is not practical to manually run the entire battery of the end-to-end tests for each change. We use an automated system that does this for each pull request as part of continuous integration checks. It measures whether the code changes introduced by a pull request influenced the performance, for which kinds of queries and by how much, and alerts the developer if there is a regression. Here is how a typical report looks. + + + +To talk about "changes in performance", we first have to measure this performance. The most natural measure for a single query is elapsed time. It is susceptible to random variations, so we have to take several measurements and average them in some way. From the application point of view, the most interesting statistic is maximum. We want to guarantee that e.g. an analytical dashboard built on ClickHouse is responsive. However, the query time can grow almost without limit due to random factor such as sudden disk load spikes or network delays, so using the maximum is not practical. The minimum is also interesting — after all, there is a theoretical bound on it. We know that the particular algorithm can run only so fast on the particular hardware, in ideal conditions. But if we only look at the minimum, we are going to miss cases where some runs of the query are slow and some are not (e.g. boundary effects in some cache). So we compromise by measuring the median. It is a robust statistic that is reasonably sensitive to outliers and stable enough against noise. + +After measuring the performance, how do we determine that it has changed? Due to various random and systematic factors, the query time always drifts, so the number always changes, but the question is whether this change is meaningful. If we have an old version of the server, and a new version of the server, are they going to consistently give a different result for this query, or was it just a fluke? To answer this, we have to employ some statistical method. The core idea of these methods is comparing the observed values to some reference distribution, and deciding whether what we observed can plausibly belong to this distribution, or, on the contrary, it cannot, which means that the performance characteristics of the patched server are indeed different. + +Choosing the reference distribution is the starting point. One way to obtain it is to build a mathematical model of the process. This works well for simple things like tossing a coin a fixed number of times. We can analytically deduce that the number of heads we get follows the binomial distribution, and get a confidence interval on this number, given the required [level of significance](https://en.wikipedia.org/wiki/P-value#Definition_and_interpretation). If the observed number of heads doesn't belong to this interval, we can conclude that the coin is biased. However, modeling the query execution from first principles is too complex. The best we can do is to use the hardware capabilities to estimate how fast the query could run, in principle, and try to achieve this throughput. + +For complex processes which resist modeling, a practical option is to use the historical data from the same process. We actually used to do this for ClickHouse. For each tested commit, we measured the run times for each test query and saved them into a database. We could compare the patched server to these reference values, build graphs of changes over time and so on. The main problem with this approach is systematic errors induced by environment. Sometimes the performance testing task ends up on a machine with dying HDD, or they update `atop` to a broken version that slows every kernel call in half, et cetera, ad infinitum. This is why now we employ another approach. + +We run the reference version of the server process and the tested version, simultaneously on the same machine, and run the test queries on each of them in turn, one by one. This way we eliminate most systematic errors, because both servers are equally influenced by them. We can then compare the set of results we got from the reference server process, and the set from the test server process, to see whether they look the same. Comparing the distributions using two samples is a very interesting problem in itself. We use a non-parametric bootstrap method to build a randomization distribution for the observed difference of median query run times. This method is described in detail in [[1]](#ref1), where they apply it to see how changing a fertilizer mixture changes the yield of tomato plants. ClickHouse is not much different from tomatoes, only we have to check how the changes in code influence the performance. + +This method ultimately gives a single threshold number _T_: what is the largest difference in median query run times between old and new server, that we can observe even if nothing has changed. Then we have a simple decision protocol given this threshold _T_ and the measured difference of medians _D_: +1. _abs(D) <= T_ — the changes are not statistically significant, +2. _abs(D) <= 5%_ — the changes are too small to be important, +3. _abs(T) >= 10%_ — the test query has excessive run time variance that leads to poor sensitivity, +4. finally, _abs(D) >= T and abs(D) >= 5%_ — there are statistically significant changes of significant magnitude. + +The most interesting case are the unstable queries _(3)_. When the elapsed time changes significantly between runs even on the same version of server, it means we won't be able to detect the changes of performance, because they are going to be drowned out by the noise. Such queries tend to be the most difficult to debug, because there is no straightforward way to compare "good" and "bad" server. This topic deserves its own article which we will publish next. For now, let's consider the happy path _(4)_. This is the case of real and notable changes in performance that this system is intended to catch. What do we do next? + +# Understanding the Reasons Behind the Changes + +An investigation of code performance often starts with applying a profiler. On Linux, you would use `perf`, a sampling profiler that periodically collects the stack trace of the process, so that you can then see an aggregate picture of where your program spends the most time. In ClickHouse, we actually have a built-in sampling profiler that saves results into a system table, so no external tools are needed. It can be enabled for all queries or for a particular one, by passing the settings [as described in the docs](https://clickhouse.tech/docs/en/operations/optimizing-performance/sampling-query-profiler/). It is on by default, so if you use a recent version of ClickHouse, you already have a combined profile of your production server load. To visualize it, we can use a well-known script for building [flamegraphs](https://github.com/brendangregg/FlameGraph): +``` +clickhouse-client -q "SELECT + arrayStringConcat( + arrayMap( + x -> concat(splitByChar('/', addressToLine(x))[-1], + '#', demangle(addressToSymbol(x))), + trace), + ';') AS stack, + count(*) AS samples +FROM system.trace_log +WHERE trace_type = 'Real' +AND query_id = '4aac5305-b27f-4a5a-91c3-61c0cf52ec2a' +GROUP BY trace" \ +| flamegraph.pl + +``` + +As an example, let's use the test run we've seen above. The tested [pull request](https://github.com/ClickHouse/ClickHouse/pull/26248) is supposed to speed up the `sum` aggregate function for nullable integer types. Let's look at the query #8 of the test 'sum': `SELECT sum(toNullable(number)) FROM numbers(100000000)`. The test system reported that its performance increased by 38.5%, and built a "differential" variant of flamegraph for it, that shows the relative time spent in different functions. We can see that the function that calculates the sum, `DB::AggregateFunctionSumData::addManyNotNull`, now takes 15% less time. + + + +To get more leads into why the performance has changed, we can check how the various query metrics have changed between the old and the new servers. This includes all the metrics from `system.query_log.ProfileEvents`, such as `SelectedRows` or `RealTimeMicroseconds`. ClickHouse also tracks the hardware CPU metrics such as the number of branch or cache misses, using the Linux `perf_event_open` API. After downloading the test output archive, we can use a simple ad hoc [script](https://gist.github.com/akuzm/bb28a442f882349e0a5ec2b5262b97d0) to build some statistics and graphs of these metrics. + + + +This graph shows the number of branch instructions per second, on the old and the new server. We can see that the number of branch instructions has dramatically decreased, which might explain the performance difference. The tested pull request removes some `if`s and replaces them with multiplication, so this explanation sounds plausible. + +While side-to-side comparison is more robust against the systemic errors, the historical data is still very valuable for finding where a regression was introduced or investigating the unstable test queries. This is why we save the results of all test runs into a ClickHouse database. Let's consider the same query #8 from the `sum` test. We can build the history of performance changes with this [SQL query](https://play-ci.clickhouse.tech/play?user=play#V0lUSCAwLjA1IEFTIHMKU0VMRUNUIG9sZF9zaGEsIG5ld19zaGEsIGV2ZW50X3RpbWUsIG1lc3NhZ2UsIG9sZF92YWx1ZSBBUyBgb2xkIHNlcnZlcmAsICAgbmV3X3ZhbHVlIEFTIGBuZXcgc2VydmVyYCwgYmVmb3JlIEFTIGBwcmV2IDExIHJ1bnNgLCBhZnRlciBBUyBgbmV4dCAxMSBydW5zYCwgICAgZGlmZiBBUyBgZGlmZiwgcmF0aW9gLCBzdGF0X3RocmVzaG9sZF9oaXN0b3JpY2FsIEFTIGBzdGF0IHRocmVzaG9sZCwgcmF0aW8sIGhpc3RvcmljYWxgLCBzdGF0X3RocmVzaG9sZCBBUyBgc3RhdCB0aHJlc2hvbGQsIHJhdGlvLCBwZXItcnVuYCwgY3B1X21vZGVsLHF1ZXJ5X2Rpc3BsYXlfbmFtZQpGUk9NIAooU0VMRUNUICosIHJ1bl9hdHRyaWJ1dGVzX3YxLnZhbHVlIEFTIGNwdV9tb2RlbCwKICAgICAgICBtZWRpYW4ob2xkX3ZhbHVlKSBPVkVSIChQQVJUSVRJT04gQlkgcnVuX2F0dHJpYnV0ZXNfdjEudmFsdWUsIHRlc3QsIHF1ZXJ5X2luZGV4LCBxdWVyeV9kaXNwbGF5X25hbWUgT1JERVIgQlkgZXZlbnRfZGF0ZSBBU0MgUk9XUyBCRVRXRUVOIDExIFBSRUNFRElORyBBTkQgQ1VSUkVOVCBST1cpIEFTIGJlZm9yZSwKICAgICAgICBtZWRpYW4obmV3X3ZhbHVlKSBPVkVSIChQQVJUSVRJT04gQlkgcnVuX2F0dHJpYnV0ZXNfdjEudmFsdWUsIHRlc3QsIHF1ZXJ5X2luZGV4LCBxdWVyeV9kaXNwbGF5X25hbWUgT1JERVIgQlkgZXZlbnRfZGF0ZSBBU0MgUk9XUyBCRVRXRUVOIENVUlJFTlQgUk9XIEFORCAxMSBGT0xMT1dJTkcpIEFTIGFmdGVyLAogICAgICAgIHF1YW50aWxlRXhhY3QoMC45NSkoYWJzKGRpZmYpKSBPVkVSIChQQVJUSVRJT04gQlkgcnVuX2F0dHJpYnV0ZXNfdjEudmFsdWUsIHRlc3QsIHF1ZXJ5X2luZGV4LCBxdWVyeV9kaXNwbGF5X25hbWUgT1JERVIgQlkgZXZlbnRfZGF0ZSBBU0MgUk9XUyBCRVRXRUVOIDM3IFBSRUNFRElORyBBTkQgQ1VSUkVOVCBST1cpIEFTIHN0YXRfdGhyZXNob2xkX2hpc3RvcmljYWwKICAgIEZST00gcGVyZnRlc3QucXVlcnlfbWV0cmljc192MgogICAgTEVGVCBKT0lOIHBlcmZ0ZXN0LnJ1bl9hdHRyaWJ1dGVzX3YxIFVTSU5HIChvbGRfc2hhLCBuZXdfc2hhKQogICAgV0hFUkUgKGF0dHJpYnV0ZSA9ICdsc2NwdS1tb2RlbC1uYW1lJykgQU5EIChtZXRyaWMgPSAnY2xpZW50X3RpbWUnKQogICAgICAgIC0tIG9ubHkgZm9yIGNvbW1pdHMgaW4gbWFzdGVyCiAgICAgICAgQU5EIChwcl9udW1iZXIgPSAwKQogICAgICAgIC0tIHNlbGVjdCB0aGUgcXVlcmllcyB3ZSBhcmUgaW50ZXJlc3RlZCBpbgogICAgICAgIEFORCAodGVzdCA9ICdzdW0nKSBBTkQgKHF1ZXJ5X2luZGV4ID0gOCkKKSBBUyB0CkFOWSBMRUZUIEpPSU4gYGdoLWRhdGFgLmNvbW1pdHMgT04gbmV3X3NoYSA9IHNoYQpXSEVSRQogICAgLS0gQ2hlY2sgZm9yIGEgcGVyc2lzdGVudCBhbmQgc2lnbmlmaWNhbnQgY2hhbmdlIGluIHF1ZXJ5IHJ1biB0aW1lLCBpbnRyb2R1Y2VkIGJ5IGEgY29tbWl0OgogICAgLS0gMSkgb24gYSBoaXN0b3JpY2FsIGdyYXBoIG9mIHF1ZXJ5IHJ1biB0aW1lLCB0aGVyZSBpcyBhIHN0ZXAgYmV0d2VlbiB0aGUgYWRqYWNlbnQgY29tbWl0cywKICAgIC0tIHRoYXQgaXMgaGlnaGVyIHRoYW4gdGhlIG5vcm1hbCB2YXJpYW5jZSwKICAgICgoKGFicyhhZnRlciAtIGJlZm9yZSkgLyBpZihhZnRlciA+IGJlZm9yZSwgYWZ0ZXIsIGJlZm9yZSkpIEFTIHN0ZXBfaGVpZ2h0KSA+PSBncmVhdGVzdChzLCBzdGF0X3RocmVzaG9sZF9oaXN0b3JpY2FsKSkKICAgIC0tIDIpIGluIHNpZGUtdG8tc2lkZSBjb21wYXJpc29uIG9mIHRoZXNlIHR3byBjb21taXRzLCB0aGVyZSB3YXMgYSBzdGF0aXN0aWNhbGx5IHNpZ25pZmljYW50IGRpZmZlcmVuY2UKICAgIC0tIHRoYXQgaXMgYWxzbyBoaWdoZXIgdGhhbiB0aGUgbm9ybWFsIHZhcmlhbmNlLAogICAgICAgIEFORCAoYWJzKGRpZmYpID49IGdyZWF0ZXN0KHN0YXRfdGhyZXNob2xkLCBzdGF0X3RocmVzaG9sZF9oaXN0b3JpY2FsLCBzKSkKICAgIC0tIDMpIGZpbmFsbHksIHRoaXMgc2lkZS10by1zaWRlIGRpZmZlcmVuY2UgaXMgb2YgbWFnbml0dWRlIGNvbXBhcmFibGUgdG8gdGhlIHN0ZXAgaW4gaGlzdG9yaWNhbCBncmFwaHMuCiAgICAgICAgQU5EIChhYnMoZGlmZikgPj0gKDAuNyAqIHN0ZXBfaGVpZ2h0KSkKb3JkZXIgYnkgZXZlbnRfdGltZSBkZXNjCmZvcm1hdCBWZXJ0aWNhbAoKCg==) to the live ClickHouse CI database. Open the link and run the query so that you can examine the query and see the result for yourself. There were three significant changes of performance throughout the test history. The most recent is a speedup in PR we started with. The second speedup is related to fully switching to clang 11. Curiously, there is also a small slowdown introduced by a PR that was supposed to speed it up instead. + +# Usability Considerations + +Regardless of how it works inside, a test system must be actually usable as a part of the development process. First and foremost, the false positive rate should be as low as possible. False positives are costly to investigate, and if they happen often, developers perceive the test as generally unreliable and tend to ignore the true positives as well. The test must also provide a concise report that makes it obvious what went wrong. We have not really succeeded in this. This test has many more failure modes than a plain functional test, and worse, some of these failures are quantitative, not binary. Much of the complexity is essential, and we try to alleviate it by providing good documentation and linking to the relevant parts of it right from the report page. Another important thing is that the user must be able to investigate a problematic query post-mortem, without running it again locally. This is why we try to export every metric and every intermediate result we have, in easily-manipulated plain text formats. + +Organizationally, it is hard to prevent devolving into a system that does a lot of busywork to just show a green check without giving any insight. I like to call this process "mining the green check", by analogy to cryptocurrencies. Our previous system did just that. It used increasingly complex heuristics tailored to each test query to prevent false positives, restarted itself many times if the results didn't look good, and so on. Ultimately, it wasted a lot of processing power without giving the real picture of the server performance. If you wanted to be sure that the performance did or did not change, you had to recheck by hand. This sorry state is the result of how the incentives are aligned around development — most of the time, the developers just want to merge their pull requests and not be bothered by some obscure test failures. Writing a good performance test query is also not always simple. Just any other query won't do — it has to give predictable performance, be not too fast and not too slow, actually measure something, and so on. After gathering more precise statistics, we discovered that several hundred of our test queries don't measure anything meaningful, e.g. they give a result that varies by 100% between runs. Another problem is that the performance often changes in statistically significant ways (true positive) with no relevant code changes (due to e.g. random differences in layout of the executable). Given all these difficulties, a working performance test system is bound to add noticeable friction to the development process. Most of the "obvious" ways to remove this friction ultimately boil down to "mining the green check". + +Implementation-wise, our system is peculiar in that it doesn't rely on well-known statistical packages, but instead heavily uses `clickhouse-local`, a tool that turns the ClickHouse SQL query processor into a [command line utility](https://altinity.com/blog/2019/6/11/clickhouse-local-the-power-of-clickhouse-sql-in-a-single-command). Doing all the computations in ClickHouse SQL helped us find bugs and usability problems with `clickhouse-local`. The performance test continues to work in dual purpose as a heavy SQL test, and sometimes catches newly introduced bugs in complex joins and the like. The query profiler is always on in the performance tests, and this finds bugs in our fork of `libunwind`. To run the test queries, we use a third-party [Python driver](https://github.com/mymarilyn/clickhouse-driver). This is the only use of this driver in our CI, and it also helped us find some bugs in native protocol handling. A not so honorable fact is that the scaffolding consists of an unreasonable amount of bash, but this at least served to convince us that running [shellcheck](https://github.com/koalaman/shellcheck) in CI is very helpful. + +This concludes the overview of the ClickHouse performance test system. Stay tuned for the next article where we will discuss the most problematic kind of a performance test failure — the unstable query run time. + +_2021-08-20 [Alexander Kuzmenkov](https://github.com/akuzm). Title photo by [Alexander Tokmakov](https://github.com/tavplubix)_ + +References: + +1. Box, Hunter, Hunter, 2005. Statistics for experimenters, p. 78: A Randomized Design Used in the Comparison of Standard and Modified Fertilizer Mixtures for Tomato Plants. \ No newline at end of file