diff --git a/.clang-tidy b/.clang-tidy index 4dd8b9859c9..7241c372319 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -23,9 +23,12 @@ Checks: '*, -bugprone-implicit-widening-of-multiplication-result, -bugprone-narrowing-conversions, -bugprone-not-null-terminated-result, + -bugprone-reserved-identifier, -bugprone-unchecked-optional-access, -cert-dcl16-c, + -cert-dcl37-c, + -cert-dcl51-cpp, -cert-err58-cpp, -cert-msc32-c, -cert-msc51-cpp, @@ -129,6 +132,7 @@ Checks: '*, -readability-function-cognitive-complexity, -readability-function-size, -readability-identifier-length, + -readability-identifier-naming, -readability-implicit-bool-conversion, -readability-isolate-declaration, -readability-magic-numbers, @@ -158,56 +162,28 @@ Checks: '*, WarningsAsErrors: '*' -# TODO: use dictionary syntax for CheckOptions when minimum clang-tidy level rose to 15 -# some-check.SomeOption: 'some value' -# instead of -# - key: some-check.SomeOption -# value: 'some value' CheckOptions: - - key: readability-identifier-naming.ClassCase - value: CamelCase - - key: readability-identifier-naming.EnumCase - value: CamelCase - - key: readability-identifier-naming.LocalVariableCase - value: lower_case - - key: readability-identifier-naming.StaticConstantCase - value: aNy_CasE - - key: readability-identifier-naming.MemberCase - value: lower_case - - key: readability-identifier-naming.PrivateMemberPrefix - value: '' - - key: readability-identifier-naming.ProtectedMemberPrefix - value: '' - - key: readability-identifier-naming.PublicMemberCase - value: lower_case - - key: readability-identifier-naming.MethodCase - value: camelBack - - key: readability-identifier-naming.PrivateMethodPrefix - value: '' - - key: readability-identifier-naming.ProtectedMethodPrefix - value: '' - - key: readability-identifier-naming.ParameterPackCase - value: lower_case - - key: readability-identifier-naming.StructCase - value: CamelCase - - key: readability-identifier-naming.TemplateTemplateParameterCase - value: CamelCase - - key: readability-identifier-naming.TemplateUsingCase - value: lower_case - - key: readability-identifier-naming.TypeTemplateParameterCase - value: CamelCase - - key: readability-identifier-naming.TypedefCase - value: CamelCase - - key: readability-identifier-naming.UnionCase - value: CamelCase - - key: readability-identifier-naming.UsingCase - value: CamelCase - - key: modernize-loop-convert.UseCxx20ReverseRanges - value: false - - key: performance-move-const-arg.CheckTriviallyCopyableMove - value: false - # Workaround clang-tidy bug: https://github.com/llvm/llvm-project/issues/46097 - - key: readability-identifier-naming.TypeTemplateParameterIgnoredRegexp - value: expr-type - - key: cppcoreguidelines-avoid-do-while.IgnoreMacros - value: true + readability-identifier-naming.ClassCase: CamelCase + readability-identifier-naming.EnumCase: CamelCase + readability-identifier-naming.LocalVariableCase: lower_case + readability-identifier-naming.StaticConstantCase: aNy_CasE + readability-identifier-naming.MemberCase: lower_case + readability-identifier-naming.PrivateMemberPrefix: '' + readability-identifier-naming.ProtectedMemberPrefix: '' + readability-identifier-naming.PublicMemberCase: lower_case + readability-identifier-naming.MethodCase: camelBack + readability-identifier-naming.PrivateMethodPrefix: '' + readability-identifier-naming.ProtectedMethodPrefix: '' + readability-identifier-naming.ParameterPackCase: lower_case + readability-identifier-naming.StructCase: CamelCase + readability-identifier-naming.TemplateTemplateParameterCase: CamelCase + readability-identifier-naming.TemplateUsingCase: lower_case + readability-identifier-naming.TypeTemplateParameterCase: CamelCase + readability-identifier-naming.TypedefCase: CamelCase + readability-identifier-naming.UnionCase: CamelCase + readability-identifier-naming.UsingCase: CamelCase + modernize-loop-convert.UseCxx20ReverseRanges: false + performance-move-const-arg.CheckTriviallyCopyableMove: false + # Workaround clang-tidy bug: https://github.com/llvm/llvm-project/issues/46097 + readability-identifier-naming.TypeTemplateParameterIgnoredRegexp: expr-type + cppcoreguidelines-avoid-do-while.IgnoreMacros: true diff --git a/CMakeLists.txt b/CMakeLists.txt index cbb666b81c3..59b38e7763f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -184,26 +184,12 @@ if (OS_DARWIN) set (ENABLE_CURL_BUILD OFF) endif () -# Ignored if `lld` is used -option(ADD_GDB_INDEX_FOR_GOLD "Add .gdb-index to resulting binaries for gold linker.") - if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE") # Can be lld or ld-lld or lld-13 or /path/to/lld. if (LINKER_NAME MATCHES "lld" AND OS_LINUX) set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gdb-index") set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gdb-index") message (STATUS "Adding .gdb-index via --gdb-index linker option.") - # we use another tool for gdb-index, because gold linker removes section .debug_aranges, which used inside clickhouse stacktraces - # http://sourceware-org.1504.n7.nabble.com/gold-No-debug-aranges-section-when-linking-with-gdb-index-td540965.html#a556932 - elseif (LINKER_NAME MATCHES "gold$" AND ADD_GDB_INDEX_FOR_GOLD) - find_program (GDB_ADD_INDEX_EXE NAMES "gdb-add-index" DOC "Path to gdb-add-index executable") - if (NOT GDB_ADD_INDEX_EXE) - set (USE_GDB_ADD_INDEX 0) - message (WARNING "Cannot add gdb index to binaries, because gold linker is used, but gdb-add-index executable not found.") - else() - set (USE_GDB_ADD_INDEX 1) - message (STATUS "gdb-add-index found: ${GDB_ADD_INDEX_EXE}") - endif() endif () endif() @@ -301,12 +287,12 @@ if (ENABLE_BUILD_PROFILING) endif () endif () -set (CMAKE_CXX_STANDARD 20) -set (CMAKE_CXX_EXTENSIONS ON) # Same as gnu++2a (ON) vs c++2a (OFF): https://cmake.org/cmake/help/latest/prop_tgt/CXX_EXTENSIONS.html +set (CMAKE_CXX_STANDARD 23) +set (CMAKE_CXX_EXTENSIONS OFF) set (CMAKE_CXX_STANDARD_REQUIRED ON) set (CMAKE_C_STANDARD 11) -set (CMAKE_C_EXTENSIONS ON) +set (CMAKE_C_EXTENSIONS ON) # required by most contribs written in C set (CMAKE_C_STANDARD_REQUIRED ON) if (COMPILER_GCC OR COMPILER_CLANG) diff --git a/base/base/CMakeLists.txt b/base/base/CMakeLists.txt index 64785d575c5..8ab3c8a0711 100644 --- a/base/base/CMakeLists.txt +++ b/base/base/CMakeLists.txt @@ -2,6 +2,10 @@ if (USE_CLANG_TIDY) set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") endif () +# TODO: Remove this. We like to compile with C++23 (set by top-level CMakeLists) but Clang crashes with our libcxx +# when instantiated from JSON.cpp. Try again when libcxx(abi) and Clang are upgraded to 16. +set (CMAKE_CXX_STANDARD 20) + set (SRCS argsToConfig.cpp coverage.cpp diff --git a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Extractor.h b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Extractor.h index 82e2f895638..3914f33df76 100644 --- a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Extractor.h +++ b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Extractor.h @@ -466,7 +466,7 @@ namespace Data bool extractManualImpl(std::size_t pos, T & val, SQLSMALLINT cType) { SQLRETURN rc = 0; - T value = (T)0; + T value; resizeLengths(pos); diff --git a/base/poco/Foundation/src/Message.cpp b/base/poco/Foundation/src/Message.cpp index 6e9076579c6..663c96e47a2 100644 --- a/base/poco/Foundation/src/Message.cpp +++ b/base/poco/Foundation/src/Message.cpp @@ -27,8 +27,7 @@ Message::Message(): _tid(0), _file(0), _line(0), - _pMap(0), - _fmt_str(0) + _pMap(0) { init(); } diff --git a/cmake/tools.cmake b/cmake/tools.cmake index 4d4d741cc3a..4e1954f27f7 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -50,15 +50,18 @@ endif () string (REGEX MATCHALL "[0-9]+" COMPILER_VERSION_LIST ${CMAKE_CXX_COMPILER_VERSION}) list (GET COMPILER_VERSION_LIST 0 COMPILER_VERSION_MAJOR) -# Example values: `lld-10`, `gold`. +# Example values: `lld-10` option (LINKER_NAME "Linker name or full path") +if (LINKER_NAME MATCHES "gold") + message (FATAL_ERROR "Linking with gold is unsupported. Please use lld.") +endif () + # s390x doesnt support lld if (NOT ARCH_S390X) if (NOT LINKER_NAME) if (COMPILER_GCC) find_program (LLD_PATH NAMES "ld.lld") - find_program (GOLD_PATH NAMES "ld.gold") elseif (COMPILER_CLANG) # llvm lld is a generic driver. # Invoke ld.lld (Unix), ld64.lld (macOS), lld-link (Windows), wasm-ld (WebAssembly) instead @@ -67,13 +70,11 @@ if (NOT ARCH_S390X) elseif (OS_DARWIN) find_program (LLD_PATH NAMES "ld64.lld-${COMPILER_VERSION_MAJOR}" "ld64.lld") endif () - find_program (GOLD_PATH NAMES "ld.gold" "gold") endif () endif() endif() if ((OS_LINUX OR OS_DARWIN) AND NOT LINKER_NAME) - # prefer lld linker over gold or ld on linux and macos if (LLD_PATH) if (COMPILER_GCC) # GCC driver requires one of supported linker names like "lld". @@ -83,17 +84,6 @@ if ((OS_LINUX OR OS_DARWIN) AND NOT LINKER_NAME) set (LINKER_NAME ${LLD_PATH}) endif () endif () - - if (NOT LINKER_NAME) - if (GOLD_PATH) - message (FATAL_ERROR "Linking with gold is unsupported. Please use lld.") - if (COMPILER_GCC) - set (LINKER_NAME "gold") - else () - set (LINKER_NAME ${GOLD_PATH}) - endif () - endif () - endif () endif () # TODO: allow different linker on != OS_LINUX diff --git a/contrib/grpc-cmake/CMakeLists.txt b/contrib/grpc-cmake/CMakeLists.txt index b1ed7e464b6..b4cf0ad5e66 100644 --- a/contrib/grpc-cmake/CMakeLists.txt +++ b/contrib/grpc-cmake/CMakeLists.txt @@ -48,6 +48,9 @@ set(gRPC_ABSL_PROVIDER "clickhouse" CACHE STRING "" FORCE) # We don't want to build C# extensions. set(gRPC_BUILD_CSHARP_EXT OFF) +# TODO: Remove this. We generally like to compile with C++23 but grpc isn't ready yet. +set (CMAKE_CXX_STANDARD 20) + set(_gRPC_CARES_LIBRARIES ch_contrib::c-ares) set(gRPC_CARES_PROVIDER "clickhouse" CACHE STRING "" FORCE) add_subdirectory("${_gRPC_SOURCE_DIR}" "${_gRPC_BINARY_DIR}") diff --git a/contrib/krb5 b/contrib/krb5 index f8262a1b548..9453aec0d50 160000 --- a/contrib/krb5 +++ b/contrib/krb5 @@ -1 +1 @@ -Subproject commit f8262a1b548eb29d97e059260042036255d07f8d +Subproject commit 9453aec0d50e5aff9b189051611b321b40935d02 diff --git a/contrib/krb5-cmake/CMakeLists.txt b/contrib/krb5-cmake/CMakeLists.txt index ceaa270ad85..93b90c15201 100644 --- a/contrib/krb5-cmake/CMakeLists.txt +++ b/contrib/krb5-cmake/CMakeLists.txt @@ -160,6 +160,8 @@ set(ALL_SRCS # "${KRB5_SOURCE_DIR}/lib/gssapi/spnego/negoex_trace.c" + "${KRB5_SOURCE_DIR}/lib/crypto/builtin/kdf.c" + "${KRB5_SOURCE_DIR}/lib/crypto/builtin/cmac.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/prng.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/enc_dk_cmac.c" # "${KRB5_SOURCE_DIR}/lib/crypto/krb/crc32.c" @@ -183,7 +185,6 @@ set(ALL_SRCS "${KRB5_SOURCE_DIR}/lib/crypto/krb/block_size.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/string_to_key.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/verify_checksum.c" - "${KRB5_SOURCE_DIR}/lib/crypto/krb/crypto_libinit.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/derive.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/random_to_key.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/verify_checksum_iov.c" @@ -217,9 +218,7 @@ set(ALL_SRCS "${KRB5_SOURCE_DIR}/lib/crypto/krb/s2k_rc4.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/valid_cksumtype.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/nfold.c" - "${KRB5_SOURCE_DIR}/lib/crypto/krb/prng_fortuna.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/encrypt_length.c" - "${KRB5_SOURCE_DIR}/lib/crypto/krb/cmac.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/keyblocks.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/prf_rc4.c" "${KRB5_SOURCE_DIR}/lib/crypto/krb/s2k_pbkdf2.c" @@ -228,11 +227,11 @@ set(ALL_SRCS "${KRB5_SOURCE_DIR}/lib/crypto/openssl/enc_provider/rc4.c" "${KRB5_SOURCE_DIR}/lib/crypto/openssl/enc_provider/des3.c" #"${KRB5_SOURCE_DIR}/lib/crypto/openssl/enc_provider/camellia.c" + "${KRB5_SOURCE_DIR}/lib/crypto/openssl/cmac.c" "${KRB5_SOURCE_DIR}/lib/crypto/openssl/sha256.c" "${KRB5_SOURCE_DIR}/lib/crypto/openssl/hmac.c" + "${KRB5_SOURCE_DIR}/lib/crypto/openssl/kdf.c" "${KRB5_SOURCE_DIR}/lib/crypto/openssl/pbkdf2.c" - "${KRB5_SOURCE_DIR}/lib/crypto/openssl/init.c" - "${KRB5_SOURCE_DIR}/lib/crypto/openssl/stubs.c" # "${KRB5_SOURCE_DIR}/lib/crypto/openssl/hash_provider/hash_crc32.c" "${KRB5_SOURCE_DIR}/lib/crypto/openssl/hash_provider/hash_evp.c" "${KRB5_SOURCE_DIR}/lib/crypto/openssl/des/des_keys.c" @@ -312,7 +311,6 @@ set(ALL_SRCS "${KRB5_SOURCE_DIR}/lib/krb5/krb/allow_weak.c" "${KRB5_SOURCE_DIR}/lib/krb5/krb/mk_rep.c" "${KRB5_SOURCE_DIR}/lib/krb5/krb/mk_priv.c" - "${KRB5_SOURCE_DIR}/lib/krb5/krb/s4u_authdata.c" "${KRB5_SOURCE_DIR}/lib/krb5/krb/preauth_otp.c" "${KRB5_SOURCE_DIR}/lib/krb5/krb/init_keyblock.c" "${KRB5_SOURCE_DIR}/lib/krb5/krb/ser_addr.c" @@ -688,6 +686,7 @@ target_include_directories(_krb5 PRIVATE target_compile_definitions(_krb5 PRIVATE KRB5_PRIVATE + CRYPTO_OPENSSL _GSS_STATIC_LINK=1 KRB5_DEPRECATED=1 LOCALEDIR="/usr/local/share/locale" diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index 15f58d6c3a3..314e9c2acfd 100644 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -44,6 +44,8 @@ if [ "$is_tsan_build" -eq "0" ]; then fi export ZOOKEEPER_FAULT_INJECTION=1 +# Initial run without S3 to create system.*_log on local file system to make it +# available for dump via clickhouse-local configure azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log & diff --git a/docker/test/upgrade/run.sh b/docker/test/upgrade/run.sh index f637ea24df3..b9abe5b51fe 100644 --- a/docker/test/upgrade/run.sh +++ b/docker/test/upgrade/run.sh @@ -49,17 +49,19 @@ echo -e "Successfully cloned previous release tests$OK" >> /test_output/test_res echo -e "Successfully downloaded previous release packages$OK" >> /test_output/test_results.tsv # Make upgrade check more funny by forcing Ordinary engine for system database -mkdir /var/lib/clickhouse/metadata +mkdir -p /var/lib/clickhouse/metadata echo "ATTACH DATABASE system ENGINE=Ordinary" > /var/lib/clickhouse/metadata/system.sql # Install previous release packages install_packages previous_release_package_folder -# Start server from previous release -# Let's enable S3 storage by default -export USE_S3_STORAGE_FOR_MERGE_TREE=1 -# Previous version may not be ready for fault injections -export ZOOKEEPER_FAULT_INJECTION=0 +# Initial run without S3 to create system.*_log on local file system to make it +# available for dump via clickhouse-local +configure + +start +stop +mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.initial.log # force_sync=false doesn't work correctly on some older versions sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \ @@ -67,8 +69,6 @@ sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \ > /etc/clickhouse-server/config.d/keeper_port.xml.tmp sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml -configure - # But we still need default disk because some tables loaded only into it sudo cat /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml \ | sed "s|
s3
|
s3
default|" \ @@ -76,6 +76,13 @@ sudo cat /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml \ sudo chown clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml sudo chgrp clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml +# Start server from previous release +# Let's enable S3 storage by default +export USE_S3_STORAGE_FOR_MERGE_TREE=1 +# Previous version may not be ready for fault injections +export ZOOKEEPER_FAULT_INJECTION=0 +configure + start clickhouse-client --query="SELECT 'Server version: ', version()" @@ -185,8 +192,6 @@ tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||: collect_query_and_trace_logs -check_oom_in_dmesg - mv /var/log/clickhouse-server/stderr.log /test_output/ # Write check result into check_status.tsv diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md index 9af6df0c87d..32f323a63d5 100644 --- a/docs/en/interfaces/http.md +++ b/docs/en/interfaces/http.md @@ -309,6 +309,7 @@ The HTTP interface allows passing external data (external temporary tables) for ## Response Buffering {#response-buffering} You can enable response buffering on the server-side. The `buffer_size` and `wait_end_of_query` URL parameters are provided for this purpose. +Also settings `http_response_buffer_size` and `http_wait_end_of_query` can be used. `buffer_size` determines the number of bytes in the result to buffer in the server memory. If a result body is larger than this threshold, the buffer is written to the HTTP channel, and the remaining data is sent directly to the HTTP channel. diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 99daddeeb99..0424c3520e0 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -765,7 +765,7 @@ Default value: `0`. ## concurrent_threads_soft_limit_ratio_to_cores {#concurrent_threads_soft_limit_ratio_to_cores} The maximum number of query processing threads as multiple of number of logical cores. -More details: [concurrent_threads_soft_limit_num](#concurrent-threads-soft-limit-num). +More details: [concurrent_threads_soft_limit_num](#concurrent_threads_soft_limit_num). Possible values: diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 37c6841225b..3c53f4fd0cf 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -966,10 +966,10 @@ This is an expert-level setting, and you shouldn't change it if you're just gett ## max_query_size {#settings-max_query_size} -The maximum part of a query that can be taken to RAM for parsing with the SQL parser. -The INSERT query also contains data for INSERT that is processed by a separate stream parser (that consumes O(1) RAM), which is not included in this restriction. +The maximum number of bytes of a query string parsed by the SQL parser. +Data in the VALUES clause of INSERT queries is processed by a separate stream parser (that consumes O(1) RAM) and not affected by this restriction. -Default value: 256 KiB. +Default value: 262144 (= 256 KiB). ## max_parser_depth {#max_parser_depth} diff --git a/docs/en/operations/storing-data.md b/docs/en/operations/storing-data.md index 3f9a0f67187..e019a3741cf 100644 --- a/docs/en/operations/storing-data.md +++ b/docs/en/operations/storing-data.md @@ -80,7 +80,7 @@ Required parameters: - `type` — `encrypted`. Otherwise the encrypted disk is not created. - `disk` — Type of disk for data storage. -- `key` — The key for encryption and decryption. Type: [Uint64](/docs/en/sql-reference/data-types/int-uint.md). You can use `key_hex` parameter to encrypt in hexadecimal form. +- `key` — The key for encryption and decryption. Type: [Uint64](/docs/en/sql-reference/data-types/int-uint.md). You can use `key_hex` parameter to encode the key in hexadecimal form. You can specify multiple keys using the `id` attribute (see example above). Optional parameters: diff --git a/docs/en/operations/system-tables/marked_dropped_tables.md b/docs/en/operations/system-tables/marked_dropped_tables.md new file mode 100644 index 00000000000..23e969f7624 --- /dev/null +++ b/docs/en/operations/system-tables/marked_dropped_tables.md @@ -0,0 +1,37 @@ +--- +slug: /en/operations/system-tables/marked_dropped_tables +--- +# marked_dropped_tables + +Contains information about tables that drop table has been executed but data cleanup has not been actually performed. + +Columns: + +- `index` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Index in marked_dropped_tables queue. +- `database` ([String](../../sql-reference/data-types/string.md)) — Database. +- `table` ([String](../../sql-reference/data-types/string.md)) — Table name. +- `uuid` ([UUID](../../sql-reference/data-types/uuid.md)) — Table uuid. +- `engine` ([String](../../sql-reference/data-types/string.md)) — Table engine name. +- `metadata_dropped_path` ([String](../../sql-reference/data-types/string.md)) — Path of table's metadata file in metadate_dropped directory. +- `table_dropped_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — The time when the next attempt to remove table's data is scheduled on. Usually it's the table when the table was dropped plus `database_atomic_delay_before_drop_table_sec` + +**Example** + +The following example shows how to get information about marked_dropped_tables. + +``` sql +SELECT * +FROM system.marked_dropped_tables\G +``` + +``` text +Row 1: +────── +index: 0 +database: default +table: test +uuid: 03141bb2-e97a-4d7c-a172-95cc066bb3bd +engine: MergeTree +metadata_dropped_path: /data/ClickHouse/build/programs/data/metadata_dropped/default.test.03141bb2-e97a-4d7c-a172-95cc066bb3bd.sql +table_dropped_time: 2023-03-16 23:43:31 +``` diff --git a/docs/en/operations/utilities/clickhouse-format.md b/docs/en/operations/utilities/clickhouse-format.md index bf2e618b791..101310cc65e 100644 --- a/docs/en/operations/utilities/clickhouse-format.md +++ b/docs/en/operations/utilities/clickhouse-format.md @@ -27,7 +27,7 @@ $ clickhouse-format --query "select number from numbers(10) where number%2 order Result: -```text +```sql SELECT number FROM numbers(10) WHERE number % 2 @@ -54,7 +54,7 @@ $ clickhouse-format -n <<< "SELECT * FROM (SELECT 1 AS x UNION ALL SELECT 1 UNIO Result: -```text +```sql SELECT * FROM ( @@ -75,7 +75,7 @@ $ clickhouse-format --seed Hello --obfuscate <<< "SELECT cost_first_screen BETWE Result: -```text +```sql SELECT treasury_mammoth_hazelnut BETWEEN nutmeg AND span, CASE WHEN chive >= 116 THEN switching ELSE ANYTHING END; ``` @@ -87,7 +87,7 @@ $ clickhouse-format --seed World --obfuscate <<< "SELECT cost_first_screen BETWE Result: -```text +```sql SELECT horse_tape_summer BETWEEN folklore AND moccasins, CASE WHEN intestine >= 116 THEN nonconformist ELSE FORESTRY END; ``` @@ -99,7 +99,7 @@ $ clickhouse-format --backslash <<< "SELECT * FROM (SELECT 1 AS x UNION ALL SELE Result: -```text +```sql SELECT * \ FROM \ ( \ diff --git a/docs/en/sql-reference/functions/tuple-functions.md b/docs/en/sql-reference/functions/tuple-functions.md index a31ec3c41d2..c248499be69 100644 --- a/docs/en/sql-reference/functions/tuple-functions.md +++ b/docs/en/sql-reference/functions/tuple-functions.md @@ -22,15 +22,15 @@ tuple(x, y, …) ## tupleElement A function that allows getting a column from a tuple. -‘N’ is the column index, starting from 1. ‘N’ must be a constant. ‘N’ must be a strict postive integer no greater than the size of the tuple. -There is no cost to execute the function. -The function implements the operator `x.N`. +If the second argument is a number `n`, it is the column index, starting from 1. If the second argument is a string `s`, it represents the name of the element. Besides, we can provide the third optional argument, such that when index out of bounds or element for such name does not exist, the default value returned instead of throw exception. The second and third arguments if provided are always must be constant. There is no cost to execute the function. + +The function implements the operator `x.n` and `x.s`. **Syntax** ``` sql -tupleElement(tuple, n) +tupleElement(tuple, n/s [, default_value]) ``` ## untuple diff --git a/docs/en/sql-reference/statements/alter/comment.md b/docs/en/sql-reference/statements/alter/comment.md index f8742765619..cc49c6abf80 100644 --- a/docs/en/sql-reference/statements/alter/comment.md +++ b/docs/en/sql-reference/statements/alter/comment.md @@ -16,7 +16,7 @@ ALTER TABLE [db].name [ON CLUSTER cluster] MODIFY COMMENT 'Comment' **Examples** -Creating a table with comment (for more information, see the [COMMENT] clause(../../../sql-reference/statements/create/table.md#comment-table)): +Creating a table with comment (for more information, see the [COMMENT](../../../sql-reference/statements/create/table.md#comment-table) clause): ``` sql CREATE TABLE table_with_comment diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index ed35df9b97a..de39d960476 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -393,15 +393,15 @@ These codecs are designed to make compression more effective by using specific f #### DoubleDelta -`DoubleDelta` — Calculates delta of deltas and writes it in compact binary form. Optimal compression rates are achieved for monotonic sequences with a constant stride, such as time series data. Can be used with any fixed-width type. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. Uses 1 extra bit for 32-byte deltas: 5-bit prefixes instead of 4-bit prefixes. For additional information, see Compressing Time Stamps in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). +`DoubleDelta(bytes_size)` — Calculates delta of deltas and writes it in compact binary form. Possible `bytes_size` values: 1, 2, 4, 8, the default value is `sizeof(type)` if equal to 1, 2, 4, or 8. In all other cases, it’s 1. Optimal compression rates are achieved for monotonic sequences with a constant stride, such as time series data. Can be used with any fixed-width type. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. Uses 1 extra bit for 32-bit deltas: 5-bit prefixes instead of 4-bit prefixes. For additional information, see Compressing Time Stamps in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf). #### Gorilla -`Gorilla` — Calculates XOR between current and previous floating point value and writes it in compact binary form. The smaller the difference between consecutive values is, i.e. the slower the values of the series changes, the better the compression rate. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. For additional information, see section 4.1 in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](https://doi.org/10.14778/2824032.2824078). +`Gorilla(bytes_size)` — Calculates XOR between current and previous floating point value and writes it in compact binary form. The smaller the difference between consecutive values is, i.e. the slower the values of the series changes, the better the compression rate. Implements the algorithm used in Gorilla TSDB, extending it to support 64-bit types. Possible `bytes_size` values: 1, 2, 4, 8, the default value is `sizeof(type)` if equal to 1, 2, 4, or 8. In all other cases, it’s 1. For additional information, see section 4.1 in [Gorilla: A Fast, Scalable, In-Memory Time Series Database](https://doi.org/10.14778/2824032.2824078). #### FPC -`FPC` - Repeatedly predicts the next floating point value in the sequence using the better of two predictors, then XORs the actual with the predicted value, and leading-zero compresses the result. Similar to Gorilla, this is efficient when storing a series of floating point values that change slowly. For 64-bit values (double), FPC is faster than Gorilla, for 32-bit values your mileage may vary. For a detailed description of the algorithm see [High Throughput Compression of Double-Precision Floating-Point Data](https://userweb.cs.txstate.edu/~burtscher/papers/dcc07a.pdf). +`FPC(level, float_size)` - Repeatedly predicts the next floating point value in the sequence using the better of two predictors, then XORs the actual with the predicted value, and leading-zero compresses the result. Similar to Gorilla, this is efficient when storing a series of floating point values that change slowly. For 64-bit values (double), FPC is faster than Gorilla, for 32-bit values your mileage may vary. Possible `level` values: 1-28, the default value is 12. Possible `float_size` values: 4, 8, the default value is `sizeof(type)` if type is Float. In all other cases, it’s 4. For a detailed description of the algorithm see [High Throughput Compression of Double-Precision Floating-Point Data](https://userweb.cs.txstate.edu/~burtscher/papers/dcc07a.pdf). #### T64 @@ -473,7 +473,7 @@ ENGINE = MergeTree ORDER BY x; ClickHouse supports temporary tables which have the following characteristics: - Temporary tables disappear when the session ends, including if the connection is lost. -- A temporary table uses the Memory engine only. +- A temporary table uses the Memory table engine when engine is not specified and it may use any table engine except Replicated and `KeeperMap` engines. - The DB can’t be specified for a temporary table. It is created outside of databases. - Impossible to create a temporary table with distributed DDL query on all cluster servers (by using `ON CLUSTER`): this table exists only in the current session. - If a temporary table has the same name as another one and a query specifies the table name without specifying the DB, the temporary table will be used. @@ -487,7 +487,7 @@ CREATE TEMPORARY TABLE [IF NOT EXISTS] table_name name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], ... -) +) [ENGINE = engine] ``` In most cases, temporary tables are not created manually, but when using external data for a query, or for distributed `(GLOBAL) IN`. For more information, see the appropriate sections diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md index 3383ea70a2b..1d9b2c9ea30 100644 --- a/docs/en/sql-reference/statements/grant.md +++ b/docs/en/sql-reference/statements/grant.md @@ -105,7 +105,8 @@ Hierarchy of privileges: - [CREATE](#grant-create) - `CREATE DATABASE` - `CREATE TABLE` - - `CREATE TEMPORARY TABLE` + - `CREATE ARBITRARY TEMPORARY TABLE` + - `CREATE TEMPORARY TABLE` - `CREATE VIEW` - `CREATE DICTIONARY` - `CREATE FUNCTION` @@ -313,7 +314,8 @@ Allows executing [CREATE](../../sql-reference/statements/create/index.md) and [A - `CREATE`. Level: `GROUP` - `CREATE DATABASE`. Level: `DATABASE` - `CREATE TABLE`. Level: `TABLE` - - `CREATE TEMPORARY TABLE`. Level: `GLOBAL` + - `CREATE ARBITRARY TEMPORARY TABLE`. Level: `GLOBAL` + - `CREATE TEMPORARY TABLE`. Level: `GLOBAL` - `CREATE VIEW`. Level: `VIEW` - `CREATE DICTIONARY`. Level: `DICTIONARY` diff --git a/docs/en/sql-reference/statements/insert-into.md b/docs/en/sql-reference/statements/insert-into.md index 03a4ab3453c..f2d590d196b 100644 --- a/docs/en/sql-reference/statements/insert-into.md +++ b/docs/en/sql-reference/statements/insert-into.md @@ -91,6 +91,13 @@ INSERT INTO t FORMAT TabSeparated You can insert data separately from the query by using the command-line client or the HTTP interface. For more information, see the section “[Interfaces](../../interfaces)”. +:::note +If you want to specify `SETTINGS` for `INSERT` query then you have to do it _before_ `FORMAT` clause since everything after `FORMAT format_name` is treated as data. For example: +```sql +INSERT INTO table SETTINGS ... FORMAT format_name data_set +``` +::: + ## Constraints If table has [constraints](../../sql-reference/statements/create/table.md#constraints), their expressions will be checked for each row of inserted data. If any of those constraints is not satisfied — server will raise an exception containing constraint name and expression, the query will be stopped. diff --git a/docs/ru/sql-reference/statements/create/table.md b/docs/ru/sql-reference/statements/create/table.md index 7a930b529ed..64eae49be6c 100644 --- a/docs/ru/sql-reference/statements/create/table.md +++ b/docs/ru/sql-reference/statements/create/table.md @@ -260,8 +260,8 @@ ENGINE = MergeTree() Кодеки шифрования: -- `CODEC('AES-128-GCM-SIV')` — Зашифровывает данные с помощью AES-128 в режиме [RFC 8452](https://tools.ietf.org/html/rfc8452) GCM-SIV. -- `CODEC('AES-256-GCM-SIV')` — Зашифровывает данные с помощью AES-256 в режиме GCM-SIV. +- `CODEC('AES-128-GCM-SIV')` — Зашифровывает данные с помощью AES-128 в режиме [RFC 8452](https://tools.ietf.org/html/rfc8452) GCM-SIV. +- `CODEC('AES-256-GCM-SIV')` — Зашифровывает данные с помощью AES-256 в режиме GCM-SIV. Эти кодеки используют фиксированный одноразовый ключ шифрования. Таким образом, это детерминированное шифрование. Оно совместимо с поддерживающими дедупликацию движками, в частности, [ReplicatedMergeTree](../../../engines/table-engines/mergetree-family/replication.md). Однако у шифрования имеется недостаток: если дважды зашифровать один и тот же блок данных, текст на выходе получится одинаковым, и злоумышленник, у которого есть доступ к диску, заметит эту эквивалентность (при этом доступа к содержимому он не получит). @@ -274,10 +274,10 @@ ENGINE = MergeTree() **Пример** ```sql -CREATE TABLE mytable +CREATE TABLE mytable ( x String Codec(AES_128_GCM_SIV) -) +) ENGINE = MergeTree ORDER BY x; ``` @@ -287,10 +287,10 @@ ENGINE = MergeTree ORDER BY x; **Пример** ```sql -CREATE TABLE mytable +CREATE TABLE mytable ( x String Codec(Delta, LZ4, AES_128_GCM_SIV) -) +) ENGINE = MergeTree ORDER BY x; ``` @@ -299,7 +299,7 @@ ENGINE = MergeTree ORDER BY x; ClickHouse поддерживает временные таблицы со следующими характеристиками: - Временные таблицы исчезают после завершения сессии, в том числе при обрыве соединения. -- Временная таблица использует только модуль памяти. +- Временная таблица использует движок таблиц Memory когда движок не указан и она может использовать любой движок таблиц за исключением движков Replicated и `KeeperMap`. - Невозможно указать базу данных для временной таблицы. Она создается вне баз данных. - Невозможно создать временную таблицу распределённым DDL запросом на всех серверах кластера (с опцией `ON CLUSTER`): такая таблица существует только в рамках существующей сессии. - Если временная таблица имеет то же имя, что и некоторая другая, то, при упоминании в запросе без указания БД, будет использована временная таблица. @@ -313,7 +313,7 @@ CREATE TEMPORARY TABLE [IF NOT EXISTS] table_name name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1], name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2], ... -) +) [ENGINE = engine] ``` В большинстве случаев, временные таблицы создаются не вручную, а при использовании внешних данных для запроса, или при распределённом `(GLOBAL) IN`. Подробнее см. соответствующие разделы diff --git a/docs/ru/sql-reference/statements/grant.md b/docs/ru/sql-reference/statements/grant.md index 7c281634c98..73c63850750 100644 --- a/docs/ru/sql-reference/statements/grant.md +++ b/docs/ru/sql-reference/statements/grant.md @@ -107,7 +107,8 @@ GRANT SELECT(x,y) ON db.table TO john WITH GRANT OPTION - [CREATE](#grant-create) - `CREATE DATABASE` - `CREATE TABLE` - - `CREATE TEMPORARY TABLE` + - `CREATE ARBITRARY TEMPORARY TABLE` + - `CREATE TEMPORARY TABLE` - `CREATE VIEW` - `CREATE DICTIONARY` - `CREATE FUNCTION` @@ -314,7 +315,8 @@ GRANT INSERT(x,y) ON db.table TO john - `CREATE`. Уровень: `GROUP` - `CREATE DATABASE`. Уровень: `DATABASE` - `CREATE TABLE`. Уровень: `TABLE` - - `CREATE TEMPORARY TABLE`. Уровень: `GLOBAL` + - `CREATE ARBITRARY TEMPORARY TABLE`. Уровень: `GLOBAL` + - `CREATE TEMPORARY TABLE`. Уровень: `GLOBAL` - `CREATE VIEW`. Уровень: `VIEW` - `CREATE DICTIONARY`. Уровень: `DICTIONARY` diff --git a/packages/clickhouse-server.service b/packages/clickhouse-server.service index 090461df988..7742d8b278a 100644 --- a/packages/clickhouse-server.service +++ b/packages/clickhouse-server.service @@ -18,7 +18,7 @@ Group=clickhouse Restart=always RestartSec=30 # Since ClickHouse is systemd aware default 1m30sec may not be enough -TimeoutStartSec=infinity +TimeoutStartSec=0 # %p is resolved to the systemd unit name RuntimeDirectory=%p ExecStart=/usr/bin/clickhouse-server --config=/etc/clickhouse-server/config.xml --pid-file=%t/%p/%p.pid diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 5b97daf2998..47017a94cb5 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -400,10 +400,6 @@ endif () add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_BUNDLE}) -if (USE_GDB_ADD_INDEX) - add_custom_command(TARGET clickhouse POST_BUILD COMMAND ${GDB_ADD_INDEX_EXE} clickhouse COMMENT "Adding .gdb-index to clickhouse" VERBATIM) -endif() - if (USE_BINARY_HASH) add_custom_command(TARGET clickhouse POST_BUILD COMMAND ./clickhouse hash-binary > hash && ${OBJCOPY_PATH} --add-section .clickhouse.hash=hash clickhouse COMMENT "Adding section '.clickhouse.hash' to clickhouse binary" VERBATIM) endif() diff --git a/programs/compressor/Compressor.cpp b/programs/compressor/Compressor.cpp index b60138b5692..cc25747702a 100644 --- a/programs/compressor/Compressor.cpp +++ b/programs/compressor/Compressor.cpp @@ -66,6 +66,7 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) using namespace DB; namespace po = boost::program_options; + bool print_stacktrace = false; try { po::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); @@ -84,6 +85,7 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) ("level", po::value(), "compression level for codecs specified via flags") ("none", "use no compression instead of LZ4") ("stat", "print block statistics of compressed data") + ("stacktrace", "print stacktrace of exception") ; po::positional_options_description positional_desc; @@ -107,6 +109,7 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) bool use_deflate_qpl = options.count("deflate_qpl"); bool stat_mode = options.count("stat"); bool use_none = options.count("none"); + print_stacktrace = options.count("stacktrace"); unsigned block_size = options["block-size"].as(); std::vector codecs; if (options.count("codec")) @@ -188,11 +191,12 @@ int mainEntryClickHouseCompressor(int argc, char ** argv) /// Compression CompressedWriteBuffer to(*wb, codec, block_size); copyData(*rb, to); + to.finalize(); } } catch (...) { - std::cerr << getCurrentExceptionMessage(true) << '\n'; + std::cerr << getCurrentExceptionMessage(print_stacktrace) << '\n'; return getCurrentExceptionCode(); } diff --git a/programs/odbc-bridge/CMakeLists.txt b/programs/odbc-bridge/CMakeLists.txt index f649e81c50a..118610e4dcd 100644 --- a/programs/odbc-bridge/CMakeLists.txt +++ b/programs/odbc-bridge/CMakeLists.txt @@ -35,10 +35,6 @@ target_link_libraries(clickhouse-odbc-bridge PRIVATE set_target_properties(clickhouse-odbc-bridge PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) target_compile_options (clickhouse-odbc-bridge PRIVATE -Wno-reserved-id-macro -Wno-keyword-macro) -if (USE_GDB_ADD_INDEX) - add_custom_command(TARGET clickhouse-odbc-bridge POST_BUILD COMMAND ${GDB_ADD_INDEX_EXE} ../clickhouse-odbc-bridge COMMENT "Adding .gdb-index to clickhouse-odbc-bridge" VERBATIM) -endif() - if (SPLIT_DEBUG_SYMBOLS) clickhouse_split_debug_symbols(TARGET clickhouse-odbc-bridge DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${SPLITTED_DEBUG_SYMBOLS_DIR} BINARY_PATH ../clickhouse-odbc-bridge) else() diff --git a/programs/odbc-bridge/ColumnInfoHandler.cpp b/programs/odbc-bridge/ColumnInfoHandler.cpp index 6e93246e59a..147ba43a51d 100644 --- a/programs/odbc-bridge/ColumnInfoHandler.cpp +++ b/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -30,7 +30,7 @@ namespace DB namespace ErrorCodes { - extern const int LOGICAL_ERROR; + extern const int UNKNOWN_TABLE; extern const int BAD_ARGUMENTS; } @@ -180,8 +180,19 @@ void ODBCColumnsInfoHandler::handleRequest(HTTPServerRequest & request, HTTPServ columns.emplace_back(column_name, std::move(column_type)); } + /// Usually this should not happen, since in case of table does not + /// exists, the call should be succeeded. + /// However it is possible sometimes because internally there are two + /// queries in ClickHouse ODBC bridge: + /// - system.tables + /// - system.columns + /// And if between this two queries the table will be removed, them + /// there will be no columns + /// + /// Also sometimes system.columns can return empty result because of + /// the cached value of total tables to scan. if (columns.empty()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Columns definition was not returned"); + throw Exception(ErrorCodes::UNKNOWN_TABLE, "Columns definition was not returned"); WriteBufferFromHTTPServerResponse out( response, diff --git a/src/Access/AccessBackup.cpp b/src/Access/AccessBackup.cpp index 53565e8e5d7..800a54e69b3 100644 --- a/src/Access/AccessBackup.cpp +++ b/src/Access/AccessBackup.cpp @@ -72,12 +72,11 @@ namespace return std::make_shared(buf.str()); } - static AccessEntitiesInBackup fromBackupEntry(const IBackupEntry & backup_entry, const String & file_path) + static AccessEntitiesInBackup fromBackupEntry(std::unique_ptr buf, const String & file_path) { try { AccessEntitiesInBackup res; - std::unique_ptr buf = backup_entry.getReadBuffer(); bool dependencies_found = false; @@ -343,8 +342,8 @@ void AccessRestorerFromBackup::addDataPath(const String & data_path) for (const String & filename : filenames) { String filepath_in_backup = data_path_in_backup_fs / filename; - auto backup_entry = backup->readFile(filepath_in_backup); - auto ab = AccessEntitiesInBackup::fromBackupEntry(*backup_entry, filepath_in_backup); + auto read_buffer_from_backup = backup->readFile(filepath_in_backup); + auto ab = AccessEntitiesInBackup::fromBackupEntry(std::move(read_buffer_from_backup), filepath_in_backup); boost::range::copy(ab.entities, std::back_inserter(entities)); boost::range::copy(ab.dependencies, std::inserter(dependencies, dependencies.end())); diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index f57cc2886e3..c73c0499fbe 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -15,6 +15,7 @@ enum class AccessType /// node_type either specifies access type's level (GLOBAL/DATABASE/TABLE/DICTIONARY/VIEW/COLUMNS), /// or specifies that the access type is a GROUP of other access types; /// parent_group_name is the name of the group containing this access type (or NONE if there is no such group). +/// NOTE A parent group must be declared AFTER all its children. #define APPLY_FOR_ACCESS_TYPES(M) \ M(SHOW_DATABASES, "", DATABASE, SHOW) /* allows to execute SHOW DATABASES, SHOW CREATE DATABASE, USE ; implicitly enabled by any grant on the database */\ @@ -86,8 +87,10 @@ enum class AccessType M(CREATE_VIEW, "", VIEW, CREATE) /* allows to execute {CREATE|ATTACH} VIEW; implicitly enabled by the grant CREATE_TABLE */\ M(CREATE_DICTIONARY, "", DICTIONARY, CREATE) /* allows to execute {CREATE|ATTACH} DICTIONARY */\ - M(CREATE_TEMPORARY_TABLE, "", GLOBAL, CREATE) /* allows to create and manipulate temporary tables; + M(CREATE_TEMPORARY_TABLE, "", GLOBAL, CREATE_ARBITRARY_TEMPORARY_TABLE) /* allows to create and manipulate temporary tables; implicitly enabled by the grant CREATE_TABLE on any table */ \ + M(CREATE_ARBITRARY_TEMPORARY_TABLE, "", GLOBAL, CREATE) /* allows to create and manipulate temporary tables + with arbitrary table engine */\ M(CREATE_FUNCTION, "", GLOBAL, CREATE) /* allows to execute CREATE FUNCTION */ \ M(CREATE_NAMED_COLLECTION, "", GLOBAL, CREATE) /* allows to execute CREATE NAMED COLLECTION */ \ M(CREATE, "", GROUP, ALL) /* allows to execute {CREATE|ATTACH} */ \ diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index fbaacb2263b..cc51183c51f 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -81,6 +81,11 @@ namespace if ((level == 0) && (max_flags_with_children & create_table)) res |= create_temporary_table; + /// CREATE TABLE (on any database/table) => CREATE_ARBITRARY_TEMPORARY_TABLE (global) + static const AccessFlags create_arbitrary_temporary_table = AccessType::CREATE_ARBITRARY_TEMPORARY_TABLE; + if ((level == 0) && (max_flags_with_children & create_table)) + res |= create_arbitrary_temporary_table; + /// ALTER_TTL => ALTER_MATERIALIZE_TTL static const AccessFlags alter_ttl = AccessType::ALTER_TTL; static const AccessFlags alter_materialize_ttl = AccessType::ALTER_MATERIALIZE_TTL; diff --git a/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp b/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp index 20f1713f8c2..0cf5310a3ad 100644 --- a/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp +++ b/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp @@ -32,17 +32,17 @@ enum class GroupByKind GROUPING_SETS }; -class GroupingFunctionResolveVisitor : public InDepthQueryTreeVisitor +class GroupingFunctionResolveVisitor : public InDepthQueryTreeVisitorWithContext { public: GroupingFunctionResolveVisitor(GroupByKind group_by_kind_, QueryTreeNodePtrWithHashMap aggregation_key_to_index_, ColumnNumbersList grouping_sets_keys_indices_, ContextPtr context_) - : group_by_kind(group_by_kind_) + : InDepthQueryTreeVisitorWithContext(std::move(context_)) + , group_by_kind(group_by_kind_) , aggregation_key_to_index(std::move(aggregation_key_to_index_)) , grouping_sets_keys_indexes(std::move(grouping_sets_keys_indices_)) - , context(std::move(context_)) { } @@ -71,7 +71,7 @@ public: FunctionOverloadResolverPtr grouping_function_resolver; bool add_grouping_set_column = false; - bool force_grouping_standard_compatibility = context->getSettingsRef().force_grouping_standard_compatibility; + bool force_grouping_standard_compatibility = getSettings().force_grouping_standard_compatibility; size_t aggregation_keys_size = aggregation_key_to_index.size(); switch (group_by_kind) @@ -132,7 +132,6 @@ private: GroupByKind group_by_kind; QueryTreeNodePtrWithHashMap aggregation_key_to_index; ColumnNumbersList grouping_sets_keys_indexes; - ContextPtr context; }; void resolveGroupingFunctions(QueryTreeNodePtr & query_node, ContextPtr context) @@ -164,12 +163,17 @@ void resolveGroupingFunctions(QueryTreeNodePtr & query_node, ContextPtr context) grouping_sets_used_aggregation_keys_list.emplace_back(); auto & grouping_sets_used_aggregation_keys = grouping_sets_used_aggregation_keys_list.back(); + QueryTreeNodePtrWithHashSet used_keys_in_set; + for (auto & grouping_set_key_node : grouping_set_keys_list_node_typed.getNodes()) { + if (used_keys_in_set.contains(grouping_set_key_node)) + continue; + used_keys_in_set.insert(grouping_set_key_node); + grouping_sets_used_aggregation_keys.push_back(grouping_set_key_node); + if (aggregation_key_to_index.contains(grouping_set_key_node)) continue; - - grouping_sets_used_aggregation_keys.push_back(grouping_set_key_node); aggregation_key_to_index.emplace(grouping_set_key_node, aggregation_node_index); ++aggregation_node_index; } diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 38575965973..f5f577a20ab 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -5727,8 +5727,27 @@ void QueryAnalyzer::resolveInterpolateColumnsNodeList(QueryTreeNodePtr & interpo { auto & interpolate_node_typed = interpolate_node->as(); + auto * column_to_interpolate = interpolate_node_typed.getExpression()->as(); + if (!column_to_interpolate) + throw Exception(ErrorCodes::LOGICAL_ERROR, "INTERPOLATE can work only for indentifiers, but {} is found", + interpolate_node_typed.getExpression()->formatASTForErrorMessage()); + auto column_to_interpolate_name = column_to_interpolate->getIdentifier().getFullName(); + resolveExpressionNode(interpolate_node_typed.getExpression(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); - resolveExpressionNode(interpolate_node_typed.getInterpolateExpression(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); + + bool is_column_constant = interpolate_node_typed.getExpression()->getNodeType() == QueryTreeNodeType::CONSTANT; + + auto & interpolation_to_resolve = interpolate_node_typed.getInterpolateExpression(); + IdentifierResolveScope interpolate_scope(interpolation_to_resolve, &scope /*parent_scope*/); + + auto fake_column_node = std::make_shared(NameAndTypePair(column_to_interpolate_name, interpolate_node_typed.getExpression()->getResultType()), interpolate_node_typed.getExpression()); + if (is_column_constant) + interpolate_scope.expression_argument_name_to_node.emplace(column_to_interpolate_name, fake_column_node); + + resolveExpressionNode(interpolation_to_resolve, interpolate_scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); + + if (is_column_constant) + interpolation_to_resolve = interpolation_to_resolve->cloneAndReplace(fake_column_node, interpolate_node_typed.getExpression()); } } diff --git a/src/Analyzer/ValidationUtils.cpp b/src/Analyzer/ValidationUtils.cpp index d70ed1170fc..58e6f26c03a 100644 --- a/src/Analyzer/ValidationUtils.cpp +++ b/src/Analyzer/ValidationUtils.cpp @@ -56,7 +56,7 @@ public: } if (!found_argument_in_group_by_keys) - throw Exception(ErrorCodes::NOT_AN_AGGREGATE, + throw Exception(ErrorCodes::BAD_ARGUMENTS, "GROUPING function argument {} is not in GROUP BY keys. In query {}", grouping_function_arguments_node->formatASTForErrorMessage(), query_node->formatASTForErrorMessage()); diff --git a/src/Backups/BackupIO.cpp b/src/Backups/BackupIO.cpp index 6ca0c8e7bee..cc252c2f1bd 100644 --- a/src/Backups/BackupIO.cpp +++ b/src/Backups/BackupIO.cpp @@ -1,9 +1,10 @@ #include #include -#include +#include #include + namespace DB { @@ -12,6 +13,15 @@ namespace ErrorCodes extern const int NOT_IMPLEMENTED; } +void IBackupReader::copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings) +{ + auto read_buffer = readFile(file_name); + auto write_buffer = destination_disk->writeFile(destination_path, std::min(size, DBMS_DEFAULT_BUFFER_SIZE), write_mode, write_settings); + copyData(*read_buffer, *write_buffer, size); + write_buffer->finalize(); +} + void IBackupWriter::copyDataToFile(const CreateReadBufferFunction & create_read_buffer, UInt64 offset, UInt64 size, const String & dest_file_name) { auto read_buffer = create_read_buffer(); diff --git a/src/Backups/BackupIO.h b/src/Backups/BackupIO.h index fe2bed6aa1a..cf3d29ee51e 100644 --- a/src/Backups/BackupIO.h +++ b/src/Backups/BackupIO.h @@ -17,6 +17,8 @@ public: virtual bool fileExists(const String & file_name) = 0; virtual UInt64 getFileSize(const String & file_name) = 0; virtual std::unique_ptr readFile(const String & file_name) = 0; + virtual void copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings); virtual DataSourceDescription getDataSourceDescription() const = 0; }; diff --git a/src/Backups/BackupIO_Disk.cpp b/src/Backups/BackupIO_Disk.cpp index 1b7202a5c28..cc6076541d0 100644 --- a/src/Backups/BackupIO_Disk.cpp +++ b/src/Backups/BackupIO_Disk.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -12,7 +13,8 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -BackupReaderDisk::BackupReaderDisk(const DiskPtr & disk_, const String & path_) : disk(disk_), path(path_) +BackupReaderDisk::BackupReaderDisk(const DiskPtr & disk_, const String & path_) + : disk(disk_), path(path_), log(&Poco::Logger::get("BackupReaderDisk")) { } @@ -33,6 +35,21 @@ std::unique_ptr BackupReaderDisk::readFile(const String & fi return disk->readFile(path / file_name); } +void BackupReaderDisk::copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings) +{ + if (write_mode == WriteMode::Rewrite) + { + LOG_TRACE(log, "Copying {}/{} from disk {} to {} by the disk", path, file_name, disk->getName(), destination_disk->getName()); + disk->copyFile(path / file_name, *destination_disk, destination_path, write_settings); + return; + } + + LOG_TRACE(log, "Copying {}/{} from disk {} to {} through buffers", path, file_name, disk->getName(), destination_disk->getName()); + IBackupReader::copyFileToDisk(file_name, size, destination_disk, destination_path, write_mode, write_settings); +} + + BackupWriterDisk::BackupWriterDisk(const DiskPtr & disk_, const String & path_) : disk(disk_), path(path_) { } diff --git a/src/Backups/BackupIO_Disk.h b/src/Backups/BackupIO_Disk.h index 5e5c431da7d..600e4f8ff39 100644 --- a/src/Backups/BackupIO_Disk.h +++ b/src/Backups/BackupIO_Disk.h @@ -17,11 +17,14 @@ public: bool fileExists(const String & file_name) override; UInt64 getFileSize(const String & file_name) override; std::unique_ptr readFile(const String & file_name) override; + void copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings) override; DataSourceDescription getDataSourceDescription() const override; private: DiskPtr disk; std::filesystem::path path; + Poco::Logger * log; }; class BackupWriterDisk : public IBackupWriter diff --git a/src/Backups/BackupIO_File.cpp b/src/Backups/BackupIO_File.cpp index c010cae15de..5bf6d54928d 100644 --- a/src/Backups/BackupIO_File.cpp +++ b/src/Backups/BackupIO_File.cpp @@ -1,15 +1,18 @@ #include +#include #include #include #include #include +#include + namespace fs = std::filesystem; namespace DB { -BackupReaderFile::BackupReaderFile(const String & path_) : path(path_) +BackupReaderFile::BackupReaderFile(const String & path_) : path(path_), log(&Poco::Logger::get("BackupReaderFile")) { } @@ -30,6 +33,22 @@ std::unique_ptr BackupReaderFile::readFile(const String & fi return createReadBufferFromFileBase(path / file_name, {}); } +void BackupReaderFile::copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings) +{ + if (destination_disk->getDataSourceDescription() == getDataSourceDescription()) + { + /// Use more optimal way. + LOG_TRACE(log, "Copying {}/{} to disk {} locally", path, file_name, destination_disk->getName()); + fs::copy(path / file_name, fullPath(destination_disk, destination_path), fs::copy_options::overwrite_existing); + return; + } + + LOG_TRACE(log, "Copying {}/{} to disk {} through buffers", path, file_name, destination_disk->getName()); + IBackupReader::copyFileToDisk(path / file_name, size, destination_disk, destination_path, write_mode, write_settings); +} + + BackupWriterFile::BackupWriterFile(const String & path_) : path(path_) { } diff --git a/src/Backups/BackupIO_File.h b/src/Backups/BackupIO_File.h index 1727323ba4e..e1f4324a39f 100644 --- a/src/Backups/BackupIO_File.h +++ b/src/Backups/BackupIO_File.h @@ -15,10 +15,13 @@ public: bool fileExists(const String & file_name) override; UInt64 getFileSize(const String & file_name) override; std::unique_ptr readFile(const String & file_name) override; + void copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings) override; DataSourceDescription getDataSourceDescription() const override; private: std::filesystem::path path; + Poco::Logger * log; }; class BackupWriterFile : public IBackupWriter diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 2f315e8d488..0a757f94a49 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -2,6 +2,7 @@ #if USE_AWS_S3 #include +#include #include #include #include @@ -96,6 +97,7 @@ BackupReaderS3::BackupReaderS3( , client(makeS3Client(s3_uri_, access_key_id_, secret_access_key_, context_)) , read_settings(context_->getReadSettings()) , request_settings(context_->getStorageS3Settings().getSettings(s3_uri.uri.toString()).request_settings) + , log(&Poco::Logger::get("BackupReaderS3")) { request_settings.max_single_read_retries = context_->getSettingsRef().s3_max_single_read_retries; // FIXME: Avoid taking value for endpoint } @@ -127,6 +129,27 @@ std::unique_ptr BackupReaderS3::readFile(const String & file client, s3_uri.bucket, fs::path(s3_uri.key) / file_name, s3_uri.version_id, request_settings, read_settings); } +void BackupReaderS3::copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings) +{ + LOG_TRACE(log, "Copying {} to disk {}", file_name, destination_disk->getName()); + + copyS3FileToDisk( + client, + s3_uri.bucket, + fs::path(s3_uri.key) / file_name, + s3_uri.version_id, + 0, + size, + destination_disk, + destination_path, + write_mode, + read_settings, + write_settings, + request_settings, + threadPoolCallbackRunner(BackupsIOThreadPool::get(), "BackupReaderS3")); +} + BackupWriterS3::BackupWriterS3( const S3::URI & s3_uri_, const String & access_key_id_, const String & secret_access_key_, const ContextPtr & context_) diff --git a/src/Backups/BackupIO_S3.h b/src/Backups/BackupIO_S3.h index 9c3132c5689..94e61248428 100644 --- a/src/Backups/BackupIO_S3.h +++ b/src/Backups/BackupIO_S3.h @@ -22,6 +22,8 @@ public: bool fileExists(const String & file_name) override; UInt64 getFileSize(const String & file_name) override; std::unique_ptr readFile(const String & file_name) override; + void copyFileToDisk(const String & file_name, size_t size, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings) override; DataSourceDescription getDataSourceDescription() const override; private: @@ -29,6 +31,7 @@ private: std::shared_ptr client; ReadSettings read_settings; S3Settings::RequestSettings request_settings; + Poco::Logger * log; }; diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp index fb8abee814a..a89dc2ff2b3 100644 --- a/src/Backups/BackupImpl.cpp +++ b/src/Backups/BackupImpl.cpp @@ -79,66 +79,6 @@ namespace } -class BackupImpl::BackupEntryFromBackupImpl : public IBackupEntry -{ -public: - BackupEntryFromBackupImpl( - const std::shared_ptr & backup_, - const String & archive_suffix_, - const String & data_file_name_, - UInt64 size_, - const UInt128 checksum_, - BackupEntryPtr base_backup_entry_ = {}) - : backup(backup_), archive_suffix(archive_suffix_), data_file_name(data_file_name_), size(size_), checksum(checksum_), - base_backup_entry(std::move(base_backup_entry_)) - { - } - - std::unique_ptr getReadBuffer() const override - { - std::unique_ptr read_buffer; - if (backup->use_archives) - read_buffer = backup->getArchiveReader(archive_suffix)->readFile(data_file_name); - else - read_buffer = backup->reader->readFile(data_file_name); - if (base_backup_entry) - { - size_t base_size = base_backup_entry->getSize(); - read_buffer = std::make_unique( - base_backup_entry->getReadBuffer(), base_size, std::move(read_buffer), size - base_size); - } - return read_buffer; - } - - UInt64 getSize() const override { return size; } - std::optional getChecksum() const override { return checksum; } - - String getFilePath() const override - { - return data_file_name; - } - - DiskPtr tryGetDiskIfExists() const override - { - return nullptr; - } - - DataSourceDescription getDataSourceDescription() const override - { - return backup->reader->getDataSourceDescription(); - } - - -private: - const std::shared_ptr backup; - const String archive_suffix; - const String data_file_name; - const UInt64 size; - const UInt128 checksum; - BackupEntryPtr base_backup_entry; -}; - - BackupImpl::BackupImpl( const String & backup_name_for_logging_, const ArchiveParams & archive_params_, @@ -645,24 +585,22 @@ SizeAndChecksum BackupImpl::getFileSizeAndChecksum(const String & file_name) con return {info->size, info->checksum}; } -BackupEntryPtr BackupImpl::readFile(const String & file_name) const +std::unique_ptr BackupImpl::readFile(const String & file_name) const { return readFile(getFileSizeAndChecksum(file_name)); } -BackupEntryPtr BackupImpl::readFile(const SizeAndChecksum & size_and_checksum) const +std::unique_ptr BackupImpl::readFile(const SizeAndChecksum & size_and_checksum) const { - std::lock_guard lock{mutex}; if (open_mode != OpenMode::READ) throw Exception(ErrorCodes::LOGICAL_ERROR, "Backup is not opened for reading"); - ++num_read_files; - num_read_bytes += size_and_checksum.first; - - if (!size_and_checksum.first) + if (size_and_checksum.first == 0) { /// Entry's data is empty. - return std::make_unique(nullptr, 0, UInt128{0, 0}); + std::lock_guard lock{mutex}; + ++num_read_files; + return std::make_unique(static_cast(nullptr), 0); } auto info_opt = coordination->getFileInfo(size_and_checksum); @@ -677,45 +615,149 @@ BackupEntryPtr BackupImpl::readFile(const SizeAndChecksum & size_and_checksum) c const auto & info = *info_opt; + std::unique_ptr read_buffer; + std::unique_ptr base_read_buffer; + + if (info.size > info.base_size) + { + /// Make `read_buffer` if there is data for this backup entry in this backup. + if (use_archives) + { + std::shared_ptr archive_reader; + { + std::lock_guard lock{mutex}; + archive_reader = getArchiveReader(info.archive_suffix); + } + read_buffer = archive_reader->readFile(info.data_file_name); + } + else + { + read_buffer = reader->readFile(info.data_file_name); + } + } + + if (info.base_size) + { + /// Make `base_read_buffer` if there is data for this backup entry in the base backup. + if (!base_backup) + { + throw Exception( + ErrorCodes::NO_BASE_BACKUP, + "Backup {}: Entry {} is marked to be read from a base backup, but there is no base backup specified", + backup_name_for_logging, formatSizeAndChecksum(size_and_checksum)); + } + + if (!base_backup->fileExists(std::pair(info.base_size, info.base_checksum))) + { + throw Exception( + ErrorCodes::WRONG_BASE_BACKUP, + "Backup {}: Entry {} is marked to be read from a base backup, but doesn't exist there", + backup_name_for_logging, formatSizeAndChecksum(size_and_checksum)); + } + + base_read_buffer = base_backup->readFile(std::pair{info.base_size, info.base_checksum}); + } + + { + /// Update number of read files. + std::lock_guard lock{mutex}; + ++num_read_files; + num_read_bytes += info.size; + } + if (!info.base_size) { - /// Data goes completely from this backup, the base backup isn't used. - return std::make_unique( - std::static_pointer_cast(shared_from_this()), info.archive_suffix, info.data_file_name, info.size, info.checksum); + /// Data comes completely from this backup, the base backup isn't used. + return read_buffer; } - - if (!base_backup) + else if (info.size == info.base_size) { - throw Exception( - ErrorCodes::NO_BASE_BACKUP, - "Backup {}: Entry {} is marked to be read from a base backup, but there is no base backup specified", - backup_name_for_logging, formatSizeAndChecksum(size_and_checksum)); + /// Data comes completely from the base backup (nothing comes from this backup). + return base_read_buffer; } - - if (!base_backup->fileExists(std::pair(info.base_size, info.base_checksum))) + else { - throw Exception( - ErrorCodes::WRONG_BASE_BACKUP, - "Backup {}: Entry {} is marked to be read from a base backup, but doesn't exist there", - backup_name_for_logging, formatSizeAndChecksum(size_and_checksum)); - } - - auto base_entry = base_backup->readFile(std::pair{info.base_size, info.base_checksum}); - - if (info.size == info.base_size) - { - /// Data goes completely from the base backup (nothing goes from this backup). - return base_entry; - } - - { - /// The beginning of the data goes from the base backup, - /// and the ending goes from this backup. - return std::make_unique( - static_pointer_cast(shared_from_this()), info.archive_suffix, info.data_file_name, info.size, info.checksum, std::move(base_entry)); + /// The beginning of the data comes from the base backup, + /// and the ending comes from this backup. + return std::make_unique( + std::move(base_read_buffer), info.base_size, std::move(read_buffer), info.size - info.base_size); } } +size_t BackupImpl::copyFileToDisk(const String & file_name, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings) const +{ + return copyFileToDisk(getFileSizeAndChecksum(file_name), destination_disk, destination_path, write_mode, write_settings); +} + +size_t BackupImpl::copyFileToDisk(const SizeAndChecksum & size_and_checksum, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings) const +{ + if (open_mode != OpenMode::READ) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Backup is not opened for reading"); + + if (size_and_checksum.first == 0) + { + /// Entry's data is empty. + if (write_mode == WriteMode::Rewrite) + { + /// Just create an empty file. + destination_disk->createFile(destination_path); + } + std::lock_guard lock{mutex}; + ++num_read_files; + return 0; + } + + auto info_opt = coordination->getFileInfo(size_and_checksum); + if (!info_opt) + { + throw Exception( + ErrorCodes::BACKUP_ENTRY_NOT_FOUND, + "Backup {}: Entry {} not found in the backup", + backup_name_for_logging, + formatSizeAndChecksum(size_and_checksum)); + } + + const auto & info = *info_opt; + + bool file_copied = false; + + if (info.size && !info.base_size && !use_archives) + { + /// Data comes completely from this backup. + reader->copyFileToDisk(info.data_file_name, info.size, destination_disk, destination_path, write_mode, write_settings); + file_copied = true; + + } + else if (info.size && (info.size == info.base_size)) + { + /// Data comes completely from the base backup (nothing comes from this backup). + base_backup->copyFileToDisk(std::pair{info.base_size, info.base_checksum}, destination_disk, destination_path, write_mode, write_settings); + file_copied = true; + } + + if (file_copied) + { + /// The file is already copied, but `num_read_files` is not updated yet. + std::lock_guard lock{mutex}; + ++num_read_files; + num_read_bytes += info.size; + } + else + { + /// Use the generic way to copy data. `readFile()` will update `num_read_files`. + auto read_buffer = readFile(size_and_checksum); + auto write_buffer = destination_disk->writeFile(destination_path, std::min(info.size, DBMS_DEFAULT_BUFFER_SIZE), + write_mode, write_settings); + copyData(*read_buffer, *write_buffer, info.size); + write_buffer->finalize(); + } + + return info.size; +} + + namespace { diff --git a/src/Backups/BackupImpl.h b/src/Backups/BackupImpl.h index 4aa300d5021..c33ca7c94ad 100644 --- a/src/Backups/BackupImpl.h +++ b/src/Backups/BackupImpl.h @@ -73,8 +73,12 @@ public: UInt64 getFileSize(const String & file_name) const override; UInt128 getFileChecksum(const String & file_name) const override; SizeAndChecksum getFileSizeAndChecksum(const String & file_name) const override; - BackupEntryPtr readFile(const String & file_name) const override; - BackupEntryPtr readFile(const SizeAndChecksum & size_and_checksum) const override; + std::unique_ptr readFile(const String & file_name) const override; + std::unique_ptr readFile(const SizeAndChecksum & size_and_checksum) const override; + size_t copyFileToDisk(const String & file_name, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings) const override; + size_t copyFileToDisk(const SizeAndChecksum & size_and_checksum, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode, const WriteSettings & write_settings) const override; void writeFile(const String & file_name, BackupEntryPtr entry) override; void finalizeWriting() override; bool supportsWritingInMultipleThreads() const override { return !use_archives; } diff --git a/src/Backups/IBackup.h b/src/Backups/IBackup.h index 208305e3f35..03fab6a25d6 100644 --- a/src/Backups/IBackup.h +++ b/src/Backups/IBackup.h @@ -1,6 +1,8 @@ #pragma once #include +#include +#include #include #include @@ -8,7 +10,10 @@ namespace DB { class IBackupEntry; +class IDisk; using BackupEntryPtr = std::shared_ptr; +using DiskPtr = std::shared_ptr; +class SeekableReadBuffer; /// Represents a backup, i.e. a storage of BackupEntries which can be accessed by their names. /// A backup can be either incremental or non-incremental. An incremental backup doesn't store @@ -95,8 +100,15 @@ public: virtual SizeAndChecksum getFileSizeAndChecksum(const String & file_name) const = 0; /// Reads an entry from the backup. - virtual BackupEntryPtr readFile(const String & file_name) const = 0; - virtual BackupEntryPtr readFile(const SizeAndChecksum & size_and_checksum) const = 0; + virtual std::unique_ptr readFile(const String & file_name) const = 0; + virtual std::unique_ptr readFile(const SizeAndChecksum & size_and_checksum) const = 0; + + /// Copies a file from the backup to a specified destination disk. Returns the number of bytes written. + virtual size_t copyFileToDisk(const String & file_name, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode = WriteMode::Rewrite, const WriteSettings & write_settings = {}) const = 0; + + virtual size_t copyFileToDisk(const SizeAndChecksum & size_and_checksum, DiskPtr destination_disk, const String & destination_path, + WriteMode write_mode = WriteMode::Rewrite, const WriteSettings & write_settings = {}) const = 0; /// Puts a new entry to the backup. virtual void writeFile(const String & file_name, BackupEntryPtr entry) = 0; diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index b19cbaf975c..68a68379f79 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -316,7 +316,7 @@ void RestorerFromBackup::findTableInBackup(const QualifiedTableName & table_name = *root_path_in_use / "data" / escapeForFileName(table_name_in_backup.database) / escapeForFileName(table_name_in_backup.table); } - auto read_buffer = backup->readFile(*metadata_path)->getReadBuffer(); + auto read_buffer = backup->readFile(*metadata_path); String create_query_str; readStringUntilEOF(create_query_str, *read_buffer); read_buffer.reset(); @@ -410,7 +410,7 @@ void RestorerFromBackup::findDatabaseInBackup(const String & database_name_in_ba if (metadata_path) { - auto read_buffer = backup->readFile(*metadata_path)->getReadBuffer(); + auto read_buffer = backup->readFile(*metadata_path); String create_query_str; readStringUntilEOF(create_query_str, *read_buffer); read_buffer.reset(); diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 4f7cf893328..6643a94c3bc 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -216,6 +216,7 @@ void Connection::disconnect() socket->close(); socket = nullptr; connected = false; + nonce.reset(); } @@ -324,6 +325,14 @@ void Connection::receiveHello() password_complexity_rules.push_back({std::move(original_pattern), std::move(exception_message)}); } } + if (server_revision >= DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2) + { + chassert(!nonce.has_value()); + + UInt64 read_nonce; + readIntBinary(read_nonce, *in); + nonce.emplace(read_nonce); + } } else if (packet_type == Protocol::Server::Exception) receiveException()->rethrow(); @@ -584,6 +593,9 @@ void Connection::sendQuery( { #if USE_SSL std::string data(salt); + // For backward compatibility + if (nonce.has_value()) + data += std::to_string(nonce.value()); data += cluster_secret; data += query; data += query_id; @@ -593,8 +605,8 @@ void Connection::sendQuery( std::string hash = encodeSHA256(data); writeStringBinary(hash, *out); #else - throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, - "Inter-server secret support is disabled, because ClickHouse was built without SSL library"); + throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, + "Inter-server secret support is disabled, because ClickHouse was built without SSL library"); #endif } else diff --git a/src/Client/Connection.h b/src/Client/Connection.h index d806c5e8b1f..b86567e2ed0 100644 --- a/src/Client/Connection.h +++ b/src/Client/Connection.h @@ -167,7 +167,10 @@ private: /// For inter-server authorization String cluster; String cluster_secret; + /// For DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET String salt; + /// For DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2 + std::optional nonce; /// Address is resolved during the first connection (or the following reconnects) /// Use it only for logging purposes diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index d696070aa41..5b20d98aa01 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -11,13 +11,21 @@ M(ReplicatedSend, "Number of data parts being sent to replicas") \ M(ReplicatedChecks, "Number of data parts checking for consistency") \ M(BackgroundMergesAndMutationsPoolTask, "Number of active merges and mutations in an associated background pool") \ + M(BackgroundMergesAndMutationsPoolSize, "Limit on number of active merges and mutations in an associated background pool") \ M(BackgroundFetchesPoolTask, "Number of active fetches in an associated background pool") \ + M(BackgroundFetchesPoolSize, "Limit on number of simultaneous fetches in an associated background pool") \ M(BackgroundCommonPoolTask, "Number of active tasks in an associated background pool") \ + M(BackgroundCommonPoolSize, "Limit on number of tasks in an associated background pool") \ M(BackgroundMovePoolTask, "Number of active tasks in BackgroundProcessingPool for moves") \ + M(BackgroundMovePoolSize, "Limit on number of tasks in BackgroundProcessingPool for moves") \ M(BackgroundSchedulePoolTask, "Number of active tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc.") \ + M(BackgroundSchedulePoolSize, "Limit on number of tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc.") \ M(BackgroundBufferFlushSchedulePoolTask, "Number of active tasks in BackgroundBufferFlushSchedulePool. This pool is used for periodic Buffer flushes") \ + M(BackgroundBufferFlushSchedulePoolSize, "Limit on number of tasks in BackgroundBufferFlushSchedulePool") \ M(BackgroundDistributedSchedulePoolTask, "Number of active tasks in BackgroundDistributedSchedulePool. This pool is used for distributed sends that is done in background.") \ + M(BackgroundDistributedSchedulePoolSize, "Limit on number of tasks in BackgroundDistributedSchedulePool") \ M(BackgroundMessageBrokerSchedulePoolTask, "Number of active tasks in BackgroundProcessingPool for message streaming") \ + M(BackgroundMessageBrokerSchedulePoolSize, "Limit on number of tasks in BackgroundProcessingPool for message streaming") \ M(CacheDictionaryUpdateQueueBatches, "Number of 'batches' (a set of keys) in update queue in CacheDictionaries.") \ M(CacheDictionaryUpdateQueueKeys, "Exact number of keys in update queue in CacheDictionaries.") \ M(DiskSpaceReservedForMerge, "Disk space reserved for currently running background merges. It is slightly more than the total size of currently merging parts.") \ diff --git a/src/Common/LRUCachePolicy.h b/src/Common/LRUCachePolicy.h index 3c069eb276b..b6c0ef0d3ef 100644 --- a/src/Common/LRUCachePolicy.h +++ b/src/Common/LRUCachePolicy.h @@ -31,7 +31,7 @@ public: * max_elements_size == 0 means no elements size restrictions. */ explicit LRUCachePolicy(size_t max_size_, size_t max_elements_size_ = 0, OnWeightLossFunction on_weight_loss_function_ = {}) - : max_size(std::max(static_cast(1), max_size_)), max_elements_size(max_elements_size_) + : max_size(std::max(1uz, max_size_)), max_elements_size(max_elements_size_) { Base::on_weight_loss_function = on_weight_loss_function_; } diff --git a/src/Common/RWLock.cpp b/src/Common/RWLock.cpp index c2419d0c1b7..2d0fcfa3e74 100644 --- a/src/Common/RWLock.cpp +++ b/src/Common/RWLock.cpp @@ -97,7 +97,7 @@ private: * Note: "SM" in the commentaries below stands for STATE MODIFICATION */ RWLockImpl::LockHolder -RWLockImpl::getLock(RWLockImpl::Type type, const String & query_id, const std::chrono::milliseconds & lock_timeout_ms) +RWLockImpl::getLock(RWLockImpl::Type type, const String & query_id, const std::chrono::milliseconds & lock_timeout_ms, bool throw_in_fast_path) { const auto lock_deadline_tp = (lock_timeout_ms == std::chrono::milliseconds(0)) @@ -130,11 +130,19 @@ RWLockImpl::getLock(RWLockImpl::Type type, const String & query_id, const std::c if (owner_query_it != owner_queries.end()) { if (wrlock_owner != writers_queue.end()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "RWLockImpl::getLock(): RWLock is already locked in exclusive mode"); + { + if (throw_in_fast_path) + throw Exception(ErrorCodes::LOGICAL_ERROR, "RWLockImpl::getLock(): RWLock is already locked in exclusive mode"); + return nullptr; + } /// Lock upgrading is not supported if (type == Write) - throw Exception(ErrorCodes::LOGICAL_ERROR, "RWLockImpl::getLock(): Cannot acquire exclusive lock while RWLock is already locked"); + { + if (throw_in_fast_path) + throw Exception(ErrorCodes::LOGICAL_ERROR, "RWLockImpl::getLock(): Cannot acquire exclusive lock while RWLock is already locked"); + return nullptr; + } /// N.B. Type is Read here, query_id is not empty and it_query is a valid iterator ++owner_query_it->second; /// SM1: nothrow diff --git a/src/Common/RWLock.h b/src/Common/RWLock.h index cb4cf7f9200..dd965b65026 100644 --- a/src/Common/RWLock.h +++ b/src/Common/RWLock.h @@ -56,7 +56,7 @@ public: /// Empty query_id means the lock is acquired from outside of query context (e.g. in a background thread). LockHolder getLock(Type type, const String & query_id, - const std::chrono::milliseconds & lock_timeout_ms = std::chrono::milliseconds(0)); + const std::chrono::milliseconds & lock_timeout_ms = std::chrono::milliseconds(0), bool throw_in_fast_path = true); /// Use as query_id to acquire a lock outside the query context. inline static const String NO_QUERY = String(); diff --git a/src/Common/ZooKeeper/ZooKeeperLock.cpp b/src/Common/ZooKeeper/ZooKeeperLock.cpp index 1200dcdb533..a52c942a35f 100644 --- a/src/Common/ZooKeeper/ZooKeeperLock.cpp +++ b/src/Common/ZooKeeper/ZooKeeperLock.cpp @@ -41,6 +41,16 @@ ZooKeeperLock::~ZooKeeperLock() } } +bool ZooKeeperLock::isLocked() const +{ + return locked; +} + +const std::string & ZooKeeperLock::getLockPath() const +{ + return lock_path; +} + void ZooKeeperLock::unlock() { if (!locked) diff --git a/src/Common/ZooKeeper/ZooKeeperLock.h b/src/Common/ZooKeeper/ZooKeeperLock.h index f249e69dcc3..755ca1333b8 100644 --- a/src/Common/ZooKeeper/ZooKeeperLock.h +++ b/src/Common/ZooKeeper/ZooKeeperLock.h @@ -37,6 +37,8 @@ public: void unlock(); bool tryLock(); + bool isLocked() const; + const std::string & getLockPath() const; private: zkutil::ZooKeeperPtr zookeeper; diff --git a/src/Compression/CompressionCodecDelta.cpp b/src/Compression/CompressionCodecDelta.cpp index 6d6078b9ee1..37f9230da14 100644 --- a/src/Compression/CompressionCodecDelta.cpp +++ b/src/Compression/CompressionCodecDelta.cpp @@ -193,7 +193,8 @@ void registerCodecDelta(CompressionCodecFactory & factory) UInt8 method_code = static_cast(CompressionMethodByte::Delta); auto codec_builder = [&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr { - UInt8 delta_bytes_size = 0; + /// Default bytes size is 1. + UInt8 delta_bytes_size = 1; if (arguments && !arguments->children.empty()) { @@ -202,8 +203,8 @@ void registerCodecDelta(CompressionCodecFactory & factory) const auto children = arguments->children; const auto * literal = children[0]->as(); - if (!literal) - throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "Delta codec argument must be integer"); + if (!literal || literal->value.getType() != Field::Types::Which::UInt64) + throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "Delta codec argument must be unsigned integer"); size_t user_bytes_size = literal->value.safeGet(); if (user_bytes_size != 1 && user_bytes_size != 2 && user_bytes_size != 4 && user_bytes_size != 8) diff --git a/src/Compression/CompressionCodecDoubleDelta.cpp b/src/Compression/CompressionCodecDoubleDelta.cpp index 782675dfd32..dea15f99a5a 100644 --- a/src/Compression/CompressionCodecDoubleDelta.cpp +++ b/src/Compression/CompressionCodecDoubleDelta.cpp @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include @@ -31,7 +31,7 @@ namespace DB /** DoubleDelta column codec implementation. * * Based on Gorilla paper: http://www.vldb.org/pvldb/vol8/p1816-teller.pdf, which was extended - * to support 64bit types. The drawback is 1 extra bit for 32-byte wide deltas: 5-bit prefix + * to support 64bit types. The drawback is 1 extra bit for 32-bit wide deltas: 5-bit prefix * instead of 4-bit prefix. * * This codec is best used against monotonic integer sequences with constant (or almost constant) @@ -145,6 +145,8 @@ namespace ErrorCodes extern const int CANNOT_COMPRESS; extern const int CANNOT_DECOMPRESS; extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; + extern const int ILLEGAL_CODEC_PARAMETER; } namespace @@ -549,10 +551,28 @@ void registerCodecDoubleDelta(CompressionCodecFactory & factory) factory.registerCompressionCodecWithType("DoubleDelta", method_code, [&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr { - if (arguments) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Codec DoubleDelta does not accept any arguments"); + /// Default bytes size is 1. + UInt8 data_bytes_size = 1; + if (arguments && !arguments->children.empty()) + { + if (arguments->children.size() > 1) + throw Exception(ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE, "DoubleDelta codec must have 1 parameter, given {}", arguments->children.size()); + + const auto children = arguments->children; + const auto * literal = children[0]->as(); + if (!literal || literal->value.getType() != Field::Types::Which::UInt64) + throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "DoubleDelta codec argument must be unsigned integer"); + + size_t user_bytes_size = literal->value.safeGet(); + if (user_bytes_size != 1 && user_bytes_size != 2 && user_bytes_size != 4 && user_bytes_size != 8) + throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "Argument value for DoubleDelta codec can be 1, 2, 4 or 8, given {}", user_bytes_size); + data_bytes_size = static_cast(user_bytes_size); + } + else if (column_type) + { + data_bytes_size = getDataBytesSize(column_type); + } - UInt8 data_bytes_size = column_type ? getDataBytesSize(column_type) : 0; return std::make_shared(data_bytes_size); }); } diff --git a/src/Compression/CompressionCodecFPC.cpp b/src/Compression/CompressionCodecFPC.cpp index 31b12b762c8..8c3e518ed62 100644 --- a/src/Compression/CompressionCodecFPC.cpp +++ b/src/Compression/CompressionCodecFPC.cpp @@ -109,28 +109,42 @@ void registerCodecFPC(CompressionCodecFactory & factory) auto method_code = static_cast(CompressionMethodByte::FPC); auto codec_builder = [&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr { - UInt8 float_width = 0; + /// Set default float width to 4. + UInt8 float_width = 4; if (column_type != nullptr) float_width = getFloatBytesSize(*column_type); UInt8 level = CompressionCodecFPC::DEFAULT_COMPRESSION_LEVEL; if (arguments && !arguments->children.empty()) { - if (arguments->children.size() > 1) + if (arguments->children.size() > 2) { throw Exception(ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE, - "FPC codec must have 1 parameter, given {}", arguments->children.size()); + "FPC codec must have from 0 to 2 parameters, given {}", arguments->children.size()); } const auto * literal = arguments->children.front()->as(); - if (!literal) - throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "FPC codec argument must be integer"); + if (!literal || literal->value.getType() != Field::Types::Which::UInt64) + throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "FPC codec argument must be unsigned integer"); level = literal->value.safeGet(); if (level < 1 || level > CompressionCodecFPC::MAX_COMPRESSION_LEVEL) throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "FPC codec level must be between {} and {}", 1, static_cast(CompressionCodecFPC::MAX_COMPRESSION_LEVEL)); + + if (arguments->children.size() == 2) + { + literal = arguments->children[1]->as(); + if (!literal || !isInt64OrUInt64FieldType(literal->value.getType())) + throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "FPC codec argument must be unsigned integer"); + + size_t user_float_width = literal->value.safeGet(); + if (user_float_width != 4 && user_float_width != 8) + throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "Float size for FPC codec can be 4 or 8, given {}", user_float_width); + float_width = static_cast(user_float_width); + } } + return std::make_shared(float_width, level); }; factory.registerCompressionCodecWithType("FPC", method_code, codec_builder); diff --git a/src/Compression/CompressionCodecGorilla.cpp b/src/Compression/CompressionCodecGorilla.cpp index d68648bd83c..568640153ac 100644 --- a/src/Compression/CompressionCodecGorilla.cpp +++ b/src/Compression/CompressionCodecGorilla.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -134,6 +135,8 @@ namespace ErrorCodes extern const int CANNOT_COMPRESS; extern const int CANNOT_DECOMPRESS; extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE; + extern const int ILLEGAL_CODEC_PARAMETER; } namespace @@ -445,10 +448,28 @@ void registerCodecGorilla(CompressionCodecFactory & factory) UInt8 method_code = static_cast(CompressionMethodByte::Gorilla); auto codec_builder = [&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr { - if (arguments) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Codec Gorilla does not accept any arguments"); + /// Default bytes size is 1 + UInt8 data_bytes_size = 1; + if (arguments && !arguments->children.empty()) + { + if (arguments->children.size() > 1) + throw Exception(ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE, "Gorilla codec must have 1 parameter, given {}", arguments->children.size()); + + const auto children = arguments->children; + const auto * literal = children[0]->as(); + if (!literal || literal->value.getType() != Field::Types::Which::UInt64) + throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "Gorilla codec argument must be unsigned integer"); + + size_t user_bytes_size = literal->value.safeGet(); + if (user_bytes_size != 1 && user_bytes_size != 2 && user_bytes_size != 4 && user_bytes_size != 8) + throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "Argument value for Gorilla codec can be 1, 2, 4 or 8, given {}", user_bytes_size); + data_bytes_size = static_cast(user_bytes_size); + } + else if (column_type) + { + data_bytes_size = getDataBytesSize(column_type); + } - UInt8 data_bytes_size = column_type ? getDataBytesSize(column_type) : 0; return std::make_shared(data_bytes_size); }; factory.registerCompressionCodecWithType("Gorilla", method_code, codec_builder); diff --git a/src/Compression/CompressionCodecT64.cpp b/src/Compression/CompressionCodecT64.cpp index e7f1615128a..1f8331c8a5f 100644 --- a/src/Compression/CompressionCodecT64.cpp +++ b/src/Compression/CompressionCodecT64.cpp @@ -33,7 +33,8 @@ public: Bit }; - CompressionCodecT64(TypeIndex type_idx_, Variant variant_); + // type_idx_ is required for compression, but not for decompression. + CompressionCodecT64(std::optional type_idx_, Variant variant_); uint8_t getMethodByte() const override; @@ -53,7 +54,7 @@ protected: bool isGenericCompression() const override { return false; } private: - TypeIndex type_idx; + std::optional type_idx; Variant variant; }; @@ -91,9 +92,12 @@ enum class MagicNumber : uint8_t IPv4 = 21, }; -MagicNumber serializeTypeId(TypeIndex type_id) +MagicNumber serializeTypeId(std::optional type_id) { - switch (type_id) + if (!type_id) + throw Exception(ErrorCodes::CANNOT_COMPRESS, "T64 codec doesn't support compression without information about column type"); + + switch (*type_id) { case TypeIndex::UInt8: return MagicNumber::UInt8; case TypeIndex::UInt16: return MagicNumber::UInt16; @@ -115,7 +119,7 @@ MagicNumber serializeTypeId(TypeIndex type_id) break; } - throw Exception(ErrorCodes::LOGICAL_ERROR, "Type is not supported by T64 codec: {}", static_cast(type_id)); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Type is not supported by T64 codec: {}", static_cast(*type_id)); } TypeIndex deserializeTypeId(uint8_t serialized_type_id) @@ -632,7 +636,7 @@ UInt32 CompressionCodecT64::doCompressData(const char * src, UInt32 src_size, ch memcpy(dst, &cookie, 1); dst += 1; - switch (baseType(type_idx)) + switch (baseType(*type_idx)) { case TypeIndex::Int8: return 1 + compressData(src, src_size, dst, variant); @@ -699,7 +703,7 @@ uint8_t CompressionCodecT64::getMethodByte() const return codecId(); } -CompressionCodecT64::CompressionCodecT64(TypeIndex type_idx_, Variant variant_) +CompressionCodecT64::CompressionCodecT64(std::optional type_idx_, Variant variant_) : type_idx(type_idx_) , variant(variant_) { @@ -712,7 +716,7 @@ CompressionCodecT64::CompressionCodecT64(TypeIndex type_idx_, Variant variant_) void CompressionCodecT64::updateHash(SipHash & hash) const { getCodecDesc()->updateTreeHash(hash); - hash.update(type_idx); + hash.update(type_idx.value_or(TypeIndex::Nothing)); hash.update(variant); } @@ -742,9 +746,14 @@ void registerCodecT64(CompressionCodecFactory & factory) throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "Wrong modification for T64: {}", name); } - auto type_idx = typeIdx(type); - if (type && type_idx == TypeIndex::Nothing) - throw Exception(ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE, "T64 codec is not supported for specified type {}", type->getName()); + std::optional type_idx; + if (type) + { + type_idx = typeIdx(type); + if (type_idx == TypeIndex::Nothing) + throw Exception( + ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE, "T64 codec is not supported for specified type {}", type->getName()); + } return std::make_shared(type_idx, variant); }; diff --git a/src/Core/BackgroundSchedulePool.cpp b/src/Core/BackgroundSchedulePool.cpp index 993cfb6ef04..5384ee7f961 100644 --- a/src/Core/BackgroundSchedulePool.cpp +++ b/src/Core/BackgroundSchedulePool.cpp @@ -149,8 +149,9 @@ Coordination::WatchCallback BackgroundSchedulePoolTaskInfo::getWatchCallback() } -BackgroundSchedulePool::BackgroundSchedulePool(size_t size_, CurrentMetrics::Metric tasks_metric_, const char *thread_name_) +BackgroundSchedulePool::BackgroundSchedulePool(size_t size_, CurrentMetrics::Metric tasks_metric_, CurrentMetrics::Metric size_metric_, const char *thread_name_) : tasks_metric(tasks_metric_) + , size_metric(size_metric_, size_) , thread_name(thread_name_) { LOG_INFO(&Poco::Logger::get("BackgroundSchedulePool/" + thread_name), "Create BackgroundSchedulePool with {} threads", size_); @@ -177,6 +178,8 @@ void BackgroundSchedulePool::increaseThreadsCount(size_t new_threads_count) threads.resize(new_threads_count); for (size_t i = old_threads_count; i < new_threads_count; ++i) threads[i] = ThreadFromGlobalPoolNoTracingContextPropagation([this] { threadFunction(); }); + + size_metric.changeTo(new_threads_count); } diff --git a/src/Core/BackgroundSchedulePool.h b/src/Core/BackgroundSchedulePool.h index 0fb70b1f715..ef6fbfa68e9 100644 --- a/src/Core/BackgroundSchedulePool.h +++ b/src/Core/BackgroundSchedulePool.h @@ -54,7 +54,7 @@ public: void increaseThreadsCount(size_t new_threads_count); /// thread_name_ cannot be longer then 13 bytes (2 bytes is reserved for "/D" suffix for delayExecutionThreadFunction()) - BackgroundSchedulePool(size_t size_, CurrentMetrics::Metric tasks_metric_, const char *thread_name_); + BackgroundSchedulePool(size_t size_, CurrentMetrics::Metric tasks_metric_, CurrentMetrics::Metric size_metric_, const char *thread_name_); ~BackgroundSchedulePool(); private: @@ -91,6 +91,7 @@ private: DelayedTasks delayed_tasks; CurrentMetrics::Metric tasks_metric; + CurrentMetrics::Increment size_metric; std::string thread_name; }; diff --git a/src/Core/ProtocolDefines.h b/src/Core/ProtocolDefines.h index 3bbfb95f020..f687145668a 100644 --- a/src/Core/ProtocolDefines.h +++ b/src/Core/ProtocolDefines.h @@ -35,7 +35,6 @@ #define DBMS_MERGE_TREE_PART_INFO_VERSION 1 -/// Minimum revision supporting interserver secret. #define DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET 54441 #define DBMS_MIN_REVISION_WITH_X_FORWARDED_FOR_IN_CLIENT_INFO 54443 @@ -54,7 +53,7 @@ /// NOTE: DBMS_TCP_PROTOCOL_VERSION has nothing common with VERSION_REVISION, /// later is just a number for server version (one number instead of commit SHA) /// for simplicity (sometimes it may be more convenient in some use cases). -#define DBMS_TCP_PROTOCOL_VERSION 54461 +#define DBMS_TCP_PROTOCOL_VERSION 54462 #define DBMS_MIN_PROTOCOL_VERSION_WITH_INITIAL_QUERY_START_TIME 54449 @@ -72,3 +71,5 @@ #define DBMS_MIN_PROTOCOL_VERSION_WITH_SERVER_QUERY_TIME_IN_PROGRESS 54460 #define DBMS_MIN_PROTOCOL_VERSION_WITH_PASSWORD_COMPLEXITY_RULES 54461 + +#define DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2 54462 diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 2ebf50e766e..755e52adb06 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -50,7 +50,7 @@ class IColumn; M(UInt64, max_download_buffer_size, 10*1024*1024, "The maximal size of buffer for parallel downloading (e.g. for URL engine) per each thread.", 0) \ M(UInt64, max_read_buffer_size, DBMS_DEFAULT_BUFFER_SIZE, "The maximum size of the buffer to read from the filesystem.", 0) \ M(UInt64, max_distributed_connections, 1024, "The maximum number of connections for distributed processing of one query (should be greater than max_threads).", 0) \ - M(UInt64, max_query_size, DBMS_DEFAULT_MAX_QUERY_SIZE, "Which part of the query can be read into RAM for parsing (the remaining data for INSERT, if any, is read later)", 0) \ + M(UInt64, max_query_size, DBMS_DEFAULT_MAX_QUERY_SIZE, "The maximum number of bytes of a query string parsed by the SQL parser. Data in the VALUES clause of INSERT queries is processed by a separate stream parser (that consumes O(1) RAM) and not affected by this restriction.", 0) \ M(UInt64, interactive_delay, 100000, "The interval in microseconds to check if the request is cancelled, and to send progress info.", 0) \ M(Seconds, connect_timeout, DBMS_DEFAULT_CONNECT_TIMEOUT_SEC, "Connection timeout if there are no replicas.", 0) \ M(Milliseconds, connect_timeout_with_failover_ms, 50, "Connection timeout for selecting first healthy replica.", 0) \ @@ -253,6 +253,8 @@ class IColumn; M(Bool, send_progress_in_http_headers, false, "Send progress notifications using X-ClickHouse-Progress headers. Some clients do not support high amount of HTTP headers (Python requests in particular), so it is disabled by default.", 0) \ \ M(UInt64, http_headers_progress_interval_ms, 100, "Do not send HTTP headers X-ClickHouse-Progress more frequently than at each specified interval.", 0) \ + M(Bool, http_wait_end_of_query, false, "Enable HTTP response buffering on the server-side.", 0) \ + M(UInt64, http_response_buffer_size, false, "The number of bytes to buffer in the server memory before sending a HTTP response to the client or flushing to disk (when http_wait_end_of_query is enabled).", 0) \ \ M(Bool, fsync_metadata, true, "Do fsync after changing metadata for tables and databases (.sql files). Could be disabled in case of poor latency on server with high load of DDL queries and high load of disk subsystem.", 0) \ \ @@ -716,6 +718,7 @@ class IColumn; M(Float, insert_keeper_fault_injection_probability, 0.0f, "Approximate probability of failure for a keeper request during insert. Valid value is in interval [0.0f, 1.0f]", 0) \ M(UInt64, insert_keeper_fault_injection_seed, 0, "0 - random seed, otherwise the setting value", 0) \ M(Bool, force_aggregation_in_order, false, "Force use of aggregation in order on remote nodes during distributed aggregation. PLEASE, NEVER CHANGE THIS SETTING VALUE MANUALLY!", IMPORTANT) \ + M(UInt64, http_max_request_param_data_size, 10_MiB, "Limit on size of request data used as a query parameter in predefined HTTP requests.", 0) \ // End of COMMON_SETTINGS // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS and move obsolete settings to OBSOLETE_SETTINGS. diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index fda0bbe8032..3ede69d5362 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -26,7 +26,12 @@ namespace ErrorCodes DatabaseMemory::DatabaseMemory(const String & name_, ContextPtr context_) : DatabaseWithOwnTablesBase(name_, "DatabaseMemory(" + name_ + ")", context_) , data_path("data/" + escapeForFileName(database_name) + "/") -{} +{ + /// Temporary database should not have any data on the moment of its creation + /// In case of sudden server shutdown remove database folder of temporary database + if (name_ == DatabaseCatalog::TEMPORARY_DATABASE) + removeDataPath(context_); +} void DatabaseMemory::createTable( ContextPtr /*context*/, @@ -71,8 +76,7 @@ void DatabaseMemory::dropTable( if (table->storesDataOnDisk()) { - assert(getDatabaseName() != DatabaseCatalog::TEMPORARY_DATABASE); - fs::path table_data_dir{getTableDataPath(table_name)}; + fs::path table_data_dir{fs::path{getContext()->getPath()} / getTableDataPath(table_name)}; if (fs::exists(table_data_dir)) fs::remove_all(table_data_dir); } @@ -80,7 +84,6 @@ void DatabaseMemory::dropTable( catch (...) { std::lock_guard lock{mutex}; - assert(database_name != DatabaseCatalog::TEMPORARY_DATABASE); attachTableUnlocked(table_name, table); throw; } @@ -129,10 +132,15 @@ UUID DatabaseMemory::tryGetTableUUID(const String & table_name) const return UUIDHelpers::Nil; } +void DatabaseMemory::removeDataPath(ContextPtr local_context) +{ + std::filesystem::remove_all(local_context->getPath() + data_path); +} + void DatabaseMemory::drop(ContextPtr local_context) { /// Remove data on explicit DROP DATABASE - std::filesystem::remove_all(local_context->getPath() + data_path); + removeDataPath(local_context); } void DatabaseMemory::alterTable(ContextPtr local_context, const StorageID & table_id, const StorageInMemoryMetadata & metadata) diff --git a/src/Databases/DatabaseMemory.h b/src/Databases/DatabaseMemory.h index 6262543b0c1..0f703a0b46e 100644 --- a/src/Databases/DatabaseMemory.h +++ b/src/Databases/DatabaseMemory.h @@ -53,6 +53,8 @@ public: std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const override; private: + void removeDataPath(ContextPtr local_context); + const String data_path; using NameToASTCreate = std::unordered_map; NameToASTCreate create_queries TSA_GUARDED_BY(mutex); diff --git a/src/Disks/FakeDiskTransaction.h b/src/Disks/FakeDiskTransaction.h index 46be885739e..5dae17041e1 100644 --- a/src/Disks/FakeDiskTransaction.h +++ b/src/Disks/FakeDiskTransaction.h @@ -68,6 +68,15 @@ public: return disk.writeFile(path, buf_size, mode, settings); } + void writeFileUsingCustomWriteObject( + const String & path, + WriteMode mode, + std::function & object_attributes)> + custom_write_object_function) override + { + disk.writeFileUsingCustomWriteObject(path, mode, std::move(custom_write_object_function)); + } + void removeFile(const std::string & path) override { disk.removeFile(path); diff --git a/src/Disks/IDisk.cpp b/src/Disks/IDisk.cpp index 2a60f32929c..4969cc7c700 100644 --- a/src/Disks/IDisk.cpp +++ b/src/Disks/IDisk.cpp @@ -38,6 +38,15 @@ void IDisk::copyFile(const String & from_file_path, IDisk & to_disk, const Strin out->finalize(); } +void IDisk::writeFileUsingCustomWriteObject( + const String &, WriteMode, std::function &)>) +{ + throw Exception( + ErrorCodes::NOT_IMPLEMENTED, + "Method `writeFileUsingCustomWriteObject()` is not implemented for disk: {}", + getDataSourceDescription().type); +} + DiskTransactionPtr IDisk::createTransaction() { diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 79b2fefe964..ea117c0e0c6 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -209,6 +209,15 @@ public: WriteMode mode = WriteMode::Rewrite, const WriteSettings & settings = {}) = 0; + /// Write a file using a custom function to write an object to the disk's object storage. + /// This method is alternative to writeFile(), the difference is that writeFile() calls IObjectStorage::writeObject() + /// to write an object to the object storage while this method allows to specify a callback for that. + virtual void writeFileUsingCustomWriteObject( + const String & path, + WriteMode mode, + std::function & object_attributes)> + custom_write_object_function); + /// Remove file. Throws exception if file doesn't exists or it's a directory. /// Return whether file was finally removed. (For remote disks it is not always removed). virtual void removeFile(const String & path) = 0; diff --git a/src/Disks/IDiskTransaction.h b/src/Disks/IDiskTransaction.h index 02c8731428d..2edbe858c06 100644 --- a/src/Disks/IDiskTransaction.h +++ b/src/Disks/IDiskTransaction.h @@ -68,6 +68,13 @@ public: const WriteSettings & settings = {}, bool autocommit = true) = 0; + /// Write a file using a custom function to write an object to the disk's object storage. + virtual void writeFileUsingCustomWriteObject( + const String & path, + WriteMode mode, + std::function & object_attributes)> + custom_write_object_function) = 0; + /// Remove file. Throws exception if file doesn't exists or it's a directory. virtual void removeFile(const std::string & path) = 0; diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.cpp b/src/Disks/ObjectStorages/DiskObjectStorage.cpp index d55b1c91c07..44cb80558af 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorage.cpp @@ -577,6 +577,17 @@ std::unique_ptr DiskObjectStorage::writeFile( return result; } +void DiskObjectStorage::writeFileUsingCustomWriteObject( + const String & path, + WriteMode mode, + std::function & object_attributes)> + custom_write_object_function) +{ + LOG_TEST(log, "Write file: {}", path); + auto transaction = createObjectStorageTransaction(); + return transaction->writeFileUsingCustomWriteObject(path, mode, std::move(custom_write_object_function)); +} + void DiskObjectStorage::applyNewSettings( const Poco::Util::AbstractConfiguration & config, ContextPtr context_, const String &, const DisksMap &) { diff --git a/src/Disks/ObjectStorages/DiskObjectStorage.h b/src/Disks/ObjectStorages/DiskObjectStorage.h index a24acc270c0..d6723d1eb71 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorage.h +++ b/src/Disks/ObjectStorages/DiskObjectStorage.h @@ -152,6 +152,12 @@ public: WriteMode mode, const WriteSettings & settings) override; + void writeFileUsingCustomWriteObject( + const String & path, + WriteMode mode, + std::function & object_attributes)> + custom_write_object_function) override; + void copy(const String & from_path, const std::shared_ptr & to_disk, const String & to_path) override; void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context_, const String &, const DisksMap &) override; diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp index a9d82a3e0b1..072e747aa4a 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp @@ -670,6 +670,44 @@ std::unique_ptr DiskObjectStorageTransaction::writeFile } +void DiskObjectStorageTransaction::writeFileUsingCustomWriteObject( + const String & path, + WriteMode mode, + std::function & object_attributes)> + custom_write_object_function) +{ + /// This function is a simplified and adapted version of DiskObjectStorageTransaction::writeFile(). + auto blob_name = object_storage.generateBlobNameForPath(path); + std::optional object_attributes; + + if (metadata_helper) + { + auto revision = metadata_helper->revision_counter + 1; + metadata_helper->revision_counter++; + object_attributes = { + {"path", path} + }; + blob_name = "r" + revisionToString(revision) + "-file-" + blob_name; + } + + auto object = StoredObject::create(object_storage, fs::path(metadata_storage.getObjectStorageRootPath()) / blob_name); + auto write_operation = std::make_unique(object_storage, metadata_storage, object); + + operations_to_execute.emplace_back(std::move(write_operation)); + + /// We always use mode Rewrite because we simulate append using metadata and different files + size_t object_size = std::move(custom_write_object_function)(object, WriteMode::Rewrite, object_attributes); + + /// Create metadata (see create_metadata_callback in DiskObjectStorageTransaction::writeFile()). + if (mode == WriteMode::Rewrite) + metadata_transaction->createMetadataFile(path, blob_name, object_size); + else + metadata_transaction->addBlobToMetadata(path, blob_name, object_size); + + metadata_transaction->commit(); +} + + void DiskObjectStorageTransaction::createHardLink(const std::string & src_path, const std::string & dst_path) { operations_to_execute.emplace_back( diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h index 9e6bd5b6307..080a3e42057 100644 --- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h +++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.h @@ -99,6 +99,13 @@ public: const WriteSettings & settings = {}, bool autocommit = true) override; + /// Write a file using a custom function to write an object to the disk's object storage. + void writeFileUsingCustomWriteObject( + const String & path, + WriteMode mode, + std::function & object_attributes)> + custom_write_object_function) override; + void removeFile(const std::string & path) override; void removeFileIfExists(const std::string & path) override; void removeDirectory(const std::string & path) override; diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index cbba5ed64f9..83c0c7446a8 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -1,7 +1,4 @@ #include -#include -#include - #if USE_AWS_S3 @@ -18,10 +15,12 @@ #include #include #include +#include #include #include #include +#include #include #include #include diff --git a/src/Disks/ObjectStorages/S3/copyS3FileToDisk.cpp b/src/Disks/ObjectStorages/S3/copyS3FileToDisk.cpp new file mode 100644 index 00000000000..098e02595f5 --- /dev/null +++ b/src/Disks/ObjectStorages/S3/copyS3FileToDisk.cpp @@ -0,0 +1,69 @@ +#include + +#if USE_AWS_S3 + +#include +#include +#include +#include +#include + + +namespace DB +{ + +void copyS3FileToDisk( + const std::shared_ptr & s3_client, + const String & src_bucket, + const String & src_key, + const std::optional & version_id, + std::optional src_offset, + std::optional src_size, + DiskPtr destination_disk, + const String & destination_path, + WriteMode write_mode, + const ReadSettings & read_settings, + const WriteSettings & write_settings, + const S3Settings::RequestSettings & request_settings, + ThreadPoolCallbackRunner scheduler) +{ + if (!src_offset) + src_offset = 0; + + if (!src_size) + src_size = S3::getObjectSize(*s3_client, src_bucket, src_key, version_id.value_or(""), request_settings) - *src_offset; + + auto destination_data_source_description = destination_disk->getDataSourceDescription(); + if (destination_data_source_description != DataSourceDescription{DataSourceType::S3, s3_client->getInitialEndpoint(), false, false}) + { + LOG_TRACE(&Poco::Logger::get("copyS3FileToDisk"), "Copying {} to disk {} through buffers", src_key, destination_disk->getName()); + ReadBufferFromS3 read_buffer{s3_client, src_bucket, src_key, {}, request_settings, read_settings}; + if (*src_offset) + read_buffer.seek(*src_offset, SEEK_SET); + auto write_buffer = destination_disk->writeFile(destination_path, std::min(*src_size, DBMS_DEFAULT_BUFFER_SIZE), write_mode, write_settings); + copyData(read_buffer, *write_buffer, *src_size); + write_buffer->finalize(); + return; + } + + LOG_TRACE(&Poco::Logger::get("copyS3FileToDisk"), "Copying {} to disk {} using native copy", src_key, destination_disk->getName()); + + String dest_bucket = destination_disk->getObjectStorage()->getObjectsNamespace(); + + auto custom_write_object = [&](const StoredObject & object_, WriteMode write_mode_, const std::optional & object_attributes_) -> size_t + { + /// Object storage always uses mode `Rewrite` because it simulates append using metadata and different files. + chassert(write_mode_ == WriteMode::Rewrite); + + copyS3File(s3_client, src_bucket, src_key, *src_offset, *src_size, dest_bucket, /* dest_key= */ object_.absolute_path, + request_settings, object_attributes_, scheduler, /* for_disk_s3= */ true); + + return *src_size; + }; + + destination_disk->writeFileUsingCustomWriteObject(destination_path, write_mode, custom_write_object); +} + +} + +#endif diff --git a/src/Disks/ObjectStorages/S3/copyS3FileToDisk.h b/src/Disks/ObjectStorages/S3/copyS3FileToDisk.h new file mode 100644 index 00000000000..21c92ec9623 --- /dev/null +++ b/src/Disks/ObjectStorages/S3/copyS3FileToDisk.h @@ -0,0 +1,36 @@ +#pragma once + +#include "config.h" + +#if USE_AWS_S3 + +#include +#include +#include + + +namespace DB +{ + +/// Copies an object from S3 bucket to a disk of any type. +/// Depending on the disk the function can either do copying though buffers +/// (i.e. download the object by portions and then write those portions to the specified disk), +/// or perform a server-side copy. +void copyS3FileToDisk( + const std::shared_ptr & s3_client, + const String & src_bucket, + const String & src_key, + const std::optional & version_id, + std::optional src_offset, + std::optional src_size, + DiskPtr destination_disk, + const String & destination_path, + WriteMode write_mode = WriteMode::Rewrite, + const ReadSettings & read_settings = {}, + const WriteSettings & write_settings = {}, + const S3Settings::RequestSettings & request_settings = {}, + ThreadPoolCallbackRunner scheduler = {}); + +} + +#endif diff --git a/src/Disks/getDiskConfigurationFromAST.cpp b/src/Disks/getDiskConfigurationFromAST.cpp index e6b08046036..4b1323b4db8 100644 --- a/src/Disks/getDiskConfigurationFromAST.cpp +++ b/src/Disks/getDiskConfigurationFromAST.cpp @@ -83,4 +83,24 @@ DiskConfigurationPtr getDiskConfigurationFromAST(const std::string & root_name, return conf; } + +ASTs convertDiskConfigurationToAST(const Poco::Util::AbstractConfiguration & configuration, const std::string & config_path) +{ + ASTs result; + + Poco::Util::AbstractConfiguration::Keys keys; + configuration.keys(config_path, keys); + + for (const auto & key : keys) + { + result.push_back( + makeASTFunction( + "equals", + std::make_shared(key), + std::make_shared(configuration.getString(config_path + "." + key)))); + } + + return result; +} + } diff --git a/src/Disks/getDiskConfigurationFromAST.h b/src/Disks/getDiskConfigurationFromAST.h index 1f9d7c1bfe6..5697955e914 100644 --- a/src/Disks/getDiskConfigurationFromAST.h +++ b/src/Disks/getDiskConfigurationFromAST.h @@ -25,4 +25,12 @@ using DiskConfigurationPtr = Poco::AutoPtr; */ DiskConfigurationPtr getDiskConfigurationFromAST(const std::string & root_name, const ASTs & disk_args, ContextPtr context); +/// The same as above function, but return XML::Document for easier modification of result configuration. +[[ maybe_unused ]] Poco::AutoPtr getDiskConfigurationFromASTImpl(const std::string & root_name, const ASTs & disk_args, ContextPtr context); + +/* + * A reverse function. + */ +[[ maybe_unused ]] ASTs convertDiskConfigurationToAST(const Poco::Util::AbstractConfiguration & configuration, const std::string & config_path); + } diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 55003044ff5..f832bf404a8 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -803,7 +803,7 @@ struct ConvertImpl, DataTypeNumber, Name, Con } }; -static ColumnUInt8::MutablePtr copyNullMap(ColumnPtr col) +static inline ColumnUInt8::MutablePtr copyNullMap(ColumnPtr col) { ColumnUInt8::MutablePtr null_map = nullptr; if (const auto * col_null = checkAndGetColumn(col.get())) diff --git a/src/Functions/FunctionsStringSimilarity.cpp b/src/Functions/FunctionsStringSimilarity.cpp index 0cc0248baf4..faa01abb675 100644 --- a/src/Functions/FunctionsStringSimilarity.cpp +++ b/src/Functions/FunctionsStringSimilarity.cpp @@ -285,9 +285,9 @@ struct NgramDistanceImpl size_t first_size = dispatchSearcher(calculateHaystackStatsAndMetric, data.data(), data_size, common_stats.get(), distance, nullptr); /// For !symmetric version we should not use first_size. if constexpr (symmetric) - res = distance * 1.f / std::max(first_size + second_size, static_cast(1)); + res = distance * 1.f / std::max(first_size + second_size, 1uz); else - res = 1.f - distance * 1.f / std::max(second_size, static_cast(1)); + res = 1.f - distance * 1.f / std::max(second_size, 1uz); } else { @@ -353,9 +353,9 @@ struct NgramDistanceImpl /// For !symmetric version we should not use haystack_stats_size. if constexpr (symmetric) - res[i] = distance * 1.f / std::max(haystack_stats_size + needle_stats_size, static_cast(1)); + res[i] = distance * 1.f / std::max(haystack_stats_size + needle_stats_size, 1uz); else - res[i] = 1.f - distance * 1.f / std::max(needle_stats_size, static_cast(1)); + res[i] = 1.f - distance * 1.f / std::max(needle_stats_size, 1uz); } else { @@ -424,7 +424,7 @@ struct NgramDistanceImpl for (size_t j = 0; j < needle_stats_size; ++j) --common_stats[needle_ngram_storage[j]]; - res[i] = 1.f - distance * 1.f / std::max(needle_stats_size, static_cast(1)); + res[i] = 1.f - distance * 1.f / std::max(needle_stats_size, 1uz); } else { @@ -471,9 +471,9 @@ struct NgramDistanceImpl ngram_storage.get()); /// For !symmetric version we should not use haystack_stats_size. if constexpr (symmetric) - res[i] = distance * 1.f / std::max(haystack_stats_size + needle_stats_size, static_cast(1)); + res[i] = distance * 1.f / std::max(haystack_stats_size + needle_stats_size, 1uz); else - res[i] = 1.f - distance * 1.f / std::max(needle_stats_size, static_cast(1)); + res[i] = 1.f - distance * 1.f / std::max(needle_stats_size, 1uz); } else { diff --git a/src/Functions/UserDefined/UserDefinedSQLObjectsBackup.cpp b/src/Functions/UserDefined/UserDefinedSQLObjectsBackup.cpp index 6962c21280d..25c309aeb65 100644 --- a/src/Functions/UserDefined/UserDefinedSQLObjectsBackup.cpp +++ b/src/Functions/UserDefined/UserDefinedSQLObjectsBackup.cpp @@ -71,8 +71,7 @@ restoreUserDefinedSQLObjects(RestorerFromBackup & restorer, const String & data_ String function_name = unescapeForFileName(escaped_function_name); String filepath = data_path_in_backup_fs / filename; - auto backup_entry = backup->readFile(filepath); - auto in = backup_entry->getReadBuffer(); + auto in = backup->readFile(filepath); String statement_def; readStringUntilEOF(statement_def, *in); diff --git a/src/Functions/bitSlice.cpp b/src/Functions/bitSlice.cpp index 9b0ee4d5f1e..e2b455846d8 100644 --- a/src/Functions/bitSlice.cpp +++ b/src/Functions/bitSlice.cpp @@ -291,7 +291,7 @@ public: ssize_t remain_byte = src.getElementSize() - offset_byte; if (length < 0) { - length_byte = std::max(remain_byte + (length / word_size), static_cast(0)); + length_byte = std::max(remain_byte + (length / word_size), 0z); over_bit = word_size + (length % word_size); if (length_byte == 1 && over_bit <= offset_bit) // begin and end are in same byte AND there are no gaps length_byte = 0; @@ -330,7 +330,7 @@ public: size_t size = src.getElementSize(); if (length < 0) { - length_byte = std::max(static_cast(offset_byte) + (length / word_size), static_cast(0)); + length_byte = std::max(static_cast(offset_byte) + (length / word_size), 0z); over_bit = word_size + (length % word_size); if (length_byte == 1 && over_bit <= offset_bit) // begin and end are in same byte AND there are no gaps length_byte = 0; @@ -395,7 +395,7 @@ public: } else { - length_byte = std::max(remain_byte + (static_cast(length) / word_size), static_cast(0)); + length_byte = std::max(remain_byte + (static_cast(length) / word_size), 0z); over_bit = word_size + (length % word_size); if (length_byte == 1 && over_bit <= offset_bit) // begin and end are in same byte AND there are no gaps length_byte = 0; diff --git a/src/Functions/positionCaseInsensitive.cpp b/src/Functions/positionCaseInsensitive.cpp index 4e3b670fe54..f71ce0078cc 100644 --- a/src/Functions/positionCaseInsensitive.cpp +++ b/src/Functions/positionCaseInsensitive.cpp @@ -20,5 +20,6 @@ using FunctionPositionCaseInsensitive = FunctionsStringSearch(); + factory.registerAlias("instr", NamePositionCaseInsensitive::name, FunctionFactory::CaseInsensitive); } } diff --git a/src/IO/MemoryReadWriteBuffer.cpp b/src/IO/MemoryReadWriteBuffer.cpp index 8958390fe03..93ce5ce7ce9 100644 --- a/src/IO/MemoryReadWriteBuffer.cpp +++ b/src/IO/MemoryReadWriteBuffer.cpp @@ -106,7 +106,7 @@ void MemoryWriteBuffer::addChunk() } else { - next_chunk_size = std::max(static_cast(1), static_cast(chunk_tail->size() * growth_rate)); + next_chunk_size = std::max(1uz, static_cast(chunk_tail->size() * growth_rate)); next_chunk_size = std::min(next_chunk_size, max_chunk_size); } diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index 5c0539ee486..aba884948da 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -123,9 +123,8 @@ Client::Client( { auto * endpoint_provider = dynamic_cast(accessEndpointProvider().get()); endpoint_provider->GetBuiltInParameters().GetParameter("Region").GetString(explicit_region); - std::string endpoint; - endpoint_provider->GetBuiltInParameters().GetParameter("Endpoint").GetString(endpoint); - detect_region = explicit_region == Aws::Region::AWS_GLOBAL && endpoint.find(".amazonaws.com") != std::string::npos; + endpoint_provider->GetBuiltInParameters().GetParameter("Endpoint").GetString(initial_endpoint); + detect_region = explicit_region == Aws::Region::AWS_GLOBAL && initial_endpoint.find(".amazonaws.com") != std::string::npos; cache = std::make_shared(); ClientCacheRegistry::instance().registerClient(cache); @@ -133,6 +132,7 @@ Client::Client( Client::Client(const Client & other) : Aws::S3::S3Client(other) + , initial_endpoint(other.initial_endpoint) , explicit_region(other.explicit_region) , detect_region(other.detect_region) , max_redirects(other.max_redirects) diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h index 18ba62d1006..7ac97555dd1 100644 --- a/src/IO/S3/Client.h +++ b/src/IO/S3/Client.h @@ -109,6 +109,9 @@ public: } } + /// Returns the initial endpoint. + const String & getInitialEndpoint() const { return initial_endpoint; } + /// Decorator for RetryStrategy needed for this client to work correctly. /// We want to manually handle permanent moves (status code 301) because: /// - redirect location is written in XML format inside the response body something that doesn't exist for HEAD @@ -198,6 +201,8 @@ private: bool checkIfWrongRegionDefined(const std::string & bucket, const Aws::S3::S3Error & error, std::string & region) const; void insertRegionOverride(const std::string & bucket, const std::string & region) const; + String initial_endpoint; + std::string explicit_region; mutable bool detect_region = true; diff --git a/src/IO/copyData.cpp b/src/IO/copyData.cpp index b189c318d67..07222a930b5 100644 --- a/src/IO/copyData.cpp +++ b/src/IO/copyData.cpp @@ -10,6 +10,7 @@ namespace DB namespace ErrorCodes { extern const int ATTEMPT_TO_READ_AFTER_EOF; + extern const int CANNOT_READ_ALL_DATA; } namespace @@ -91,6 +92,13 @@ void copyData(ReadBuffer & from, WriteBuffer & to, size_t bytes, std::function & is_cancelled, ThrottlerPtr throttler) { copyDataImpl(from, to, false, std::numeric_limits::max(), &is_cancelled, throttler); diff --git a/src/IO/copyData.h b/src/IO/copyData.h index 2202f36f79e..b67088d8e47 100644 --- a/src/IO/copyData.h +++ b/src/IO/copyData.h @@ -27,6 +27,9 @@ void copyData(ReadBuffer & from, WriteBuffer & to, size_t bytes, const std::atom void copyData(ReadBuffer & from, WriteBuffer & to, std::function cancellation_hook); void copyData(ReadBuffer & from, WriteBuffer & to, size_t bytes, std::function cancellation_hook); +/// Copies at most `max_bytes` bytes from ReadBuffer to WriteBuffer. If there are more bytes, then throws an exception. +void copyDataMaxBytes(ReadBuffer & from, WriteBuffer & to, size_t max_bytes); + /// Same as above but also use throttler to limit maximum speed void copyDataWithThrottler(ReadBuffer & from, WriteBuffer & to, const std::atomic & is_cancelled, ThrottlerPtr throttler); void copyDataWithThrottler(ReadBuffer & from, WriteBuffer & to, size_t bytes, const std::atomic & is_cancelled, ThrottlerPtr throttler); diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index f5e98250af1..09c2eebfdd6 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1442,7 +1442,8 @@ void Aggregator::prepareAggregateInstructions( aggregate_columns[i][j] = materialized_columns.back().get(); /// Sparse columns without defaults may be handled incorrectly. - if (aggregate_columns[i][j]->getNumberOfDefaultRows() == 0) + if (aggregate_columns[i][j]->isSparse() + && aggregate_columns[i][j]->getNumberOfDefaultRows() == 0) allow_sparse_arguments = false; auto full_column = allow_sparse_arguments diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index cf1d5203bf7..2cfa55f0d87 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -129,13 +129,21 @@ namespace CurrentMetrics { extern const Metric ContextLockWait; extern const Metric BackgroundMovePoolTask; + extern const Metric BackgroundMovePoolSize; extern const Metric BackgroundSchedulePoolTask; + extern const Metric BackgroundSchedulePoolSize; extern const Metric BackgroundBufferFlushSchedulePoolTask; + extern const Metric BackgroundBufferFlushSchedulePoolSize; extern const Metric BackgroundDistributedSchedulePoolTask; + extern const Metric BackgroundDistributedSchedulePoolSize; extern const Metric BackgroundMessageBrokerSchedulePoolTask; + extern const Metric BackgroundMessageBrokerSchedulePoolSize; extern const Metric BackgroundMergesAndMutationsPoolTask; + extern const Metric BackgroundMergesAndMutationsPoolSize; extern const Metric BackgroundFetchesPoolTask; + extern const Metric BackgroundFetchesPoolSize; extern const Metric BackgroundCommonPoolTask; + extern const Metric BackgroundCommonPoolSize; } namespace DB @@ -2175,6 +2183,7 @@ BackgroundSchedulePool & Context::getBufferFlushSchedulePool() const shared->buffer_flush_schedule_pool = std::make_unique( background_buffer_flush_schedule_pool_size, CurrentMetrics::BackgroundBufferFlushSchedulePoolTask, + CurrentMetrics::BackgroundBufferFlushSchedulePoolSize, "BgBufSchPool"); } @@ -2226,6 +2235,7 @@ BackgroundSchedulePool & Context::getSchedulePool() const shared->schedule_pool = std::make_unique( background_schedule_pool_size, CurrentMetrics::BackgroundSchedulePoolTask, + CurrentMetrics::BackgroundSchedulePoolSize, "BgSchPool"); } @@ -2246,6 +2256,7 @@ BackgroundSchedulePool & Context::getDistributedSchedulePool() const shared->distributed_schedule_pool = std::make_unique( background_distributed_schedule_pool_size, CurrentMetrics::BackgroundDistributedSchedulePoolTask, + CurrentMetrics::BackgroundDistributedSchedulePoolSize, "BgDistSchPool"); } @@ -2266,6 +2277,7 @@ BackgroundSchedulePool & Context::getMessageBrokerSchedulePool() const shared->message_broker_schedule_pool = std::make_unique( background_message_broker_schedule_pool_size, CurrentMetrics::BackgroundMessageBrokerSchedulePoolTask, + CurrentMetrics::BackgroundMessageBrokerSchedulePoolSize, "BgMBSchPool"); } @@ -3826,6 +3838,7 @@ void Context::initializeBackgroundExecutorsIfNeeded() /*max_threads_count*/background_pool_size, /*max_tasks_count*/background_pool_size * background_merges_mutations_concurrency_ratio, CurrentMetrics::BackgroundMergesAndMutationsPoolTask, + CurrentMetrics::BackgroundMergesAndMutationsPoolSize, background_merges_mutations_scheduling_policy ); LOG_INFO(shared->log, "Initialized background executor for merges and mutations with num_threads={}, num_tasks={}, scheduling_policy={}", @@ -3836,7 +3849,8 @@ void Context::initializeBackgroundExecutorsIfNeeded() "Move", background_move_pool_size, background_move_pool_size, - CurrentMetrics::BackgroundMovePoolTask + CurrentMetrics::BackgroundMovePoolTask, + CurrentMetrics::BackgroundMovePoolSize ); LOG_INFO(shared->log, "Initialized background executor for move operations with num_threads={}, num_tasks={}", background_move_pool_size, background_move_pool_size); @@ -3845,7 +3859,8 @@ void Context::initializeBackgroundExecutorsIfNeeded() "Fetch", background_fetches_pool_size, background_fetches_pool_size, - CurrentMetrics::BackgroundFetchesPoolTask + CurrentMetrics::BackgroundFetchesPoolTask, + CurrentMetrics::BackgroundFetchesPoolSize ); LOG_INFO(shared->log, "Initialized background executor for fetches with num_threads={}, num_tasks={}", background_fetches_pool_size, background_fetches_pool_size); @@ -3854,7 +3869,8 @@ void Context::initializeBackgroundExecutorsIfNeeded() "Common", background_common_pool_size, background_common_pool_size, - CurrentMetrics::BackgroundCommonPoolTask + CurrentMetrics::BackgroundCommonPoolTask, + CurrentMetrics::BackgroundCommonPoolSize ); LOG_INFO(shared->log, "Initialized background executor for common operations (e.g. clearing old parts) with num_threads={}, num_tasks={}", background_common_pool_size, background_common_pool_size); diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 975e0da66ce..b11a973c7b7 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -121,9 +121,16 @@ TemporaryTableHolder::~TemporaryTableHolder() { if (id != UUIDHelpers::Nil) { - auto table = getTable(); - table->flushAndShutdown(); - temporary_tables->dropTable(getContext(), "_tmp_" + toString(id)); + try + { + auto table = getTable(); + table->flushAndShutdown(); + temporary_tables->dropTable(getContext(), "_tmp_" + toString(id)); + } + catch (...) + { + tryLogCurrentException("TemporaryTableHolder"); + } } } @@ -140,7 +147,6 @@ StoragePtr TemporaryTableHolder::getTable() const return table; } - void DatabaseCatalog::initializeAndLoadTemporaryDatabase() { drop_delay_sec = getContext()->getConfigRef().getInt("database_atomic_delay_before_drop_table_sec", default_drop_delay_sec); diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 4200373018d..88645ff72af 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -235,6 +235,21 @@ public: void checkTableCanBeRemovedOrRenamed(const StorageID & table_id, bool check_referential_dependencies, bool check_loading_dependencies, bool is_drop_database = false) const; + + struct TableMarkedAsDropped + { + StorageID table_id = StorageID::createEmpty(); + StoragePtr table; + String metadata_path; + time_t drop_time{}; + }; + using TablesMarkedAsDropped = std::list; + + TablesMarkedAsDropped getTablesMarkedDropped() + { + std::lock_guard lock(tables_marked_dropped_mutex); + return tables_marked_dropped; + } private: // The global instance of database catalog. unique_ptr is to allow // deferred initialization. Thought I'd use std::optional, but I can't @@ -263,15 +278,6 @@ private: return uuid.toUnderType().items[0] >> (64 - bits_for_first_level); } - struct TableMarkedAsDropped - { - StorageID table_id = StorageID::createEmpty(); - StoragePtr table; - String metadata_path; - time_t drop_time{}; - }; - using TablesMarkedAsDropped = std::list; - void dropTableDataTask(); void dropTableFinally(const TableMarkedAsDropped & table); diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 9931ae97286..4c9f47e5915 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -848,6 +848,23 @@ std::string ExpressionActions::dumpActions() const return ss.str(); } +void ExpressionActions::describeActions(WriteBuffer & out, std::string_view prefix) const +{ + bool first = true; + + for (const auto & action : actions) + { + out << prefix << (first ? "Actions: " : " "); + out << action.toString() << '\n'; + first = false; + } + + out << prefix << "Positions:"; + for (const auto & pos : result_positions) + out << ' ' << pos; + out << '\n'; +} + JSONBuilder::ItemPtr ExpressionActions::toTree() const { auto inputs_array = std::make_unique(); diff --git a/src/Interpreters/ExpressionActions.h b/src/Interpreters/ExpressionActions.h index 11957997a30..db6670c50b9 100644 --- a/src/Interpreters/ExpressionActions.h +++ b/src/Interpreters/ExpressionActions.h @@ -109,6 +109,9 @@ public: const Block & getSampleBlock() const { return sample_block; } std::string dumpActions() const; + + void describeActions(WriteBuffer & out, std::string_view prefix) const; + JSONBuilder::ItemPtr toTree() const; static NameAndTypePair getSmallestColumn(const NamesAndTypesList & columns); diff --git a/src/Interpreters/FillingRow.cpp b/src/Interpreters/FillingRow.cpp index b03049a209f..5c2ad548c93 100644 --- a/src/Interpreters/FillingRow.cpp +++ b/src/Interpreters/FillingRow.cpp @@ -107,39 +107,4 @@ void FillingRow::initFromDefaults(size_t from_pos) row[i] = getFillDescription(i).fill_from; } -void insertFromFillingRow(MutableColumns & filling_columns, MutableColumns & interpolate_columns, MutableColumns & other_columns, - const FillingRow & filling_row, const Block & interpolate_block) -{ - for (size_t i = 0, size = filling_columns.size(); i < size; ++i) - { - if (filling_row[i].isNull()) - { - filling_columns[i]->insertDefault(); - } - else - { - filling_columns[i]->insert(filling_row[i]); - } - } - - if (size_t size = interpolate_block.columns()) - { - Columns columns = interpolate_block.getColumns(); - for (size_t i = 0; i < size; ++i) - interpolate_columns[i]->insertFrom(*columns[i]->convertToFullColumnIfConst(), 0); - } - else - for (const auto & interpolate_column : interpolate_columns) - interpolate_column->insertDefault(); - - for (const auto & other_column : other_columns) - other_column->insertDefault(); -} - -void copyRowFromColumns(MutableColumns & dest, const Columns & source, size_t row_num) -{ - for (size_t i = 0, size = source.size(); i < size; ++i) - dest[i]->insertFrom(*source[i], row_num); -} - } diff --git a/src/Interpreters/FillingRow.h b/src/Interpreters/FillingRow.h index 331c237285b..8d47094d0de 100644 --- a/src/Interpreters/FillingRow.h +++ b/src/Interpreters/FillingRow.h @@ -39,8 +39,4 @@ private: SortDescription sort_description; }; -void insertFromFillingRow(MutableColumns & filling_columns, MutableColumns & interpolate_columns, MutableColumns & other_columns, - const FillingRow & filling_row, const Block & interpolate_block); -void copyRowFromColumns(MutableColumns & dest, const Columns & source, size_t row_num); - } diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index fba985da41c..b4376426700 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -495,7 +495,7 @@ size_t HashJoin::getTotalByteCount() const if (!data) return 0; -#ifdef NDEBUG +#ifndef NDEBUG size_t debug_blocks_allocated_size = 0; for (const auto & block : data->blocks) debug_blocks_allocated_size += block.allocatedBytes(); diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index c352280b7ed..7a4d65a4d57 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -940,23 +940,32 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const if (create.temporary) { - if (create.storage && create.storage->engine && create.storage->engine->name != "Memory") - throw Exception(ErrorCodes::INCORRECT_QUERY, "Temporary tables can only be created with ENGINE = Memory, not {}", - create.storage->engine->name); - /// It's possible if some part of storage definition (such as PARTITION BY) is specified, but ENGINE is not. /// It makes sense when default_table_engine setting is used, but not for temporary tables. /// For temporary tables we ignore this setting to allow CREATE TEMPORARY TABLE query without specifying ENGINE - /// even if setting is set to MergeTree or something like that (otherwise MergeTree will be substituted and query will fail). - if (create.storage && !create.storage->engine) - throw Exception(ErrorCodes::INCORRECT_QUERY, "Invalid storage definition for temporary table: must be either ENGINE = Memory or empty"); - auto engine_ast = std::make_shared(); - engine_ast->name = "Memory"; - engine_ast->no_empty_args = true; - auto storage_ast = std::make_shared(); - storage_ast->set(storage_ast->engine, engine_ast); - create.set(create.storage, storage_ast); + if (!create.cluster.empty()) + throw Exception(ErrorCodes::INCORRECT_QUERY, "Temporary tables cannot be created with ON CLUSTER clause"); + + if (create.storage) + { + if (create.storage->engine) + { + if (create.storage->engine->name.starts_with("Replicated") || create.storage->engine->name == "KeeperMap") + throw Exception(ErrorCodes::INCORRECT_QUERY, "Temporary tables cannot be created with Replicated or KeeperMap table engines"); + } + else + throw Exception(ErrorCodes::INCORRECT_QUERY, "Invalid storage definition for temporary table"); + } + else + { + auto engine_ast = std::make_shared(); + engine_ast->name = "Memory"; + engine_ast->no_empty_args = true; + auto storage_ast = std::make_shared(); + storage_ast->set(storage_ast->engine, engine_ast); + create.set(create.storage, storage_ast); + } return; } @@ -1284,8 +1293,21 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, if (create.if_not_exists && getContext()->tryResolveStorageID({"", create.getTable()}, Context::ResolveExternal)) return false; + DatabasePtr database = DatabaseCatalog::instance().getDatabase(DatabaseCatalog::TEMPORARY_DATABASE); + String temporary_table_name = create.getTable(); - auto temporary_table = TemporaryTableHolder(getContext(), properties.columns, properties.constraints, query_ptr); + auto creator = [&](const StorageID & table_id) + { + return StorageFactory::instance().get(create, + database->getTableDataPath(table_id.getTableName()), + getContext(), + getContext()->getGlobalContext(), + properties.columns, + properties.constraints, + false); + }; + auto temporary_table = TemporaryTableHolder(getContext(), creator, query_ptr); + getContext()->getSessionContext()->addExternalTable(temporary_table_name, std::move(temporary_table)); return true; } @@ -1712,7 +1734,13 @@ AccessRightsElements InterpreterCreateQuery::getRequiredAccess() const else { if (create.temporary) - required_access.emplace_back(AccessType::CREATE_TEMPORARY_TABLE); + { + /// Currently default table engine for temporary tables is Memory. default_table_engine does not affect temporary tables. + if (create.storage && create.storage->engine && create.storage->engine->name != "Memory") + required_access.emplace_back(AccessType::CREATE_ARBITRARY_TEMPORARY_TABLE); + else + required_access.emplace_back(AccessType::CREATE_TEMPORARY_TABLE); + } else { if (create.replace_table) diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index f4507de5ac7..e16403bed67 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -282,11 +282,6 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(const String & table_name, else if (kind == ASTDropQuery::Kind::Drop) { context_handle->removeExternalTable(table_name); - table->flushAndShutdown(); - auto table_lock = table->lockExclusively(getContext()->getCurrentQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); - /// Delete table data - table->drop(); - table->is_dropped = true; } else if (kind == ASTDropQuery::Kind::Detach) { diff --git a/src/Interpreters/ReplaceQueryParameterVisitor.cpp b/src/Interpreters/ReplaceQueryParameterVisitor.cpp index 893c93f0950..fad9d4bbfb2 100644 --- a/src/Interpreters/ReplaceQueryParameterVisitor.cpp +++ b/src/Interpreters/ReplaceQueryParameterVisitor.cpp @@ -83,7 +83,10 @@ void ReplaceQueryParameterVisitor::visitQueryParameter(ASTPtr & ast) IColumn & temp_column = *temp_column_ptr; ReadBufferFromString read_buffer{value}; FormatSettings format_settings; - data_type->getDefaultSerialization()->deserializeTextEscaped(temp_column, read_buffer, format_settings); + if (ast_param.name == "_request_body") + data_type->getDefaultSerialization()->deserializeWholeText(temp_column, read_buffer, format_settings); + else + data_type->getDefaultSerialization()->deserializeTextEscaped(temp_column, read_buffer, format_settings); if (!read_buffer.eof()) throw Exception(ErrorCodes::BAD_QUERY_PARAMETER, diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 72353a42a87..a6354cd0e81 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -2151,8 +2151,9 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserKeyword s_set("SET"); ParserKeyword s_recompress("RECOMPRESS"); ParserKeyword s_codec("CODEC"); - ParserToken s_comma(TokenType::Comma); - ParserToken s_eq(TokenType::Equals); + ParserKeyword s_materialize("MATERIALIZE"); + ParserKeyword s_remove("REMOVE"); + ParserKeyword s_modify("MODIFY"); ParserIdentifier parser_identifier; ParserStringLiteral parser_string_literal; @@ -2160,8 +2161,11 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserExpressionList parser_keys_list(false); ParserCodec parser_codec; - ParserList parser_assignment_list( - std::make_unique(), std::make_unique(TokenType::Comma)); + if (s_materialize.checkWithoutMoving(pos, expected) || + s_remove.checkWithoutMoving(pos, expected) || + s_modify.checkWithoutMoving(pos, expected)) + + return false; ASTPtr ttl_expr; if (!parser_exp.parse(pos, ttl_expr, expected)) @@ -2219,6 +2223,9 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (s_set.ignore(pos)) { + ParserList parser_assignment_list( + std::make_unique(), std::make_unique(TokenType::Comma)); + if (!parser_assignment_list.parse(pos, group_by_assignments, expected)) return false; } diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 37a4614bad3..2242bf92e6b 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -395,7 +395,11 @@ void addMergingAggregatedStep(QueryPlan & query_plan, * but it can work more slowly. */ - Aggregator::Params params(aggregation_analysis_result.aggregation_keys, + auto keys = aggregation_analysis_result.aggregation_keys; + if (!aggregation_analysis_result.grouping_sets_parameters_list.empty()) + keys.insert(keys.begin(), "__grouping_set"); + + Aggregator::Params params(keys, aggregation_analysis_result.aggregate_descriptions, query_analysis_result.aggregate_overflow_row, settings.max_threads, diff --git a/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp b/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp index 46d1872412c..0fb1a413a6c 100644 --- a/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp @@ -45,7 +45,7 @@ void PrettySpaceBlockOutputFormat::writeChunk(const Chunk & chunk, PortKind port if (col.type->shouldAlignRightInPrettyFormats()) { - for (ssize_t k = 0; k < std::max(static_cast(0), static_cast(max_widths[i] - name_widths[i])); ++k) + for (ssize_t k = 0; k < std::max(0z, static_cast(max_widths[i] - name_widths[i])); ++k) writeChar(' ', out); if (format_settings.pretty.color) @@ -62,7 +62,7 @@ void PrettySpaceBlockOutputFormat::writeChunk(const Chunk & chunk, PortKind port if (format_settings.pretty.color) writeCString("\033[0m", out); - for (ssize_t k = 0; k < std::max(static_cast(0), static_cast(max_widths[i] - name_widths[i])); ++k) + for (ssize_t k = 0; k < std::max(0z, static_cast(max_widths[i] - name_widths[i])); ++k) writeChar(' ', out); } } diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index 9bf351442b2..69dfa05899b 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -38,7 +38,6 @@ static ITransformingStep::Traits getTraits(bool should_produce_results_in_order_ return ITransformingStep::Traits { { - .preserves_distinct_columns = false, /// Actually, we may check that distinct names are in aggregation keys .returns_single_stream = should_produce_results_in_order_of_bucket_number, .preserves_number_of_streams = false, .preserves_sorting = false, diff --git a/src/Processors/QueryPlan/ArrayJoinStep.cpp b/src/Processors/QueryPlan/ArrayJoinStep.cpp index bd1908a4a6d..23a0a756f0d 100644 --- a/src/Processors/QueryPlan/ArrayJoinStep.cpp +++ b/src/Processors/QueryPlan/ArrayJoinStep.cpp @@ -14,7 +14,6 @@ static ITransformingStep::Traits getTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = false, .returns_single_stream = false, .preserves_number_of_streams = true, .preserves_sorting = false, diff --git a/src/Processors/QueryPlan/CreateSetAndFilterOnTheFlyStep.cpp b/src/Processors/QueryPlan/CreateSetAndFilterOnTheFlyStep.cpp index 53dcec9ef0a..07137e87736 100644 --- a/src/Processors/QueryPlan/CreateSetAndFilterOnTheFlyStep.cpp +++ b/src/Processors/QueryPlan/CreateSetAndFilterOnTheFlyStep.cpp @@ -40,7 +40,6 @@ static ITransformingStep::Traits getTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = true, .returns_single_stream = false, .preserves_number_of_streams = true, .preserves_sorting = true, diff --git a/src/Processors/QueryPlan/CreatingSetsStep.cpp b/src/Processors/QueryPlan/CreatingSetsStep.cpp index 23e0a17a31b..b696b77ccfe 100644 --- a/src/Processors/QueryPlan/CreatingSetsStep.cpp +++ b/src/Processors/QueryPlan/CreatingSetsStep.cpp @@ -21,7 +21,6 @@ static ITransformingStep::Traits getTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = true, .returns_single_stream = false, .preserves_number_of_streams = true, .preserves_sorting = true, diff --git a/src/Processors/QueryPlan/CubeStep.cpp b/src/Processors/QueryPlan/CubeStep.cpp index 03f952ac782..0c632c346c7 100644 --- a/src/Processors/QueryPlan/CubeStep.cpp +++ b/src/Processors/QueryPlan/CubeStep.cpp @@ -14,7 +14,6 @@ static ITransformingStep::Traits getTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = false, .returns_single_stream = true, .preserves_number_of_streams = false, .preserves_sorting = false, @@ -32,9 +31,6 @@ CubeStep::CubeStep(const DataStream & input_stream_, Aggregator::Params params_, , final(final_) , use_nulls(use_nulls_) { - /// Aggregation keys are distinct - for (const auto & key : params.keys) - output_stream->distinct_columns.insert(key); } ProcessorPtr addGroupingSetForTotals(const Block & header, const Names & keys, bool use_nulls, const BuildQueryPipelineSettings & settings, UInt64 grouping_set_number) @@ -89,9 +85,5 @@ void CubeStep::updateOutputStream() { output_stream = createOutputStream( input_streams.front(), generateOutputHeader(params.getHeader(input_streams.front().header, final), params.keys, use_nulls), getDataStreamTraits()); - - /// Aggregation keys are distinct - for (const auto & key : params.keys) - output_stream->distinct_columns.insert(key); } } diff --git a/src/Processors/QueryPlan/DistinctStep.cpp b/src/Processors/QueryPlan/DistinctStep.cpp index 323ef0bbdab..15ed02b700e 100644 --- a/src/Processors/QueryPlan/DistinctStep.cpp +++ b/src/Processors/QueryPlan/DistinctStep.cpp @@ -10,28 +10,13 @@ namespace DB { -static bool checkColumnsAlreadyDistinct(const Names & columns, const NameSet & distinct_names) -{ - if (distinct_names.empty()) - return false; - - /// Now we need to check that distinct_names is a subset of columns. - std::unordered_set columns_set(columns.begin(), columns.end()); - for (const auto & name : distinct_names) - if (!columns_set.contains(name)) - return false; - - return true; -} - -static ITransformingStep::Traits getTraits(bool pre_distinct, bool already_distinct_columns) +static ITransformingStep::Traits getTraits(bool pre_distinct) { return ITransformingStep::Traits { { - .preserves_distinct_columns = already_distinct_columns, /// Will be calculated separately otherwise - .returns_single_stream = !pre_distinct && !already_distinct_columns, - .preserves_number_of_streams = pre_distinct || already_distinct_columns, + .returns_single_stream = !pre_distinct, + .preserves_number_of_streams = pre_distinct, .preserves_sorting = true, /// Sorting is preserved indeed because of implementation. }, { @@ -62,34 +47,23 @@ DistinctStep::DistinctStep( : ITransformingStep( input_stream_, input_stream_.header, - getTraits(pre_distinct_, checkColumnsAlreadyDistinct(columns_, input_stream_.distinct_columns))) + getTraits(pre_distinct_)) , set_size_limits(set_size_limits_) , limit_hint(limit_hint_) , columns(columns_) , pre_distinct(pre_distinct_) , optimize_distinct_in_order(optimize_distinct_in_order_) { - if (!output_stream->distinct_columns.empty() /// Columns already distinct, do nothing - && (!pre_distinct /// Main distinct - || input_stream_.has_single_port)) /// pre_distinct for single port works as usual one - { - /// Build distinct set. - for (const auto & name : columns) - output_stream->distinct_columns.insert(name); - } } void DistinctStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { - const auto & input_stream = input_streams.back(); - if (checkColumnsAlreadyDistinct(columns, input_stream.distinct_columns)) - return; - if (!pre_distinct) pipeline.resize(1); if (optimize_distinct_in_order) { + const auto & input_stream = input_streams.back(); const SortDescription distinct_sort_desc = getSortDescription(input_stream.sort_description, columns); if (!distinct_sort_desc.empty()) { @@ -197,16 +171,7 @@ void DistinctStep::updateOutputStream() output_stream = createOutputStream( input_streams.front(), input_streams.front().header, - getTraits(pre_distinct, checkColumnsAlreadyDistinct(columns, input_streams.front().distinct_columns)).data_stream_traits); - - if (!output_stream->distinct_columns.empty() /// Columns already distinct, do nothing - && (!pre_distinct /// Main distinct - || input_streams.front().has_single_port)) /// pre_distinct for single port works as usual one - { - /// Build distinct set. - for (const auto & name : columns) - output_stream->distinct_columns.insert(name); - } + getTraits(pre_distinct).data_stream_traits); } } diff --git a/src/Processors/QueryPlan/ExpressionStep.cpp b/src/Processors/QueryPlan/ExpressionStep.cpp index dcfa6e5a891..250a1733caa 100644 --- a/src/Processors/QueryPlan/ExpressionStep.cpp +++ b/src/Processors/QueryPlan/ExpressionStep.cpp @@ -15,7 +15,6 @@ static ITransformingStep::Traits getTraits(const ActionsDAGPtr & actions, const return ITransformingStep::Traits { { - .preserves_distinct_columns = !actions->hasArrayJoin(), .returns_single_stream = false, .preserves_number_of_streams = true, .preserves_sorting = actions->isSortingPreserved(header, sort_description), @@ -33,8 +32,6 @@ ExpressionStep::ExpressionStep(const DataStream & input_stream_, const ActionsDA getTraits(actions_dag_, input_stream_.header, input_stream_.sort_description)) , actions_dag(actions_dag_) { - /// Some columns may be removed by expression. - updateDistinctColumns(output_stream->header, output_stream->distinct_columns); } void ExpressionStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) @@ -63,22 +60,9 @@ void ExpressionStep::transformPipeline(QueryPipelineBuilder & pipeline, const Bu void ExpressionStep::describeActions(FormatSettings & settings) const { - String prefix(settings.offset, ' '); - bool first = true; - + String prefix(settings.offset, settings.indent_char); auto expression = std::make_shared(actions_dag); - for (const auto & action : expression->getActions()) - { - settings.out << prefix << (first ? "Actions: " - : " "); - first = false; - settings.out << action.toString() << '\n'; - } - - settings.out << prefix << "Positions:"; - for (const auto & pos : expression->getResultPositions()) - settings.out << ' ' << pos; - settings.out << '\n'; + expression->describeActions(settings.out, prefix); } void ExpressionStep::describeActions(JSONBuilder::JSONMap & map) const diff --git a/src/Processors/QueryPlan/ExtremesStep.cpp b/src/Processors/QueryPlan/ExtremesStep.cpp index 4524b9883d6..010a82072cf 100644 --- a/src/Processors/QueryPlan/ExtremesStep.cpp +++ b/src/Processors/QueryPlan/ExtremesStep.cpp @@ -9,7 +9,6 @@ static ITransformingStep::Traits getTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = true, .returns_single_stream = false, .preserves_number_of_streams = true, .preserves_sorting = true, diff --git a/src/Processors/QueryPlan/FillingStep.cpp b/src/Processors/QueryPlan/FillingStep.cpp index 13b7ca625fb..20d7d6d0f8f 100644 --- a/src/Processors/QueryPlan/FillingStep.cpp +++ b/src/Processors/QueryPlan/FillingStep.cpp @@ -17,7 +17,6 @@ static ITransformingStep::Traits getTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = false, /// TODO: it seem to actually be true. Check it later. .returns_single_stream = true, .preserves_number_of_streams = true, .preserves_sorting = true, diff --git a/src/Processors/QueryPlan/FilterStep.cpp b/src/Processors/QueryPlan/FilterStep.cpp index 4699a7c1908..dc837446a96 100644 --- a/src/Processors/QueryPlan/FilterStep.cpp +++ b/src/Processors/QueryPlan/FilterStep.cpp @@ -23,7 +23,6 @@ static ITransformingStep::Traits getTraits(const ActionsDAGPtr & expression, con return ITransformingStep::Traits { { - .preserves_distinct_columns = !expression->hasArrayJoin(), /// I suppose it actually never happens .returns_single_stream = false, .preserves_number_of_streams = true, .preserves_sorting = preserves_sorting, @@ -51,8 +50,6 @@ FilterStep::FilterStep( , filter_column_name(std::move(filter_column_name_)) , remove_filter_column(remove_filter_column_) { - /// TODO: it would be easier to remove all expressions from filter step. It should only filter by column name. - updateDistinctColumns(output_stream->header, output_stream->distinct_columns); } void FilterStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) @@ -82,27 +79,15 @@ void FilterStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQ void FilterStep::describeActions(FormatSettings & settings) const { - String prefix(settings.offset, ' '); + String prefix(settings.offset, settings.indent_char); settings.out << prefix << "Filter column: " << filter_column_name; if (remove_filter_column) settings.out << " (removed)"; settings.out << '\n'; - bool first = true; auto expression = std::make_shared(actions_dag); - for (const auto & action : expression->getActions()) - { - settings.out << prefix << (first ? "Actions: " - : " "); - first = false; - settings.out << action.toString() << '\n'; - } - - settings.out << prefix << "Positions:"; - for (const auto & pos : expression->getResultPositions()) - settings.out << ' ' << pos; - settings.out << '\n'; + expression->describeActions(settings.out, prefix); } void FilterStep::describeActions(JSONBuilder::JSONMap & map) const diff --git a/src/Processors/QueryPlan/IQueryPlanStep.h b/src/Processors/QueryPlan/IQueryPlanStep.h index 316ecff9c2e..a608c6f8058 100644 --- a/src/Processors/QueryPlan/IQueryPlanStep.h +++ b/src/Processors/QueryPlan/IQueryPlanStep.h @@ -23,11 +23,6 @@ class DataStream public: Block header; - /// Tuples with those columns are distinct. - /// It doesn't mean that columns are distinct separately. - /// Removing any column from this list breaks this invariant. - NameSet distinct_columns = {}; - /// QueryPipeline has single port. Totals or extremes ports are not counted. bool has_single_port = false; @@ -51,8 +46,7 @@ public: bool hasEqualPropertiesWith(const DataStream & other) const { - return distinct_columns == other.distinct_columns - && has_single_port == other.has_single_port + return has_single_port == other.has_single_port && sort_description == other.sort_description && (sort_description.empty() || sort_scope == other.sort_scope); } diff --git a/src/Processors/QueryPlan/ITransformingStep.cpp b/src/Processors/QueryPlan/ITransformingStep.cpp index 195fa9ad68c..9ecfdb0af22 100644 --- a/src/Processors/QueryPlan/ITransformingStep.cpp +++ b/src/Processors/QueryPlan/ITransformingStep.cpp @@ -20,9 +20,6 @@ DataStream ITransformingStep::createOutputStream( { DataStream output_stream{.header = std::move(output_header)}; - if (stream_traits.preserves_distinct_columns) - output_stream.distinct_columns = input_stream.distinct_columns; - output_stream.has_single_port = stream_traits.returns_single_stream || (input_stream.has_single_port && stream_traits.preserves_number_of_streams); @@ -50,21 +47,6 @@ QueryPipelineBuilderPtr ITransformingStep::updatePipeline(QueryPipelineBuilders return std::move(pipelines.front()); } -void ITransformingStep::updateDistinctColumns(const Block & res_header, NameSet & distinct_columns) -{ - if (distinct_columns.empty()) - return; - - for (const auto & column : distinct_columns) - { - if (!res_header.has(column)) - { - distinct_columns.clear(); - break; - } - } -} - void ITransformingStep::describePipeline(FormatSettings & settings) const { IQueryPlanStep::describePipeline(processors, settings); diff --git a/src/Processors/QueryPlan/ITransformingStep.h b/src/Processors/QueryPlan/ITransformingStep.h index 1513b4307f8..77de668fbdb 100644 --- a/src/Processors/QueryPlan/ITransformingStep.h +++ b/src/Processors/QueryPlan/ITransformingStep.h @@ -18,11 +18,6 @@ public: /// They are specified in constructor and cannot be changed. struct DataStreamTraits { - /// Keep distinct_columns unchanged. - /// Examples: true for LimitStep, false for ExpressionStep with ARRAY JOIN - /// It some columns may be removed from result header, call updateDistinctColumns - bool preserves_distinct_columns; - /// True if pipeline has single output port after this step. /// Examples: MergeSortingStep, AggregatingStep bool returns_single_stream; @@ -69,8 +64,6 @@ public: input_streams.emplace_back(std::move(input_stream)); updateOutputStream(); - - updateDistinctColumns(output_stream->header, output_stream->distinct_columns); } void describePipeline(FormatSettings & settings) const override; @@ -83,9 +76,6 @@ public: } protected: - /// Clear distinct_columns if res_header doesn't contain all of them. - static void updateDistinctColumns(const Block & res_header, NameSet & distinct_columns); - /// Create output stream from header and traits. static DataStream createOutputStream( const DataStream & input_stream, diff --git a/src/Processors/QueryPlan/JoinStep.cpp b/src/Processors/QueryPlan/JoinStep.cpp index 6e212a53bc6..2ff8f161e99 100644 --- a/src/Processors/QueryPlan/JoinStep.cpp +++ b/src/Processors/QueryPlan/JoinStep.cpp @@ -83,7 +83,6 @@ static ITransformingStep::Traits getStorageJoinTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = false, .returns_single_stream = false, .preserves_number_of_streams = true, .preserves_sorting = false, diff --git a/src/Processors/QueryPlan/LimitByStep.cpp b/src/Processors/QueryPlan/LimitByStep.cpp index 39086e995fc..8b4abecc12c 100644 --- a/src/Processors/QueryPlan/LimitByStep.cpp +++ b/src/Processors/QueryPlan/LimitByStep.cpp @@ -12,7 +12,6 @@ static ITransformingStep::Traits getTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = true, .returns_single_stream = true, .preserves_number_of_streams = false, .preserves_sorting = true, diff --git a/src/Processors/QueryPlan/LimitStep.cpp b/src/Processors/QueryPlan/LimitStep.cpp index 144ac16f0d5..5e5a7387832 100644 --- a/src/Processors/QueryPlan/LimitStep.cpp +++ b/src/Processors/QueryPlan/LimitStep.cpp @@ -12,7 +12,6 @@ static ITransformingStep::Traits getTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = true, .returns_single_stream = false, .preserves_number_of_streams = true, .preserves_sorting = true, diff --git a/src/Processors/QueryPlan/MergingAggregatedStep.cpp b/src/Processors/QueryPlan/MergingAggregatedStep.cpp index e4fc332a1fd..8b5f21442b1 100644 --- a/src/Processors/QueryPlan/MergingAggregatedStep.cpp +++ b/src/Processors/QueryPlan/MergingAggregatedStep.cpp @@ -24,7 +24,6 @@ static ITransformingStep::Traits getTraits(bool should_produce_results_in_order_ return ITransformingStep::Traits { { - .preserves_distinct_columns = false, .returns_single_stream = should_produce_results_in_order_of_bucket_number, .preserves_number_of_streams = false, .preserves_sorting = false, @@ -62,10 +61,6 @@ MergingAggregatedStep::MergingAggregatedStep( , should_produce_results_in_order_of_bucket_number(should_produce_results_in_order_of_bucket_number_) , memory_bound_merging_of_aggregation_results_enabled(memory_bound_merging_of_aggregation_results_enabled_) { - /// Aggregation keys are distinct - for (const auto & key : params.keys) - output_stream->distinct_columns.insert(key); - if (memoryBoundMergingWillBeUsed() && should_produce_results_in_order_of_bucket_number) { output_stream->sort_description = group_by_sort_description; @@ -157,10 +152,6 @@ void MergingAggregatedStep::describeActions(JSONBuilder::JSONMap & map) const void MergingAggregatedStep::updateOutputStream() { output_stream = createOutputStream(input_streams.front(), params.getHeader(input_streams.front().header, final), getDataStreamTraits()); - - /// Aggregation keys are distinct - for (const auto & key : params.keys) - output_stream->distinct_columns.insert(key); } bool MergingAggregatedStep::memoryBoundMergingWillBeUsed() const diff --git a/src/Processors/QueryPlan/MergingAggregatedStep.h b/src/Processors/QueryPlan/MergingAggregatedStep.h index 2dea289ca89..3a7e2b66183 100644 --- a/src/Processors/QueryPlan/MergingAggregatedStep.h +++ b/src/Processors/QueryPlan/MergingAggregatedStep.h @@ -27,6 +27,7 @@ public: bool memory_bound_merging_of_aggregation_results_enabled_); String getName() const override { return "MergingAggregated"; } + const Aggregator::Params & getParams() const { return params; } void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override; diff --git a/src/Processors/QueryPlan/OffsetStep.cpp b/src/Processors/QueryPlan/OffsetStep.cpp index e0c70ba2f28..4bbe81f9169 100644 --- a/src/Processors/QueryPlan/OffsetStep.cpp +++ b/src/Processors/QueryPlan/OffsetStep.cpp @@ -12,7 +12,6 @@ static ITransformingStep::Traits getTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = true, .returns_single_stream = false, .preserves_number_of_streams = true, .preserves_sorting = true, diff --git a/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp index d584a27f16e..6334594de30 100644 --- a/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp @@ -1,5 +1,7 @@ #include #include +#include +#include #include #include #include @@ -7,6 +9,71 @@ namespace DB::QueryPlanOptimizations { +/// build actions DAG from stack of steps +static ActionsDAGPtr buildActionsForPlanPath(std::vector & dag_stack) +{ + if (dag_stack.empty()) + return nullptr; + + ActionsDAGPtr path_actions = dag_stack.back()->clone(); + dag_stack.pop_back(); + while (!dag_stack.empty()) + { + ActionsDAGPtr clone = dag_stack.back()->clone(); + dag_stack.pop_back(); + path_actions->mergeInplace(std::move(*clone)); + } + return path_actions; +} + +static const ActionsDAG::Node * getOriginalNodeForOutputAlias(const ActionsDAGPtr & actions, const String & output_name) +{ + /// find alias in output + const ActionsDAG::Node * output_alias = nullptr; + for (const auto * node : actions->getOutputs()) + { + if (node->result_name == output_name) + { + output_alias = node; + break; + } + } + if (!output_alias) + return nullptr; + + /// find original(non alias) node it refers to + const ActionsDAG::Node * node = output_alias; + while (node && node->type == ActionsDAG::ActionType::ALIAS) + { + chassert(!node->children.empty()); + node = node->children.front(); + } + if (node && node->type != ActionsDAG::ActionType::INPUT) + return nullptr; + + return node; +} + +static std::set +getOriginalDistinctColumns(const ColumnsWithTypeAndName & distinct_columns, std::vector & dag_stack) +{ + auto actions = buildActionsForPlanPath(dag_stack); + std::set original_distinct_columns; + for (const auto & column : distinct_columns) + { + /// const columns doesn't affect DISTINCT, so skip them + if (isColumnConst(*column.column)) + continue; + + const auto * input_node = getOriginalNodeForOutputAlias(actions, column.name); + if (!input_node) + break; + + original_distinct_columns.insert(input_node->result_name); + } + return original_distinct_columns; +} + size_t tryDistinctReadInOrder(QueryPlan::Node * parent_node) { /// check if it is preliminary distinct node @@ -22,8 +89,10 @@ size_t tryDistinctReadInOrder(QueryPlan::Node * parent_node) /// walk through the plan /// (1) check if nodes below preliminary distinct preserve sorting /// (2) gather transforming steps to update their sorting properties later + /// (3) gather actions DAG to find original names for columns in distinct step later std::vector steps_to_update; QueryPlan::Node * node = parent_node; + std::vector dag_stack; while (!node->children.empty()) { auto * step = dynamic_cast(node->step.get()); @@ -36,6 +105,11 @@ size_t tryDistinctReadInOrder(QueryPlan::Node * parent_node) steps_to_update.push_back(step); + if (const auto * const expr = typeid_cast(step); expr) + dag_stack.push_back(expr->getExpression()); + else if (const auto * const filter = typeid_cast(step); filter) + dag_stack.push_back(filter->getExpression()); + node = node->children.front(); } @@ -50,28 +124,24 @@ size_t tryDistinctReadInOrder(QueryPlan::Node * parent_node) if (read_from_merge_tree->getOutputStream().sort_description.empty()) return 0; - /// find non-const columns in DISTINCT + /// get original names for DISTINCT columns const ColumnsWithTypeAndName & distinct_columns = pre_distinct->getOutputStream().header.getColumnsWithTypeAndName(); - std::set non_const_columns; - for (const auto & column : distinct_columns) - { - if (!isColumnConst(*column.column)) - non_const_columns.emplace(column.name); - } + auto original_distinct_columns = getOriginalDistinctColumns(distinct_columns, dag_stack); - const Names& sorting_key_columns = read_from_merge_tree->getStorageMetadata()->getSortingKeyColumns(); /// check if DISTINCT has the same columns as sorting key + const Names & sorting_key_columns = read_from_merge_tree->getStorageMetadata()->getSortingKeyColumns(); size_t number_of_sorted_distinct_columns = 0; for (const auto & column_name : sorting_key_columns) { - if (non_const_columns.end() == non_const_columns.find(column_name)) + if (!original_distinct_columns.contains(column_name)) break; ++number_of_sorted_distinct_columns; } + /// apply optimization only when distinct columns match or form prefix of sorting key /// todo: check if reading in order optimization would be beneficial when sorting key is prefix of columns in DISTINCT - if (number_of_sorted_distinct_columns != non_const_columns.size()) + if (number_of_sorted_distinct_columns != original_distinct_columns.size()) return 0; /// check if another read in order optimization is already applied diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp index 02725dc3122..c9a0270f6e7 100644 --- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp +++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -100,24 +101,29 @@ namespace logDebug("aggregation_keys", aggregation_keys); logDebug("aggregation_keys size", aggregation_keys.size()); logDebug("distinct_columns size", distinct_columns.size()); - if (aggregation_keys.size() != distinct_columns.size()) - return false; - /// compare columns of two DISTINCTs + std::set original_distinct_columns; for (const auto & column : distinct_columns) { logDebug("distinct column name", column); const auto * alias_node = getOriginalNodeForOutputAlias(path_actions, String(column)); if (!alias_node) { - logDebug("original name for alias is not found for", column); - return false; + logDebug("original name for alias is not found", column); + original_distinct_columns.insert(column); } - - logDebug("alias result name", alias_node->result_name); - if (std::find(cbegin(aggregation_keys), cend(aggregation_keys), alias_node->result_name) == aggregation_keys.cend()) + else { - logDebug("alias result name is not found in aggregation keys", alias_node->result_name); + logDebug("alias result name", alias_node->result_name); + original_distinct_columns.insert(alias_node->result_name); + } + } + /// if aggregation keys are part of distinct columns then rows already distinct + for (const auto & key : aggregation_keys) + { + if (!original_distinct_columns.contains(key)) + { + logDebug("aggregation key NOT found: {}", key); return false; } } @@ -176,7 +182,7 @@ namespace while (!node->children.empty()) { const IQueryPlanStep * current_step = node->step.get(); - if (typeid_cast(current_step)) + if (typeid_cast(current_step) || typeid_cast(current_step)) { aggregation_before_distinct = current_step; break; @@ -208,6 +214,9 @@ namespace if (const auto * aggregating_step = typeid_cast(aggregation_before_distinct); aggregating_step) return compareAggregationKeysWithDistinctColumns(aggregating_step->getParams().keys, distinct_columns, actions); + else if (const auto * merging_aggregated_step = typeid_cast(aggregation_before_distinct); + merging_aggregated_step) + return compareAggregationKeysWithDistinctColumns(merging_aggregated_step->getParams().keys, distinct_columns, actions); } return false; diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 51afe96545d..753bb070c47 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -340,57 +340,55 @@ Pipe ReadFromMergeTree::readFromPool( / max_block_size * max_block_size / fixed_index_granularity; } - bool all_parts_are_remote = true; - bool all_parts_are_local = true; - for (const auto & part : parts_with_range) - { - const bool is_remote = part.data_part->isStoredOnRemoteDisk(); - all_parts_are_local &= !is_remote; - all_parts_are_remote &= is_remote; - } + bool all_parts_are_remote = true; + bool all_parts_are_local = true; + for (const auto & part : parts_with_range) + { + const bool is_remote = part.data_part->isStoredOnRemoteDisk(); + all_parts_are_local &= !is_remote; + all_parts_are_remote &= is_remote; + } - MergeTreeReadPoolPtr pool; + MergeTreeReadPoolPtr pool; - if ((all_parts_are_remote - && settings.allow_prefetched_read_pool_for_remote_filesystem - && MergeTreePrefetchedReadPool::checkReadMethodAllowed(reader_settings.read_settings.remote_fs_method)) - || (all_parts_are_local - && settings.allow_prefetched_read_pool_for_local_filesystem - && MergeTreePrefetchedReadPool::checkReadMethodAllowed(reader_settings.read_settings.local_fs_method))) - { - pool = std::make_shared( - max_streams, - sum_marks, - min_marks_for_concurrent_read, - std::move(parts_with_range), - storage_snapshot, - prewhere_info, - actions_settings, - required_columns, - virt_column_names, - settings.preferred_block_size_bytes, - reader_settings, - context, - use_uncompressed_cache, - all_parts_are_remote, - *data.getSettings()); - } - else - { - pool = std::make_shared( - max_streams, - sum_marks, - min_marks_for_concurrent_read, - std::move(parts_with_range), - storage_snapshot, - prewhere_info, - actions_settings, - reader_settings, - required_columns, - virt_column_names, - context, - false); - } + if ((all_parts_are_remote && settings.allow_prefetched_read_pool_for_remote_filesystem + && MergeTreePrefetchedReadPool::checkReadMethodAllowed(reader_settings.read_settings.remote_fs_method)) + || (all_parts_are_local && settings.allow_prefetched_read_pool_for_local_filesystem + && MergeTreePrefetchedReadPool::checkReadMethodAllowed(reader_settings.read_settings.local_fs_method))) + { + pool = std::make_shared( + max_streams, + sum_marks, + min_marks_for_concurrent_read, + std::move(parts_with_range), + storage_snapshot, + prewhere_info, + actions_settings, + required_columns, + virt_column_names, + settings.preferred_block_size_bytes, + reader_settings, + context, + use_uncompressed_cache, + all_parts_are_remote, + *data.getSettings()); + } + else + { + pool = std::make_shared( + max_streams, + sum_marks, + min_marks_for_concurrent_read, + std::move(parts_with_range), + storage_snapshot, + prewhere_info, + actions_settings, + reader_settings, + required_columns, + virt_column_names, + context, + false); + } auto * logger = &Poco::Logger::get(data.getLogName() + " (SelectExecutor)"); LOG_DEBUG(logger, "Reading approx. {} rows with {} streams", total_rows, max_streams); @@ -1732,6 +1730,36 @@ void ReadFromMergeTree::describeActions(FormatSettings & format_settings) const format_settings.out << prefix << "Parts: " << result.index_stats.back().num_parts_after << '\n'; format_settings.out << prefix << "Granules: " << result.index_stats.back().num_granules_after << '\n'; } + + if (prewhere_info) + { + format_settings.out << prefix << "Prewhere info" << '\n'; + format_settings.out << prefix << "Need filter: " << prewhere_info->need_filter << '\n'; + + prefix.push_back(format_settings.indent_char); + prefix.push_back(format_settings.indent_char); + + if (prewhere_info->prewhere_actions) + { + format_settings.out << prefix << "Prewhere filter" << '\n'; + format_settings.out << prefix << "Prewhere filter column: " << prewhere_info->prewhere_column_name; + if (prewhere_info->remove_prewhere_column) + format_settings.out << " (removed)"; + format_settings.out << '\n'; + + auto expression = std::make_shared(prewhere_info->prewhere_actions); + expression->describeActions(format_settings.out, prefix); + } + + if (prewhere_info->row_level_filter) + { + format_settings.out << prefix << "Row level filter" << '\n'; + format_settings.out << prefix << "Row level filter column: " << prewhere_info->row_level_column_name << '\n'; + + auto expression = std::make_shared(prewhere_info->row_level_filter); + expression->describeActions(format_settings.out, prefix); + } + } } void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const @@ -1743,6 +1771,35 @@ void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const map.add("Parts", result.index_stats.back().num_parts_after); map.add("Granules", result.index_stats.back().num_granules_after); } + + if (prewhere_info) + { + std::unique_ptr prewhere_info_map = std::make_unique(); + prewhere_info_map->add("Need filter", prewhere_info->need_filter); + + if (prewhere_info->prewhere_actions) + { + std::unique_ptr prewhere_filter_map = std::make_unique(); + prewhere_filter_map->add("Prewhere filter column", prewhere_info->prewhere_column_name); + prewhere_filter_map->add("Prewhere filter remove filter column", prewhere_info->remove_prewhere_column); + auto expression = std::make_shared(prewhere_info->prewhere_actions); + prewhere_filter_map->add("Prewhere filter expression", expression->toTree()); + + prewhere_info_map->add("Prewhere filter", std::move(prewhere_filter_map)); + } + + if (prewhere_info->row_level_filter) + { + std::unique_ptr row_level_filter_map = std::make_unique(); + row_level_filter_map->add("Row level filter column", prewhere_info->row_level_column_name); + auto expression = std::make_shared(prewhere_info->row_level_filter); + row_level_filter_map->add("Row level filter expression", expression->toTree()); + + prewhere_info_map->add("Row level filter", std::move(row_level_filter_map)); + } + + map.add("Prewhere info", std::move(prewhere_info_map)); + } } void ReadFromMergeTree::describeIndexes(FormatSettings & format_settings) const diff --git a/src/Processors/QueryPlan/RollupStep.cpp b/src/Processors/QueryPlan/RollupStep.cpp index 3305f24602f..136690ccfc0 100644 --- a/src/Processors/QueryPlan/RollupStep.cpp +++ b/src/Processors/QueryPlan/RollupStep.cpp @@ -11,7 +11,6 @@ static ITransformingStep::Traits getTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = false, .returns_single_stream = true, .preserves_number_of_streams = false, .preserves_sorting = false, @@ -29,9 +28,6 @@ RollupStep::RollupStep(const DataStream & input_stream_, Aggregator::Params para , final(final_) , use_nulls(use_nulls_) { - /// Aggregation keys are distinct - for (const auto & key : params.keys) - output_stream->distinct_columns.insert(key); } ProcessorPtr addGroupingSetForTotals(const Block & header, const Names & keys, bool use_nulls, const BuildQueryPipelineSettings & settings, UInt64 grouping_set_number); @@ -54,10 +50,6 @@ void RollupStep::updateOutputStream() { output_stream = createOutputStream( input_streams.front(), appendGroupingSetColumn(params.getHeader(input_streams.front().header, final)), getDataStreamTraits()); - - /// Aggregation keys are distinct - for (const auto & key : params.keys) - output_stream->distinct_columns.insert(key); } diff --git a/src/Processors/QueryPlan/SortingStep.cpp b/src/Processors/QueryPlan/SortingStep.cpp index 0308e320e3a..0ab8e091e05 100644 --- a/src/Processors/QueryPlan/SortingStep.cpp +++ b/src/Processors/QueryPlan/SortingStep.cpp @@ -45,7 +45,6 @@ static ITransformingStep::Traits getTraits(size_t limit) return ITransformingStep::Traits { { - .preserves_distinct_columns = true, .returns_single_stream = true, .preserves_number_of_streams = false, .preserves_sorting = false, diff --git a/src/Processors/QueryPlan/TotalsHavingStep.cpp b/src/Processors/QueryPlan/TotalsHavingStep.cpp index 63991655426..d1bd70fd0b2 100644 --- a/src/Processors/QueryPlan/TotalsHavingStep.cpp +++ b/src/Processors/QueryPlan/TotalsHavingStep.cpp @@ -14,7 +14,6 @@ static ITransformingStep::Traits getTraits(bool has_filter) return ITransformingStep::Traits { { - .preserves_distinct_columns = true, .returns_single_stream = true, .preserves_number_of_streams = false, .preserves_sorting = true, diff --git a/src/Processors/QueryPlan/WindowStep.cpp b/src/Processors/QueryPlan/WindowStep.cpp index 92e9948c4c7..d313b210854 100644 --- a/src/Processors/QueryPlan/WindowStep.cpp +++ b/src/Processors/QueryPlan/WindowStep.cpp @@ -15,7 +15,6 @@ static ITransformingStep::Traits getTraits() return ITransformingStep::Traits { { - .preserves_distinct_columns = true, .returns_single_stream = false, .preserves_number_of_streams = true, .preserves_sorting = true, diff --git a/src/Processors/Transforms/FillingTransform.cpp b/src/Processors/Transforms/FillingTransform.cpp index 503a40ae0e5..4a729863200 100644 --- a/src/Processors/Transforms/FillingTransform.cpp +++ b/src/Processors/Transforms/FillingTransform.cpp @@ -259,6 +259,114 @@ IProcessor::Status FillingTransform::prepare() return ISimpleTransform::prepare(); } +void FillingTransform::interpolate(const MutableColumns & result_columns, Block & interpolate_block) +{ + if (interpolate_description) + { + interpolate_block.clear(); + + if (!input_positions.empty()) + { + /// populate calculation block with required columns with values from previous row + for (const auto & [col_pos, name_type] : input_positions) + { + MutableColumnPtr column = name_type.type->createColumn(); + const auto * res_column = result_columns[col_pos].get(); + size_t size = res_column->size(); + if (size == 0) /// this is the first row in current chunk + { + /// take value from last row of previous chunk if exists, else use default + if (last_row.size() > col_pos && !last_row[col_pos]->empty()) + column->insertFrom(*last_row[col_pos], 0); + else + column->insertDefault(); + } + else /// take value from previous row of current chunk + column->insertFrom(*res_column, size - 1); + + interpolate_block.insert({std::move(column), name_type.type, name_type.name}); + } + interpolate_actions->execute(interpolate_block); + } + else /// all INTERPOLATE expressions are constants + { + size_t n = 1; + interpolate_actions->execute(interpolate_block, n); + } + } +} + +using MutableColumnRawPtrs = std::vector; + +static void insertFromFillingRow(const MutableColumnRawPtrs & filling_columns, const MutableColumnRawPtrs & interpolate_columns, const MutableColumnRawPtrs & other_columns, + const FillingRow & filling_row, const Block & interpolate_block) +{ + for (size_t i = 0, size = filling_columns.size(); i < size; ++i) + { + if (filling_row[i].isNull()) + filling_columns[i]->insertDefault(); + else + filling_columns[i]->insert(filling_row[i]); + } + + if (size_t size = interpolate_block.columns()) + { + Columns columns = interpolate_block.getColumns(); + for (size_t i = 0; i < size; ++i) + interpolate_columns[i]->insertFrom(*columns[i]->convertToFullColumnIfConst(), 0); + } + else + for (auto * interpolate_column : interpolate_columns) + interpolate_column->insertDefault(); + + for (auto * other_column : other_columns) + other_column->insertDefault(); +} + +static void copyRowFromColumns(const MutableColumnRawPtrs & dest, const Columns & source, size_t row_num) +{ + for (size_t i = 0, size = source.size(); i < size; ++i) + dest[i]->insertFrom(*source[i], row_num); +} + +static void initColumnsByPositions( + const Columns & input_columns, + Columns & input_columns_by_positions, + const MutableColumns & output_columns, + MutableColumnRawPtrs & output_columns_by_position, + const std::vector & positions) +{ + for (size_t pos : positions) + { + input_columns_by_positions.push_back(input_columns[pos]); + output_columns_by_position.push_back(output_columns[pos].get()); + } +} + +void FillingTransform::initColumns( + const Columns & input_columns, + Columns & input_fill_columns, + Columns & input_interpolate_columns, + Columns & input_other_columns, + MutableColumns & output_columns, + MutableColumnRawPtrs & output_fill_columns, + MutableColumnRawPtrs & output_interpolate_columns, + MutableColumnRawPtrs & output_other_columns) +{ + Columns non_const_columns; + non_const_columns.reserve(input_columns.size()); + + for (const auto & column : input_columns) + non_const_columns.push_back(column->convertToFullColumnIfConst()); + + for (const auto & column : non_const_columns) + output_columns.push_back(column->cloneEmpty()->assumeMutable()); + + initColumnsByPositions(non_const_columns, input_fill_columns, output_columns, output_fill_columns, fill_column_positions); + initColumnsByPositions( + non_const_columns, input_interpolate_columns, output_columns, output_interpolate_columns, interpolate_column_positions); + initColumnsByPositions(non_const_columns, input_other_columns, output_columns, output_other_columns, other_column_positions); +} void FillingTransform::transform(Chunk & chunk) { @@ -268,97 +376,58 @@ void FillingTransform::transform(Chunk & chunk) Columns old_fill_columns; Columns old_interpolate_columns; Columns old_other_columns; - MutableColumns res_fill_columns; - MutableColumns res_interpolate_columns; - MutableColumns res_other_columns; - - std::vector> res_map; - res_map.resize(input.getHeader().columns()); - - auto init_columns_by_positions = [&res_map](const Columns & old_columns, Columns & new_columns, - MutableColumns & new_mutable_columns, const Positions & positions) - { - for (size_t pos : positions) - { - auto old_column = old_columns[pos]->convertToFullColumnIfConst(); - new_columns.push_back(old_column); - res_map[pos] = {&new_mutable_columns, new_mutable_columns.size()}; - new_mutable_columns.push_back(old_column->cloneEmpty()->assumeMutable()); - } - }; + MutableColumnRawPtrs res_fill_columns; + MutableColumnRawPtrs res_interpolate_columns; + MutableColumnRawPtrs res_other_columns; + MutableColumns result_columns; Block interpolate_block; - auto interpolate = [&]() - { - if (interpolate_description) - { - interpolate_block.clear(); - - if (!input_positions.empty()) - { - /// populate calculation block with required columns with values from previous row - for (const auto & [col_pos, name_type] : input_positions) - { - MutableColumnPtr column = name_type.type->createColumn(); - auto [res_columns, pos] = res_map[col_pos]; - size_t size = (*res_columns)[pos]->size(); - if (size == 0) /// this is the first row in current chunk - { - /// take value from last row of previous chunk if exists, else use default - if (last_row.size() > col_pos && !last_row[col_pos]->empty()) - column->insertFrom(*last_row[col_pos], 0); - else - column->insertDefault(); - } - else /// take value from previous row of current chunk - column->insertFrom(*(*res_columns)[pos], size - 1); - - interpolate_block.insert({std::move(column), name_type.type, name_type.name}); - } - interpolate_actions->execute(interpolate_block); - } - else /// all INTERPOLATE expressions are constants - { - size_t n = 1; - interpolate_actions->execute(interpolate_block, n); - } - } - }; - if (generate_suffix) { const auto & empty_columns = input.getHeader().getColumns(); - init_columns_by_positions(empty_columns, old_fill_columns, res_fill_columns, fill_column_positions); - init_columns_by_positions(empty_columns, old_interpolate_columns, res_interpolate_columns, interpolate_column_positions); - init_columns_by_positions(empty_columns, old_other_columns, res_other_columns, other_column_positions); + initColumns( + empty_columns, + old_fill_columns, + old_interpolate_columns, + old_other_columns, + result_columns, + res_fill_columns, + res_interpolate_columns, + res_other_columns); if (first) filling_row.initFromDefaults(); if (should_insert_first && filling_row < next_row) { - interpolate(); + interpolate(result_columns, interpolate_block); insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, filling_row, interpolate_block); } - interpolate(); + interpolate(result_columns, interpolate_block); while (filling_row.next(next_row)) { insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, filling_row, interpolate_block); - interpolate(); + interpolate(result_columns, interpolate_block); } - setResultColumns(chunk, res_fill_columns, res_interpolate_columns, res_other_columns); + size_t num_output_rows = result_columns[0]->size(); + chunk.setColumns(std::move(result_columns), num_output_rows); return; } - size_t num_rows = chunk.getNumRows(); + const size_t num_rows = chunk.getNumRows(); auto old_columns = chunk.detachColumns(); - - init_columns_by_positions(old_columns, old_fill_columns, res_fill_columns, fill_column_positions); - init_columns_by_positions(old_columns, old_interpolate_columns, res_interpolate_columns, interpolate_column_positions); - init_columns_by_positions(old_columns, old_other_columns, res_other_columns, other_column_positions); + initColumns( + old_columns, + old_fill_columns, + old_interpolate_columns, + old_other_columns, + result_columns, + res_fill_columns, + res_interpolate_columns, + res_other_columns); if (first) { @@ -372,7 +441,7 @@ void FillingTransform::transform(Chunk & chunk) filling_row.initFromDefaults(i); if (less(fill_from, current_value, filling_row.getDirection(i))) { - interpolate(); + interpolate(result_columns, interpolate_block); insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, filling_row, interpolate_block); } break; @@ -386,7 +455,7 @@ void FillingTransform::transform(Chunk & chunk) { should_insert_first = next_row < filling_row; - for (size_t i = 0; i < filling_row.size(); ++i) + for (size_t i = 0, size = filling_row.size(); i < size; ++i) { auto current_value = (*old_fill_columns[i])[row_ind]; const auto & fill_to = filling_row.getFillDescription(i).fill_to; @@ -401,15 +470,15 @@ void FillingTransform::transform(Chunk & chunk) /// and probably we need to insert it to block. if (should_insert_first && filling_row < next_row) { - interpolate(); + interpolate(result_columns, interpolate_block); insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, filling_row, interpolate_block); } - interpolate(); + interpolate(result_columns, interpolate_block); while (filling_row.next(next_row)) { insertFromFillingRow(res_fill_columns, res_interpolate_columns, res_other_columns, filling_row, interpolate_block); - interpolate(); + interpolate(result_columns, interpolate_block); } copyRowFromColumns(res_fill_columns, old_fill_columns, row_ind); @@ -417,55 +486,24 @@ void FillingTransform::transform(Chunk & chunk) copyRowFromColumns(res_other_columns, old_other_columns, row_ind); } - saveLastRow(res_fill_columns, res_interpolate_columns, res_other_columns); - setResultColumns(chunk, res_fill_columns, res_interpolate_columns, res_other_columns); + saveLastRow(result_columns); + size_t num_output_rows = result_columns[0]->size(); + chunk.setColumns(std::move(result_columns), num_output_rows); } -void FillingTransform::setResultColumns(Chunk & chunk, MutableColumns & fill_columns, MutableColumns & interpolate_columns, MutableColumns & other_columns) const -{ - MutableColumns result_columns(fill_columns.size() + interpolate_columns.size() + other_columns.size()); - /// fill_columns always non-empty. - size_t num_rows = fill_columns[0]->size(); - - for (size_t i = 0, size = fill_columns.size(); i < size; ++i) - result_columns[fill_column_positions[i]] = std::move(fill_columns[i]); - for (size_t i = 0, size = interpolate_columns.size(); i < size; ++i) - result_columns[interpolate_column_positions[i]] = std::move(interpolate_columns[i]); - for (size_t i = 0, size = other_columns.size(); i < size; ++i) - result_columns[other_column_positions[i]] = std::move(other_columns[i]); - - chunk.setColumns(std::move(result_columns), num_rows); -} - -void FillingTransform::saveLastRow(const MutableColumns & fill_columns, const MutableColumns & interpolate_columns, const MutableColumns & other_columns) +void FillingTransform::saveLastRow(const MutableColumns & result_columns) { last_row.clear(); - last_row.resize(fill_columns.size() + interpolate_columns.size() + other_columns.size()); - size_t num_rows = fill_columns[0]->size(); + const size_t num_rows = result_columns[0]->size(); if (num_rows == 0) return; - for (size_t i = 0, size = fill_columns.size(); i < size; ++i) + for (const auto & result_column : result_columns) { - auto column = fill_columns[i]->cloneEmpty(); - column->insertFrom(*fill_columns[i], num_rows - 1); - last_row[fill_column_positions[i]] = std::move(column); - } - - for (size_t i = 0, size = interpolate_columns.size(); i < size; ++i) - { - auto column = interpolate_columns[i]->cloneEmpty(); - column->insertFrom(*interpolate_columns[i], num_rows - 1); - last_row[interpolate_column_positions[i]] = std::move(column); - } - - for (size_t i = 0, size = other_columns.size(); i < size; ++i) - { - auto column = other_columns[i]->cloneEmpty(); - column->insertFrom(*other_columns[i], num_rows - 1); - last_row[other_column_positions[i]] = std::move(column); + auto column = result_column->cloneEmpty(); + column->insertFrom(*result_column, num_rows - 1); + last_row.push_back(std::move(column)); } } - } diff --git a/src/Processors/Transforms/FillingTransform.h b/src/Processors/Transforms/FillingTransform.h index 9304e561ad3..5331254b08c 100644 --- a/src/Processors/Transforms/FillingTransform.h +++ b/src/Processors/Transforms/FillingTransform.h @@ -28,8 +28,19 @@ protected: void transform(Chunk & Chunk) override; private: - void setResultColumns(Chunk & chunk, MutableColumns & fill_columns, MutableColumns & interpolate_columns, MutableColumns & other_columns) const; - void saveLastRow(const MutableColumns & fill_columns, const MutableColumns & interpolate_columns, const MutableColumns & other_columns); + void saveLastRow(const MutableColumns & result_columns); + void interpolate(const MutableColumns& result_columns, Block & interpolate_block); + + using MutableColumnRawPtrs = std::vector; + void initColumns( + const Columns & input_columns, + Columns & input_fill_columns, + Columns & input_interpolate_columns, + Columns & input_other_columns, + MutableColumns & output_columns, + MutableColumnRawPtrs & output_fill_columns, + MutableColumnRawPtrs & output_interpolate_columns, + MutableColumnRawPtrs & output_other_columns); const SortDescription sort_description; /// Contains only columns with WITH FILL. const InterpolateDescriptionPtr interpolate_description; /// Contains INTERPOLATE columns diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp index f6797864b73..e3a69958213 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.cpp +++ b/src/QueryPipeline/RemoteQueryExecutor.cpp @@ -21,6 +21,7 @@ #include #include #include +#include namespace ProfileEvents @@ -602,6 +603,9 @@ void RemoteQueryExecutor::sendExternalTables() for (const auto & table : external_tables) { StoragePtr cur = table.second; + /// Send only temporary tables with StorageMemory + if (!std::dynamic_pointer_cast(cur)) + continue; auto data = std::make_unique(); data->table_name = table.first; diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 2c27879f7b7..665b7bb7695 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -589,11 +589,12 @@ void HTTPHandler::processQuery( /// At least, we should postpone sending of first buffer_size result bytes size_t buffer_size_total = std::max( - params.getParsed("buffer_size", DBMS_DEFAULT_BUFFER_SIZE), static_cast(DBMS_DEFAULT_BUFFER_SIZE)); + params.getParsed("buffer_size", context->getSettingsRef().http_response_buffer_size), + static_cast(DBMS_DEFAULT_BUFFER_SIZE)); /// If it is specified, the whole result will be buffered. /// First ~buffer_size bytes will be buffered in memory, the remaining bytes will be stored in temporary file. - bool buffer_until_eof = params.getParsed("wait_end_of_query", false); + bool buffer_until_eof = params.getParsed("wait_end_of_query", context->getSettingsRef().http_wait_end_of_query); size_t buffer_size_http = DBMS_DEFAULT_BUFFER_SIZE; size_t buffer_size_memory = (buffer_size_total > buffer_size_http) ? buffer_size_total : 0; @@ -782,7 +783,6 @@ void HTTPHandler::processQuery( /// they will be applied in ProcessList::insert() from executeQuery() itself. const auto & query = getQuery(request, params, context); std::unique_ptr in_param = std::make_unique(query); - in = has_external_data ? std::move(in_param) : std::make_unique(*in_param, *in_post_maybe_compressed); /// HTTP response compression is turned on only if the client signalled that they support it /// (using Accept-Encoding header) and 'enable_http_compression' setting is turned on. @@ -832,7 +832,8 @@ void HTTPHandler::processQuery( }); } - customizeContext(request, context); + customizeContext(request, context, *in_post_maybe_compressed); + in = has_external_data ? std::move(in_param) : std::make_unique(*in_param, *in_post_maybe_compressed); executeQuery(*in, *used_output.out_maybe_delayed_and_compressed, /* allow_into_outfile = */ false, context, [&response, this] (const QueryResultDetails & details) @@ -1152,7 +1153,7 @@ bool PredefinedQueryHandler::customizeQueryParam(ContextMutablePtr context, cons return false; } -void PredefinedQueryHandler::customizeContext(HTTPServerRequest & request, ContextMutablePtr context) +void PredefinedQueryHandler::customizeContext(HTTPServerRequest & request, ContextMutablePtr context, ReadBuffer & body) { /// If in the configuration file, the handler's header is regex and contains named capture group /// We will extract regex named capture groups as query parameters @@ -1186,6 +1187,15 @@ void PredefinedQueryHandler::customizeContext(HTTPServerRequest & request, Conte const auto & header_value = request.get(header_name); set_query_params(header_value.data(), header_value.data() + header_value.size(), regex); } + + if (unlikely(receive_params.contains("_request_body") && !context->getQueryParameters().contains("_request_body"))) + { + WriteBufferFromOwnString value; + const auto & settings = context->getSettingsRef(); + + copyDataMaxBytes(body, value, settings.http_max_request_param_data_size); + context->setQueryParameter("_request_body", value.str()); + } } std::string PredefinedQueryHandler::getQuery(HTTPServerRequest & request, HTMLForm & params, ContextMutablePtr context) diff --git a/src/Server/HTTPHandler.h b/src/Server/HTTPHandler.h index fa742ebc8fb..5eda5927538 100644 --- a/src/Server/HTTPHandler.h +++ b/src/Server/HTTPHandler.h @@ -36,7 +36,7 @@ public: void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) override; /// This method is called right before the query execution. - virtual void customizeContext(HTTPServerRequest & /* request */, ContextMutablePtr /* context */) {} + virtual void customizeContext(HTTPServerRequest & /* request */, ContextMutablePtr /* context */, ReadBuffer & /* body */) {} virtual bool customizeQueryParam(ContextMutablePtr context, const std::string & key, const std::string & value) = 0; @@ -163,7 +163,7 @@ public: , const CompiledRegexPtr & url_regex_, const std::unordered_map & header_name_with_regex_ , const std::optional & content_type_override_); - virtual void customizeContext(HTTPServerRequest & request, ContextMutablePtr context) override; + void customizeContext(HTTPServerRequest & request, ContextMutablePtr context, ReadBuffer & body) override; std::string getQuery(HTTPServerRequest & request, HTMLForm & params, ContextMutablePtr context) override; diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index b240c99fc7f..5e20ec40fc0 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -41,6 +41,7 @@ #include #include #include +#include #include #include @@ -49,6 +50,11 @@ #include #include +#if USE_SSL +# include +# include +#endif + #include "Core/Protocol.h" #include "Storages/MergeTree/RequestResponse.h" #include "TCPHandler.h" @@ -1224,6 +1230,22 @@ void TCPHandler::receiveHello() session = makeSession(); auto & client_info = session->getClientInfo(); + +#if USE_SSL + /// Authentication with SSL user certificate + if (dynamic_cast(socket().impl())) + { + Poco::Net::SecureStreamSocket secure_socket(socket()); + if (secure_socket.havePeerCertificate()) + { + session->authenticate( + SSLCertificateCredentials{user, secure_socket.peerCertificate().commonName()}, + getClientAddress(client_info)); + return; + } + } +#endif + session->authenticate(user, password, getClientAddress(client_info)); } @@ -1279,6 +1301,18 @@ void TCPHandler::sendHello() writeStringBinary(exception_message, *out); } } + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2) + { + chassert(!nonce.has_value()); + /// Contains lots of stuff (including time), so this should be enough for NONCE. + nonce.emplace(thread_local_rng()); + writeIntBinary(nonce.value(), *out); + } + else + { + LOG_WARNING(LogFrequencyLimiter(log, 10), + "Using deprecated interserver protocol because the client is too old. Consider upgrading all nodes in cluster."); + } out->next(); } @@ -1459,20 +1493,30 @@ void TCPHandler::receiveQuery() if (client_tcp_protocol_version >= DBMS_MIN_PROTOCOL_VERSION_WITH_PARAMETERS) passed_params.read(*in, settings_format); - /// TODO Unify interserver authentication (and make sure that it's secure enough) if (is_interserver_mode) { client_info.interface = ClientInfo::Interface::TCP_INTERSERVER; #if USE_SSL String cluster_secret = server.context()->getCluster(cluster)->getSecret(); + if (salt.empty() || cluster_secret.empty()) { - auto exception = Exception(ErrorCodes::AUTHENTICATION_FAILED, "Interserver authentication failed"); - session->onAuthenticationFailure(/* user_name */ std::nullopt, socket().peerAddress(), exception); + auto exception = Exception(ErrorCodes::AUTHENTICATION_FAILED, "Interserver authentication failed (no salt/cluster secret)"); + session->onAuthenticationFailure(/* user_name= */ std::nullopt, socket().peerAddress(), exception); + throw exception; /// NOLINT + } + + if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2 && !nonce.has_value()) + { + auto exception = Exception(ErrorCodes::AUTHENTICATION_FAILED, "Interserver authentication failed (no nonce)"); + session->onAuthenticationFailure(/* user_name= */ std::nullopt, socket().peerAddress(), exception); throw exception; /// NOLINT } std::string data(salt); + // For backward compatibility + if (nonce.has_value()) + data += std::to_string(nonce.value()); data += cluster_secret; data += state.query; data += state.query_id; diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h index 47baf16a08b..d5e14295dc3 100644 --- a/src/Server/TCPHandler.h +++ b/src/Server/TCPHandler.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -188,7 +189,10 @@ private: /// For inter-server secret (remote_server.*.secret) bool is_interserver_mode = false; + /// For DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET String salt; + /// For DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2 + std::optional nonce; String cluster; std::mutex task_callback_mutex; diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp index 9bcfff65c95..d50f335c1c9 100644 --- a/src/Storages/IStorage.cpp +++ b/src/Storages/IStorage.cpp @@ -40,7 +40,7 @@ RWLockImpl::LockHolder IStorage::tryLockTimed( { const String type_str = type == RWLockImpl::Type::Read ? "READ" : "WRITE"; throw Exception(ErrorCodes::DEADLOCK_AVOIDED, - "{} locking attempt on \"{}\" has timed out! ({}ms) Possible deadlock avoided. Client should retry.", + "{} locking attempt on \"{}\" has timed out! ({}ms) Possible deadlock avoided. Client should retry", type_str, getStorageID(), acquire_timeout.count()); } return lock_holder; diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index e017c9681e8..ed1b83048e1 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -218,9 +218,10 @@ ReplicatedMergeMutateTaskBase::PrepareResult MergeFromLogEntryTask::prepare() zero_copy_lock = storage.tryCreateZeroCopyExclusiveLock(entry.new_part_name, disk); - if (!zero_copy_lock) + if (!zero_copy_lock || !zero_copy_lock->isLocked()) { LOG_DEBUG(log, "Merge of part {} started by some other replica, will wait it and fetch merged part", entry.new_part_name); + storage.watchZeroCopyLock(entry.new_part_name, disk); /// Don't check for missing part -- it's missing because other replica still not /// finished merge. return PrepareResult{ diff --git a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp index d0469c35cef..84fa9ec2c8e 100644 --- a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp @@ -59,6 +59,7 @@ void MergeTreeBackgroundExecutor::increaseThreadsAndMaxTasksCount(size_t for (size_t number = threads_count; number < new_threads_count; ++number) pool.scheduleOrThrowOnError([this] { threadFunction(); }); + max_tasks_metric.changeTo(2 * new_max_tasks_count); // pending + active max_tasks_count.store(new_max_tasks_count, std::memory_order_relaxed); threads_count = new_threads_count; } diff --git a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h index 9305f36feb5..5c47d20865b 100644 --- a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h +++ b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -247,11 +248,13 @@ public: String name_, size_t threads_count_, size_t max_tasks_count_, - CurrentMetrics::Metric metric_) + CurrentMetrics::Metric metric_, + CurrentMetrics::Metric max_tasks_metric_) : name(name_) , threads_count(threads_count_) , max_tasks_count(max_tasks_count_) , metric(metric_) + , max_tasks_metric(max_tasks_metric_, 2 * max_tasks_count) // active + pending { if (max_tasks_count == 0) throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Task count for MergeTreeBackgroundExecutor must not be zero"); @@ -272,9 +275,10 @@ public: size_t threads_count_, size_t max_tasks_count_, CurrentMetrics::Metric metric_, + CurrentMetrics::Metric max_tasks_metric_, std::string_view policy) requires requires(Queue queue) { queue.updatePolicy(policy); } // Because we use explicit template instantiation - : MergeTreeBackgroundExecutor(name_, threads_count_, max_tasks_count_, metric_) + : MergeTreeBackgroundExecutor(name_, threads_count_, max_tasks_count_, metric_, max_tasks_metric_) { pending.updatePolicy(policy); } @@ -311,6 +315,7 @@ private: size_t threads_count TSA_GUARDED_BY(mutex) = 0; std::atomic max_tasks_count = 0; CurrentMetrics::Metric metric; + CurrentMetrics::Increment max_tasks_metric; void routine(TaskRuntimeDataPtr item); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 2a33f0245cb..e2dc048a0e8 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -5082,12 +5082,8 @@ void MergeTreeData::restorePartFromBackup(std::shared_ptr r if (filename.ends_with(IMergeTreeDataPart::TXN_VERSION_METADATA_FILE_NAME)) continue; - auto backup_entry = backup->readFile(part_path_in_backup_fs / filename); - auto read_buffer = backup_entry->getReadBuffer(); - auto write_buffer = disk->writeFile(temp_part_dir / filename); - copyData(*read_buffer, *write_buffer); - write_buffer->finalize(); - reservation->update(reservation->getSize() - backup_entry->getSize()); + size_t file_size = backup->copyFileToDisk(part_path_in_backup_fs / filename, disk, temp_part_dir / filename); + reservation->update(reservation->getSize() - file_size); } auto single_disk_volume = std::make_shared(disk->getName(), disk, 0); @@ -7488,7 +7484,7 @@ MovePartsOutcome MergeTreeData::movePartsToSpace(const DataPartsVector & parts, if (moving_tagger->parts_to_move.empty()) return MovePartsOutcome::NothingToMove; - return moveParts(moving_tagger); + return moveParts(moving_tagger, true); } MergeTreeData::CurrentlyMovingPartsTaggerPtr MergeTreeData::selectPartsForMove() @@ -7543,7 +7539,7 @@ MergeTreeData::CurrentlyMovingPartsTaggerPtr MergeTreeData::checkPartsForMove(co return std::make_shared(std::move(parts_to_move), *this); } -MovePartsOutcome MergeTreeData::moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagger) +MovePartsOutcome MergeTreeData::moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagger, bool wait_for_move_if_zero_copy) { LOG_INFO(log, "Got {} parts to move.", moving_tagger->parts_to_move.size()); @@ -7592,21 +7588,41 @@ MovePartsOutcome MergeTreeData::moveParts(const CurrentlyMovingPartsTaggerPtr & auto disk = moving_part.reserved_space->getDisk(); if (supportsReplication() && disk->supportZeroCopyReplication() && settings->allow_remote_fs_zero_copy_replication) { - /// If we acquired lock than let's try to move. After one - /// replica will actually move the part from disk to some - /// zero-copy storage other replicas will just fetch - /// metainformation. - if (auto lock = tryCreateZeroCopyExclusiveLock(moving_part.part->name, disk); lock) + /// This loop is not endless, if shutdown called/connection failed/replica became readonly + /// we will return true from waitZeroCopyLock and createZeroCopyLock will return nullopt. + while (true) { - cloned_part = parts_mover.clonePart(moving_part); - parts_mover.swapClonedPart(cloned_part); - } - else - { - /// Move will be retried but with backoff. - LOG_DEBUG(log, "Move of part {} postponed, because zero copy mode enabled and someone other moving this part right now", moving_part.part->name); - result = MovePartsOutcome::MoveWasPostponedBecauseOfZeroCopy; - continue; + /// If we acquired lock than let's try to move. After one + /// replica will actually move the part from disk to some + /// zero-copy storage other replicas will just fetch + /// metainformation. + if (auto lock = tryCreateZeroCopyExclusiveLock(moving_part.part->name, disk); lock) + { + if (lock->isLocked()) + { + cloned_part = parts_mover.clonePart(moving_part); + parts_mover.swapClonedPart(cloned_part); + break; + } + else if (wait_for_move_if_zero_copy) + { + LOG_DEBUG(log, "Other replica is working on move of {}, will wait until lock disappear", moving_part.part->name); + /// Wait and checks not only for timeout but also for shutdown and so on. + while (!waitZeroCopyLockToDisappear(*lock, 3000)) + { + LOG_DEBUG(log, "Waiting until some replica will move {} and zero copy lock disappear", moving_part.part->name); + } + } + else + break; + } + else + { + /// Move will be retried but with backoff. + LOG_DEBUG(log, "Move of part {} postponed, because zero copy mode enabled and someone other moving this part right now", moving_part.part->name); + result = MovePartsOutcome::MoveWasPostponedBecauseOfZeroCopy; + break; + } } } else /// Ordinary move as it should be diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 4a1aafe20b6..bc5e5bc2d91 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -1456,7 +1456,7 @@ private: using CurrentlyMovingPartsTaggerPtr = std::shared_ptr; /// Move selected parts to corresponding disks - MovePartsOutcome moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagger); + MovePartsOutcome moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagger, bool wait_for_move_if_zero_copy=false); /// Select parts for move and disks for them. Used in background moving processes. CurrentlyMovingPartsTaggerPtr selectPartsForMove(); @@ -1511,6 +1511,7 @@ private: /// Create zero-copy exclusive lock for part and disk. Useful for coordination of /// distributed operations which can lead to data duplication. Implemented only in ReplicatedMergeTree. virtual std::optional tryCreateZeroCopyExclusiveLock(const String &, const DiskPtr &) { return std::nullopt; } + virtual bool waitZeroCopyLockToDisappear(const ZeroCopyLock &, size_t) { return false; } /// Remove parts from disk calling part->remove(). Can do it in parallel in case of big set of parts and enabled settings. /// If we fail to remove some part and throw_on_error equal to `true` will throw an exception on the first failed part. diff --git a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp index 7a0b1d03e79..78f68ea72fe 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartChecksum.cpp @@ -46,6 +46,10 @@ void MergeTreeDataPartChecksum::checkEqual(const MergeTreeDataPartChecksum & rhs void MergeTreeDataPartChecksum::checkSize(const IDataPartStorage & storage, const String & name) const { + /// Skip inverted index files, these have a default MergeTreeDataPartChecksum with file_size == 0 + if (name.ends_with(".gin_dict") || name.ends_with(".gin_post") || name.ends_with(".gin_seg") || name.ends_with(".gin_sid")) + return; + if (!storage.exists(name)) throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "{} doesn't exist", fs::path(storage.getRelativePath()) / name); diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp index 4428f6c2bce..a72a947ad56 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp @@ -127,9 +127,11 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare() zero_copy_lock = storage.tryCreateZeroCopyExclusiveLock(entry.new_part_name, disk); - if (!zero_copy_lock) + if (!zero_copy_lock || !zero_copy_lock->isLocked()) { + storage.watchZeroCopyLock(entry.new_part_name, disk); LOG_DEBUG(log, "Mutation of part {} started by some other replica, will wait it and mutated merged part", entry.new_part_name); + return PrepareResult{ .prepared_successfully = false, .need_to_check_missing_part_in_fetch = false, diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.h b/src/Storages/MergeTree/MutateFromLogEntryTask.h index 23c9428faa9..c823df3b999 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.h +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.h @@ -30,7 +30,9 @@ public: UInt64 getPriority() override { return priority; } private: + ReplicatedMergeMutateTaskBase::PrepareResult prepare() override; + bool finalize(ReplicatedMergeMutateTaskBase::PartLogWriter write_part_log) override; bool executeInnerTask() override diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 9e6090c947b..c0aac96dd31 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1370,6 +1370,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( { constexpr auto fmt_string = "Not executing merge/mutation for the part {}, waiting for {} to execute it and will fetch after."; out_postpone_reason = fmt::format(fmt_string, entry.new_part_name, replica_to_execute_merge); + LOG_TEST(log, fmt_string, entry.new_part_name, replica_to_execute_merge); return false; } } diff --git a/src/Storages/MergeTree/ZeroCopyLock.h b/src/Storages/MergeTree/ZeroCopyLock.h index 4e73b27804c..4400ea55b8f 100644 --- a/src/Storages/MergeTree/ZeroCopyLock.h +++ b/src/Storages/MergeTree/ZeroCopyLock.h @@ -14,6 +14,7 @@ struct ZeroCopyLock { ZeroCopyLock(const zkutil::ZooKeeperPtr & zookeeper, const std::string & lock_path, const std::string & lock_message); + bool isLocked() const { return lock->isLocked(); } /// Actual lock std::unique_ptr lock; }; diff --git a/src/Storages/MergeTree/tests/gtest_executor.cpp b/src/Storages/MergeTree/tests/gtest_executor.cpp index e45887da7ef..3a4f147b456 100644 --- a/src/Storages/MergeTree/tests/gtest_executor.cpp +++ b/src/Storages/MergeTree/tests/gtest_executor.cpp @@ -15,6 +15,7 @@ using namespace DB; namespace CurrentMetrics { extern const Metric BackgroundMergesAndMutationsPoolTask; + extern const Metric BackgroundMergesAndMutationsPoolSize; } std::random_device device; @@ -102,7 +103,8 @@ TEST(Executor, Simple) "GTest", 1, // threads 100, // max_tasks - CurrentMetrics::BackgroundMergesAndMutationsPoolTask + CurrentMetrics::BackgroundMergesAndMutationsPoolTask, + CurrentMetrics::BackgroundMergesAndMutationsPoolSize ); String schedule; // mutex is not required because we have a single worker @@ -144,7 +146,8 @@ TEST(Executor, RemoveTasks) "GTest", tasks_kinds, tasks_kinds * batch, - CurrentMetrics::BackgroundMergesAndMutationsPoolTask + CurrentMetrics::BackgroundMergesAndMutationsPoolTask, + CurrentMetrics::BackgroundMergesAndMutationsPoolSize ); for (size_t i = 0; i < batch; ++i) @@ -184,7 +187,8 @@ TEST(Executor, RemoveTasksStress) "GTest", tasks_kinds, tasks_kinds * batch * (schedulers_count + removers_count), - CurrentMetrics::BackgroundMergesAndMutationsPoolTask + CurrentMetrics::BackgroundMergesAndMutationsPoolTask, + CurrentMetrics::BackgroundMergesAndMutationsPoolSize ); std::barrier barrier(schedulers_count + removers_count); @@ -234,7 +238,8 @@ TEST(Executor, UpdatePolicy) "GTest", 1, // threads 100, // max_tasks - CurrentMetrics::BackgroundMergesAndMutationsPoolTask + CurrentMetrics::BackgroundMergesAndMutationsPoolTask, + CurrentMetrics::BackgroundMergesAndMutationsPoolSize ); String schedule; // mutex is not required because we have a single worker diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 706e0e39c7e..0be4ae3a79f 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include @@ -1020,6 +1021,8 @@ QueryTreeNodePtr buildQueryTreeDistributed(SelectQueryInfo & query_info, if (!replacement_map.empty()) query_tree_to_modify = query_tree_to_modify->cloneAndReplace(replacement_map); + removeGroupingFunctionSpecializations(query_tree_to_modify); + return query_tree_to_modify; } diff --git a/src/Storages/StorageJoin.cpp b/src/Storages/StorageJoin.cpp index ab1406f9bd6..dec741beb45 100644 --- a/src/Storages/StorageJoin.cpp +++ b/src/Storages/StorageJoin.cpp @@ -21,21 +21,24 @@ #include #include #include -#include /// toLower +#include +#include +namespace fs = std::filesystem; namespace DB { namespace ErrorCodes { - extern const int NOT_IMPLEMENTED; - extern const int LOGICAL_ERROR; - extern const int UNSUPPORTED_JOIN_KEYS; - extern const int NO_SUCH_COLUMN_IN_TABLE; - extern const int INCOMPATIBLE_TYPE_OF_JOIN; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int BAD_ARGUMENTS; + extern const int DEADLOCK_AVOIDED; + extern const int INCOMPATIBLE_TYPE_OF_JOIN; + extern const int LOGICAL_ERROR; + extern const int NO_SUCH_COLUMN_IN_TABLE; + extern const int NOT_IMPLEMENTED; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int UNSUPPORTED_JOIN_KEYS; } StorageJoin::StorageJoin( @@ -78,6 +81,14 @@ RWLockImpl::LockHolder StorageJoin::tryLockTimedWithContext(const RWLock & lock, return tryLockTimed(lock, type, query_id, acquire_timeout); } +RWLockImpl::LockHolder StorageJoin::tryLockForCurrentQueryTimedWithContext(const RWLock & lock, RWLockImpl::Type type, ContextPtr context) +{ + const String query_id = context ? context->getInitialQueryId() : RWLockImpl::NO_QUERY; + const std::chrono::milliseconds acquire_timeout + = context ? context->getSettingsRef().lock_acquire_timeout : std::chrono::seconds(DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC); + return lock->getLock(type, query_id, acquire_timeout, false); +} + SinkToStoragePtr StorageJoin::write(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) { std::lock_guard mutate_lock(mutate_mutex); @@ -95,7 +106,7 @@ void StorageJoin::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPt LOG_INFO(&Poco::Logger::get("StorageJoin"), "Path {} is already removed from disk {}", path, disk->getName()); disk->createDirectories(path); - disk->createDirectories(path + "tmp/"); + disk->createDirectories(fs::path(path) / "tmp/"); increment = 0; join = std::make_shared(table_join, getRightSampleBlock(), overwrite); @@ -238,8 +249,12 @@ void StorageJoin::insertBlock(const Block & block, ContextPtr context) { Block block_to_insert = block; convertRightBlock(block_to_insert); + TableLockHolder holder = tryLockForCurrentQueryTimedWithContext(rwlock, RWLockImpl::Write, context); + + /// Protection from `INSERT INTO test_table_join SELECT * FROM test_table_join` + if (!holder) + throw Exception(ErrorCodes::DEADLOCK_AVOIDED, "StorageJoin: cannot insert data because current query tries to read from this storage"); - TableLockHolder holder = tryLockTimedWithContext(rwlock, RWLockImpl::Write, context); join->addJoinedBlock(block_to_insert, true); } diff --git a/src/Storages/StorageJoin.h b/src/Storages/StorageJoin.h index 61ea743c841..a5e85d8788a 100644 --- a/src/Storages/StorageJoin.h +++ b/src/Storages/StorageJoin.h @@ -100,12 +100,15 @@ private: /// Protect state for concurrent use in insertFromBlock and joinBlock. /// Lock is stored in HashJoin instance during query and blocks concurrent insertions. mutable RWLock rwlock = RWLockImpl::create(); + mutable std::mutex mutate_mutex; void insertBlock(const Block & block, ContextPtr context) override; void finishInsert() override {} size_t getSize(ContextPtr context) const override; RWLockImpl::LockHolder tryLockTimedWithContext(const RWLock & lock, RWLockImpl::Type type, ContextPtr context) const; + /// Same as tryLockTimedWithContext, but returns `nullptr` if lock is already acquired by current query. + static RWLockImpl::LockHolder tryLockForCurrentQueryTimedWithContext(const RWLock & lock, RWLockImpl::Type type, ContextPtr context); void convertRightBlock(Block & block) const; }; diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index 24108a5c3dd..772ed34b7a9 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -1027,11 +1027,7 @@ void StorageLog::restoreDataImpl(const BackupPtr & backup, const String & data_p if (!backup->fileExists(file_path_in_backup)) throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "File {} in backup is required to restore table", file_path_in_backup); - auto backup_entry = backup->readFile(file_path_in_backup); - auto in = backup_entry->getReadBuffer(); - auto out = disk->writeFile(data_file.path, max_compress_block_size, WriteMode::Append); - copyData(*in, *out); - out->finalize(); + backup->copyFileToDisk(file_path_in_backup, disk, data_file.path, WriteMode::Append); } if (use_marks_file) @@ -1062,8 +1058,7 @@ void StorageLog::restoreDataImpl(const BackupPtr & backup, const String & data_p old_num_rows[i] = num_marks ? data_files[i].marks[num_marks - 1].rows : 0; } - auto backup_entry = backup->readFile(file_path_in_backup); - auto marks_rb = backup_entry->getReadBuffer(); + auto marks_rb = backup->readFile(file_path_in_backup); for (size_t i = 0; i != num_extra_marks; ++i) { diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp index 5df644bf48e..e7bd7cd8e7e 100644 --- a/src/Storages/StorageMemory.cpp +++ b/src/Storages/StorageMemory.cpp @@ -595,8 +595,7 @@ void StorageMemory::restoreDataImpl(const BackupPtr & backup, const String & dat if (!backup->fileExists(index_file_path)) throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "File {} in backup is required to restore table", index_file_path); - auto backup_entry = backup->readFile(index_file_path); - auto in = backup_entry->getReadBuffer(); + auto in = backup->readFile(index_file_path); CompressedReadBuffer compressed_in{*in}; index.read(compressed_in); } @@ -610,8 +609,7 @@ void StorageMemory::restoreDataImpl(const BackupPtr & backup, const String & dat if (!backup->fileExists(data_file_path)) throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "File {} in backup is required to restore table", data_file_path); - auto backup_entry = backup->readFile(data_file_path); - std::unique_ptr in = backup_entry->getReadBuffer(); + auto in = backup->readFile(data_file_path); std::optional temp_data_file; if (!dynamic_cast(in.get())) { diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index af5a32d47f7..0f7b47255f6 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -448,7 +448,7 @@ void ReadFromMerge::initializePipeline(QueryPipelineBuilder & pipeline, const Bu size_t current_need_streams = tables_count >= num_streams ? 1 : (num_streams / tables_count); size_t current_streams = std::min(current_need_streams, remaining_streams); remaining_streams -= current_streams; - current_streams = std::max(static_cast(1), current_streams); + current_streams = std::max(1uz, current_streams); const auto & storage = std::get<1>(table); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index fe4a144deaa..5d0b04e8cb6 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -8555,7 +8555,6 @@ String StorageReplicatedMergeTree::getSharedDataReplica( return best_replica; } - Strings StorageReplicatedMergeTree::getZeroCopyPartPath( const MergeTreeSettings & settings, const std::string & disk_type, const String & table_uuid, const String & part_name, const String & zookeeper_path_old) @@ -8575,18 +8574,65 @@ Strings StorageReplicatedMergeTree::getZeroCopyPartPath( return res; } -bool StorageReplicatedMergeTree::checkZeroCopyLockExists(const String & part_name, const DiskPtr & disk, String & lock_replica) +void StorageReplicatedMergeTree::watchZeroCopyLock(const String & part_name, const DiskPtr & disk) { auto path = getZeroCopyPartPath(part_name, disk); if (path) { - /// FIXME + auto zookeeper = getZooKeeper(); auto lock_path = fs::path(*path) / "part_exclusive_lock"; - if (getZooKeeper()->tryGet(lock_path, lock_replica)) + LOG_TEST(log, "Adding zero-copy lock on {}", lock_path); + /// Looks ugly, but we cannot touch any storage fields inside Watch callback + /// because it could lead to use-after-free (storage dropped and watch triggered) + std::shared_ptr> flag = std::make_shared>(true); + std::string replica; + bool exists = zookeeper->tryGetWatch(lock_path, replica, nullptr, [flag] (const Coordination::WatchResponse &) { - return true; + *flag = false; + }); + + if (exists) + { + std::lock_guard lock(existing_zero_copy_locks_mutex); + existing_zero_copy_locks[lock_path] = ZeroCopyLockDescription{replica, flag}; } } +} + +bool StorageReplicatedMergeTree::checkZeroCopyLockExists(const String & part_name, const DiskPtr & disk, String & lock_replica) +{ + auto path = getZeroCopyPartPath(part_name, disk); + + std::lock_guard lock(existing_zero_copy_locks_mutex); + /// Cleanup abandoned locks during each check. The set of locks is small and this is quite fast loop. + /// Also it's hard to properly remove locks because we can execute replication queue + /// in arbitrary order and some parts can be replaced by covering parts without merges. + for (auto it = existing_zero_copy_locks.begin(); it != existing_zero_copy_locks.end();) + { + if (*it->second.exists) + ++it; + else + { + LOG_TEST(log, "Removing zero-copy lock on {}", it->first); + it = existing_zero_copy_locks.erase(it); + } + } + + if (path) + { + auto lock_path = fs::path(*path) / "part_exclusive_lock"; + if (auto it = existing_zero_copy_locks.find(lock_path); it != existing_zero_copy_locks.end()) + { + lock_replica = it->second.replica; + if (*it->second.exists) + { + LOG_TEST(log, "Zero-copy lock on path {} exists", it->first); + return true; + } + } + + LOG_TEST(log, "Zero-copy lock on path {} doesn't exist", lock_path); + } return false; } @@ -8599,11 +8645,37 @@ std::optional StorageReplicatedMergeTree::getZeroCopyPartPath(const Stri return getZeroCopyPartPath(*getSettings(), toString(disk->getDataSourceDescription().type), getTableSharedID(), part_name, zookeeper_path)[0]; } +bool StorageReplicatedMergeTree::waitZeroCopyLockToDisappear(const ZeroCopyLock & lock, size_t milliseconds_to_wait) +{ + if (lock.isLocked()) + return true; + + if (partial_shutdown_called.load(std::memory_order_relaxed)) + return true; + + auto lock_path = lock.lock->getLockPath(); + zkutil::ZooKeeperPtr zookeeper = tryGetZooKeeper(); + if (!zookeeper) + return true; + + Stopwatch time_waiting; + const auto & stop_waiting = [&]() + { + bool timeout_exceeded = milliseconds_to_wait < time_waiting.elapsedMilliseconds(); + return partial_shutdown_called.load(std::memory_order_relaxed) || is_readonly.load(std::memory_order_relaxed) || timeout_exceeded; + }; + + return zookeeper->waitForDisappear(lock_path, stop_waiting); +} + std::optional StorageReplicatedMergeTree::tryCreateZeroCopyExclusiveLock(const String & part_name, const DiskPtr & disk) { if (!disk || !disk->supportZeroCopyReplication()) return std::nullopt; + if (partial_shutdown_called.load(std::memory_order_relaxed) || is_readonly.load(std::memory_order_relaxed)) + return std::nullopt; + zkutil::ZooKeeperPtr zookeeper = tryGetZooKeeper(); if (!zookeeper) return std::nullopt; @@ -8616,10 +8688,8 @@ std::optional StorageReplicatedMergeTree::tryCreateZeroCopyExclusi /// Create actual lock ZeroCopyLock lock(zookeeper, zc_zookeeper_path, replica_name); - if (lock.lock->tryLock()) - return lock; - else - return std::nullopt; + lock.lock->tryLock(); + return lock; } String StorageReplicatedMergeTree::findReplicaHavingPart( diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 46c78e9064a..70f5ca0caac 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -482,6 +482,16 @@ private: std::mutex last_broken_disks_mutex; std::set last_broken_disks; + std::mutex existing_zero_copy_locks_mutex; + + struct ZeroCopyLockDescription + { + std::string replica; + std::shared_ptr> exists; + }; + + std::unordered_map existing_zero_copy_locks; + static std::optional distributedWriteFromClusterStorage(const std::shared_ptr & src_storage_cluster, const ASTInsertQuery & query, ContextPtr context); template @@ -862,13 +872,19 @@ private: void createTableSharedID() const; bool checkZeroCopyLockExists(const String & part_name, const DiskPtr & disk, String & lock_replica); + void watchZeroCopyLock(const String & part_name, const DiskPtr & disk); std::optional getZeroCopyPartPath(const String & part_name, const DiskPtr & disk); /// Create ephemeral lock in zookeeper for part and disk which support zero copy replication. - /// If somebody already holding the lock -- return std::nullopt. + /// If no connection to zookeeper, shutdown, readonly -- return std::nullopt. + /// If somebody already holding the lock -- return unlocked ZeroCopyLock object (not std::nullopt). std::optional tryCreateZeroCopyExclusiveLock(const String & part_name, const DiskPtr & disk) override; + /// Wait for ephemral lock to disappear. Return true if table shutdown/readonly/timeout exceeded, etc. + /// Or if node actually disappeared. + bool waitZeroCopyLockToDisappear(const ZeroCopyLock & lock, size_t milliseconds_to_wait) override; + void startupImpl(bool from_attach_thread); }; diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index 870f6b96ae6..30585250be2 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -622,11 +622,7 @@ void StorageStripeLog::restoreDataImpl(const BackupPtr & backup, const String & if (!backup->fileExists(file_path_in_backup)) throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "File {} in backup is required to restore table", file_path_in_backup); - auto backup_entry = backup->readFile(file_path_in_backup); - auto in = backup_entry->getReadBuffer(); - auto out = disk->writeFile(data_file_path, max_compress_block_size, WriteMode::Append); - copyData(*in, *out); - out->finalize(); + backup->copyFileToDisk(file_path_in_backup, disk, data_file_path, WriteMode::Append); } /// Append the index. @@ -636,8 +632,7 @@ void StorageStripeLog::restoreDataImpl(const BackupPtr & backup, const String & if (!backup->fileExists(index_path_in_backup)) throw Exception(ErrorCodes::CANNOT_RESTORE_TABLE, "File {} in backup is required to restore table", index_path_in_backup); - auto backup_entry = backup->readFile(index_path_in_backup); - auto index_in = backup_entry->getReadBuffer(); + auto index_in = backup->readFile(index_path_in_backup); CompressedReadBuffer index_compressed_in{*index_in}; extra_indices.read(index_compressed_in); diff --git a/src/Storages/System/StorageSystemMarkedDroppedTables.cpp b/src/Storages/System/StorageSystemMarkedDroppedTables.cpp new file mode 100644 index 00000000000..fcdd6e1edcf --- /dev/null +++ b/src/Storages/System/StorageSystemMarkedDroppedTables.cpp @@ -0,0 +1,64 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "base/types.h" + + +namespace DB +{ + +NamesAndTypesList StorageSystemMarkedDroppedTables::getNamesAndTypes() +{ + NamesAndTypesList names_and_types{ + {"index", std::make_shared()}, + {"database", std::make_shared()}, + {"table", std::make_shared()}, + {"uuid", std::make_shared()}, + {"engine", std::make_shared()}, + {"metadata_dropped_path", std::make_shared()}, + {"table_dropped_time", std::make_shared()}, + }; + return names_and_types; +} + + +void StorageSystemMarkedDroppedTables::fillData(MutableColumns & res_columns, ContextPtr, const SelectQueryInfo &) const +{ + auto tables_mark_dropped = DatabaseCatalog::instance().getTablesMarkedDropped(); + + size_t index = 0; + + auto & column_index = assert_cast(*res_columns[index++]); + auto & column_database = assert_cast(*res_columns[index++]); + auto & column_table = assert_cast(*res_columns[index++]); + auto & column_uuid = assert_cast(*res_columns[index++]).getData(); + auto & column_engine = assert_cast(*res_columns[index++]); + auto & column_metadata_dropped_path = assert_cast(*res_columns[index++]); + auto & column_table_dropped_time = assert_cast(*res_columns[index++]); + + auto add_row = [&](UInt32 idx, const DatabaseCatalog::TableMarkedAsDropped & table_mark_dropped) + { + column_index.insertValue(idx); + column_database.insertData(table_mark_dropped.table_id.getDatabaseName().data(), table_mark_dropped.table_id.getDatabaseName().size()); + column_table.insertData(table_mark_dropped.table_id.getTableName().data(), table_mark_dropped.table_id.getTableName().size()); + column_uuid.push_back(table_mark_dropped.table_id.uuid.toUnderType()); + if (table_mark_dropped.table) + column_engine.insertData(table_mark_dropped.table->getName().data(), table_mark_dropped.table->getName().size()); + else + column_engine.insertData({}, 0); + column_metadata_dropped_path.insertData(table_mark_dropped.metadata_path.data(), table_mark_dropped.metadata_path.size()); + column_table_dropped_time.insertValue(static_cast(table_mark_dropped.drop_time)); + }; + + UInt32 idx = 0; + for (const auto & table_mark_dropped : tables_mark_dropped) + add_row(idx++, table_mark_dropped); +} + +} diff --git a/src/Storages/System/StorageSystemMarkedDroppedTables.h b/src/Storages/System/StorageSystemMarkedDroppedTables.h new file mode 100644 index 00000000000..ea2a864311c --- /dev/null +++ b/src/Storages/System/StorageSystemMarkedDroppedTables.h @@ -0,0 +1,20 @@ +#pragma once + +#include + + +namespace DB +{ + +class StorageSystemMarkedDroppedTables final : public IStorageSystemOneBlock +{ +public: + std::string getName() const override { return "SystemMarkedDroppedTables"; } + static NamesAndTypesList getNamesAndTypes(); + +protected: + using IStorageSystemOneBlock::IStorageSystemOneBlock; + void fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo &) const override; +}; + +} diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 61329ab834b..fd1cf2f1623 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -79,6 +79,7 @@ #include #include #include +#include #ifdef OS_LINUX #include @@ -140,6 +141,7 @@ void attachSystemTablesLocal(ContextPtr context, IDatabase & system_database) attach(context, system_database, "time_zones"); attach(context, system_database, "backups"); attach(context, system_database, "schema_inference_cache"); + attach(context, system_database, "marked_dropped_tables"); #ifdef OS_LINUX attach(context, system_database, "stack_trace"); #endif diff --git a/src/Storages/removeGroupingFunctionSpecializations.cpp b/src/Storages/removeGroupingFunctionSpecializations.cpp new file mode 100644 index 00000000000..668596756a1 --- /dev/null +++ b/src/Storages/removeGroupingFunctionSpecializations.cpp @@ -0,0 +1,65 @@ +#include + +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +class GeneralizeGroupingFunctionForDistributedVisitor : public InDepthQueryTreeVisitor +{ +public: + static void visitImpl(QueryTreeNodePtr & node) + { + auto * function = node->as(); + if (!function) + return; + + const auto & function_name = function->getFunctionName(); + bool ordinary_grouping = function_name == "groupingOrdinary"; + + if (!ordinary_grouping + && function_name != "groupingForRollup" + && function_name != "groupingForCube" + && function_name != "groupingForGroupingSets") + return; + + + if (!ordinary_grouping) + { + auto & arguments = function->getArguments().getNodes(); + + if (arguments.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Grouping function specialization must have arguments"); + auto * grouping_set_arg = arguments[0]->as(); + if (!grouping_set_arg || grouping_set_arg->getColumnName() != "__grouping_set") + throw Exception(ErrorCodes::LOGICAL_ERROR, + "The first argument of Grouping function specialization must be '__grouping_set' column but {} found", + arguments[0]->dumpTree()); + arguments.erase(arguments.begin()); + } + + // This node will be only converted to AST, so we don't need + // to pass the correct force_compatibility flag to FunctionGrouping. + auto function_adaptor = std::make_shared( + std::make_shared(false) + ); + function->resolveAsFunction(function_adaptor); + } +}; + +void removeGroupingFunctionSpecializations(QueryTreeNodePtr & node) +{ + GeneralizeGroupingFunctionForDistributedVisitor visitor; + visitor.visit(node); +} + +} diff --git a/src/Storages/removeGroupingFunctionSpecializations.h b/src/Storages/removeGroupingFunctionSpecializations.h new file mode 100644 index 00000000000..878b87abce7 --- /dev/null +++ b/src/Storages/removeGroupingFunctionSpecializations.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +namespace DB +{ + +void removeGroupingFunctionSpecializations(QueryTreeNodePtr & node); + +} diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 44c1517a9f5..aec52981724 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -437,7 +437,7 @@ class FailureReason(enum.Enum): SERVER_DIED = "server died" EXIT_CODE = "return code: " STDERR = "having stderror: " - EXCEPTION = "having having exception in stdout: " + EXCEPTION = "having exception in stdout: " RESULT_DIFF = "result differs with reference: " TOO_LONG = "Test runs too long (> 60s). Make it faster." INTERNAL_QUERY_FAIL = "Internal query (CREATE/DROP DATABASE) failed:" @@ -523,6 +523,8 @@ class SettingsRandomizer: "merge_tree_coarse_index_granularity": lambda: random.randint(2, 32), "optimize_distinct_in_order": lambda: random.randint(0, 1), "optimize_sorting_by_input_stream_properties": lambda: random.randint(0, 1), + "http_response_buffer_size": lambda: random.randint(0, 10 * 1048576), + "http_wait_end_of_query": lambda: random.random() > 0.5, "enable_memory_bound_merging_of_aggregation_results": lambda: random.randint( 0, 1 ), @@ -2246,7 +2248,8 @@ def parse_args(): parser.add_argument( "-b", "--binary", - default=find_binary("clickhouse"), + default="clickhouse", + type=find_binary, help="Path to clickhouse binary or name of binary in PATH", ) parser.add_argument( diff --git a/tests/integration/test_backup_restore_s3/test.py b/tests/integration/test_backup_restore_s3/test.py index d5a7579df51..d046c97f8d1 100644 --- a/tests/integration/test_backup_restore_s3/test.py +++ b/tests/integration/test_backup_restore_s3/test.py @@ -138,7 +138,8 @@ def test_backup_to_s3_native_copy(): f"S3('http://minio1:9001/root/data/backups/{backup_name}', 'minio', 'minio123')" ) check_backup_and_restore(storage_policy, backup_destination) - assert node.contains_in_log("using native copy") + assert node.contains_in_log("BackupImpl.*using native copy") + assert node.contains_in_log("copyS3FileToDisk.*using native copy") assert node.contains_in_log( f"copyS3File: Single operation copy has completed. Bucket: root, Key: data/backups/{backup_name}" ) @@ -151,7 +152,8 @@ def test_backup_to_s3_native_copy_other_bucket(): f"S3('http://minio1:9001/root/data/backups/{backup_name}', 'minio', 'minio123')" ) check_backup_and_restore(storage_policy, backup_destination) - assert node.contains_in_log("using native copy") + assert node.contains_in_log("BackupImpl.*using native copy") + assert node.contains_in_log("copyS3FileToDisk.*using native copy") assert node.contains_in_log( f"copyS3File: Single operation copy has completed. Bucket: root, Key: data/backups/{backup_name}" ) @@ -162,7 +164,8 @@ def test_backup_to_s3_native_copy_multipart(): backup_name = new_backup_name() backup_destination = f"S3('http://minio1:9001/root/data/backups/multipart/{backup_name}', 'minio', 'minio123')" check_backup_and_restore(storage_policy, backup_destination, size=1000000) - assert node.contains_in_log("using native copy") + assert node.contains_in_log("BackupImpl.*using native copy") + assert node.contains_in_log("copyS3FileToDisk.*using native copy") assert node.contains_in_log( f"copyS3File: Multipart upload has completed. Bucket: root, Key: data/backups/multipart/{backup_name}/" ) diff --git a/tests/integration/test_distributed_inter_server_secret/configs/remote_servers.xml b/tests/integration/test_distributed_inter_server_secret/configs/remote_servers.xml index f5888a974f1..163850c6f2d 100644 --- a/tests/integration/test_distributed_inter_server_secret/configs/remote_servers.xml +++ b/tests/integration/test_distributed_inter_server_secret/configs/remote_servers.xml @@ -24,5 +24,17 @@ 1 + + + foo_backward + + n1 + 9000 + + + backward + 9000 + + diff --git a/tests/integration/test_distributed_inter_server_secret/configs/remote_servers_backward.xml b/tests/integration/test_distributed_inter_server_secret/configs/remote_servers_backward.xml new file mode 100644 index 00000000000..3636b402145 --- /dev/null +++ b/tests/integration/test_distributed_inter_server_secret/configs/remote_servers_backward.xml @@ -0,0 +1,15 @@ + + + + backward + + n1 + 9000 + + + backward + 9000 + + + + diff --git a/tests/integration/test_distributed_inter_server_secret/test.py b/tests/integration/test_distributed_inter_server_secret/test.py index 4f1c4b9b749..6dd25789f36 100644 --- a/tests/integration/test_distributed_inter_server_secret/test.py +++ b/tests/integration/test_distributed_inter_server_secret/test.py @@ -12,18 +12,28 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) -def make_instance(name, cfg): +def make_instance(name, cfg, *args, **kwargs): return cluster.add_instance( name, with_zookeeper=True, main_configs=["configs/remote_servers.xml", cfg], user_configs=["configs/users.xml"], + *args, + **kwargs, ) # _n1/_n2 contains cluster with different -- should fail n1 = make_instance("n1", "configs/remote_servers_n1.xml") n2 = make_instance("n2", "configs/remote_servers_n2.xml") +backward = make_instance( + "backward", + "configs/remote_servers_backward.xml", + image="clickhouse/clickhouse-server", + # version without DBMS_MIN_REVISION_WITH_INTERSERVER_SECRET_V2 + tag="23.2.3", + with_installed_binary=True, +) users = pytest.mark.parametrize( "user,password", @@ -54,6 +64,12 @@ def bootstrap(): Engine=Distributed(secure, currentDatabase(), data, key) """ ) + n.query( + """ + CREATE TABLE dist_secure_backward AS data + Engine=Distributed(secure_backward, currentDatabase(), data, key) + """ + ) n.query( """ CREATE TABLE dist_secure_from_buffer AS data_from_buffer @@ -409,3 +425,31 @@ def test_per_user_protocol_settings_secure_cluster(user, password): assert int(get_query_setting_on_shard(n1, id_, "max_memory_usage_for_user")) == int( 1e9 ) + + +@users +def test_user_secure_cluster_with_backward(user, password): + id_ = "with-backward-query-dist_secure-" + user + query_with_id( + n1, id_, "SELECT * FROM dist_secure_backward", user=user, password=password + ) + assert get_query_user_info(n1, id_) == [user, user] + assert get_query_user_info(backward, id_) == [user, user] + + +@users +def test_user_secure_cluster_from_backward(user, password): + id_ = "from-backward-query-dist_secure-" + user + query_with_id( + backward, + id_, + "SELECT * FROM dist_secure_backward", + user=user, + password=password, + ) + assert get_query_user_info(n1, id_) == [user, user] + assert get_query_user_info(backward, id_) == [user, user] + + assert n1.contains_in_log( + "Using deprecated interserver protocol because the client is too old. Consider upgrading all nodes in cluster." + ) diff --git a/tests/integration/test_http_handlers_config/test.py b/tests/integration/test_http_handlers_config/test.py index 9c64bd41b23..1b347f6271f 100644 --- a/tests/integration/test_http_handlers_config/test.py +++ b/tests/integration/test_http_handlers_config/test.py @@ -147,6 +147,18 @@ def test_predefined_query_handler(): assert b"max_final_threads\t1\nmax_threads\t1\n" == res2.content assert "application/generic+one" == res2.headers["content-type"] + cluster.instance.query( + "CREATE TABLE test_table (id UInt32, data String) Engine=TinyLog" + ) + res3 = cluster.instance.http_request( + "test_predefined_handler_post_body?id=100", + method="POST", + data="TEST".encode("utf8"), + ) + assert res3.status_code == 200 + assert cluster.instance.query("SELECT * FROM test_table") == "100\tTEST\n" + cluster.instance.query("DROP TABLE test_table") + def test_fixed_static_handler(): with contextlib.closing( diff --git a/tests/integration/test_http_handlers_config/test_predefined_handler/config.xml b/tests/integration/test_http_handlers_config/test_predefined_handler/config.xml index 5b5de63356e..1b8ddfab323 100644 --- a/tests/integration/test_http_handlers_config/test_predefined_handler/config.xml +++ b/tests/integration/test_http_handlers_config/test_predefined_handler/config.xml @@ -21,5 +21,13 @@ application/generic+one + + POST + /test_predefined_handler_post_body + + predefined_query_handler + INSERT INTO test_table(id, data) SELECT {id:UInt32}, {_request_body:String} + + diff --git a/tests/integration/test_ssl_cert_authentication/configs/ssl_config.xml b/tests/integration/test_ssl_cert_authentication/configs/ssl_config.xml index 163449872be..ed3b2b595db 100644 --- a/tests/integration/test_ssl_cert_authentication/configs/ssl_config.xml +++ b/tests/integration/test_ssl_cert_authentication/configs/ssl_config.xml @@ -9,7 +9,7 @@ You have to configure certificate to enable this interface. See the openSSL section below. --> - + 9440 diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 0246b835fd5..7c62ca0d8b6 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -1,9 +1,12 @@ import pytest +from helpers.client import Client from helpers.cluster import ClickHouseCluster from helpers.ssl_context import WrapSSLContextWithSNI import urllib.request, urllib.parse import ssl import os.path +from os import remove + # The test cluster is configured with certificate for that host name, see 'server-ext.cnf'. # The client have to verify server certificate against that name. Client uses SNI @@ -66,6 +69,54 @@ def execute_query_https( return response.decode("utf-8") +config = """ + + + none + + {certificateFile} + {privateKeyFile} + {caConfig} + + + AcceptCertificateHandler + + + +""" + + +def execute_query_native(node, query, user, cert_name): + + config_path = f"{SCRIPT_DIR}/configs/client.xml" + + formatted = config.format( + certificateFile=f"{SCRIPT_DIR}/certs/{cert_name}-cert.pem", + privateKeyFile=f"{SCRIPT_DIR}/certs/{cert_name}-key.pem", + caConfig=f"{SCRIPT_DIR}/certs/ca-cert.pem", + ) + + file = open(config_path, "w") + file.write(formatted) + file.close() + + client = Client( + node.ip_address, + 9440, + command=cluster.client_bin_path, + secure=True, + config=config_path, + ) + + try: + result = client.query(query, user=user) + remove(config_path) + return result + except: + remove(config_path) + raise + + def test_https(): assert ( execute_query_https("SELECT currentUser()", user="john", cert_name="client1") @@ -81,6 +132,27 @@ def test_https(): ) +def test_native(): + assert ( + execute_query_native( + instance, "SELECT currentUser()", user="john", cert_name="client1" + ) + == "john\n" + ) + assert ( + execute_query_native( + instance, "SELECT currentUser()", user="lucy", cert_name="client2" + ) + == "lucy\n" + ) + assert ( + execute_query_native( + instance, "SELECT currentUser()", user="lucy", cert_name="client3" + ) + == "lucy\n" + ) + + def test_https_wrong_cert(): # Wrong certificate: different user's certificate with pytest.raises(Exception) as err: @@ -107,6 +179,23 @@ def test_https_wrong_cert(): ) +def test_native_wrong_cert(): + # Wrong certificate: different user's certificate + with pytest.raises(Exception) as err: + execute_query_native( + instance, "SELECT currentUser()", user="john", cert_name="client2" + ) + assert "AUTHENTICATION_FAILED" in str(err.value) + + # Wrong certificate: self-signed certificate. + # In this case clickhouse-client itself will throw an error + with pytest.raises(Exception) as err: + execute_query_native( + instance, "SELECT currentUser()", user="john", cert_name="wrong" + ) + assert "UNKNOWN_CA" in str(err.value) + + def test_https_non_ssl_auth(): # Users with non-SSL authentication are allowed, in this case we can skip sending a client certificate at all (because "verificationMode" is set to "relaxed"). # assert execute_query_https("SELECT currentUser()", user="peter", enable_ssl_auth=False) == "peter\n" diff --git a/tests/integration/test_storage_kerberized_hdfs/secrets/krb.conf b/tests/integration/test_storage_kerberized_hdfs/secrets/krb.conf index b43a54b4dc5..dffdcaebe81 100644 --- a/tests/integration/test_storage_kerberized_hdfs/secrets/krb.conf +++ b/tests/integration/test_storage_kerberized_hdfs/secrets/krb.conf @@ -9,6 +9,7 @@ dns_lookup_kdc = false ticket_lifetime = 5s forwardable = true + rdns = false default_tgs_enctypes = des3-hmac-sha1 default_tkt_enctypes = des3-hmac-sha1 permitted_enctypes = des3-hmac-sha1 diff --git a/tests/integration/test_storage_kerberized_kafka/secrets/krb.conf b/tests/integration/test_storage_kerberized_kafka/secrets/krb.conf index 1efdf510f22..bda73a285cf 100644 --- a/tests/integration/test_storage_kerberized_kafka/secrets/krb.conf +++ b/tests/integration/test_storage_kerberized_kafka/secrets/krb.conf @@ -10,6 +10,7 @@ ticket_lifetime = 15s renew_lifetime = 15s forwardable = true + rdns = false [realms] TEST.CLICKHOUSE.TECH = { diff --git a/tests/integration/test_zero_copy_fetch/configs/storage_conf.xml b/tests/integration/test_zero_copy_fetch/configs/storage_conf.xml index 257ae0a355c..b3ce0735a3c 100644 --- a/tests/integration/test_zero_copy_fetch/configs/storage_conf.xml +++ b/tests/integration/test_zero_copy_fetch/configs/storage_conf.xml @@ -1,4 +1,8 @@ + + test + + @@ -21,6 +25,13 @@ + + +
+ s3 +
+ + diff --git a/tests/integration/test_zero_copy_fetch/test.py b/tests/integration/test_zero_copy_fetch/test.py index f13eac5e9d1..b71752528d3 100644 --- a/tests/integration/test_zero_copy_fetch/test.py +++ b/tests/integration/test_zero_copy_fetch/test.py @@ -5,6 +5,7 @@ import random import string import time +from multiprocessing.dummy import Pool import pytest from helpers.cluster import ClickHouseCluster @@ -102,3 +103,133 @@ SETTINGS index_granularity = 8192, storage_policy = 's3'""" assert part_to_disk["20230102_0_0_0"] == "s3" assert part_to_disk["20230109_0_0_0"] == "s3" assert part_to_disk["20230116_0_0_0"] == "default" + + +def test_concurrent_move_to_s3(started_cluster): + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + + node1.query( + """ +CREATE TABLE test_concurrent_move (EventDate Date, CounterID UInt32) +ENGINE = ReplicatedMergeTree('/clickhouse-tables/test_concurrent_move', 'r1') +PARTITION BY CounterID +ORDER BY (CounterID, EventDate) +SETTINGS index_granularity = 8192, storage_policy = 's3'""" + ) + + node2.query( + """ +CREATE TABLE test_concurrent_move (EventDate Date, CounterID UInt32) +ENGINE = ReplicatedMergeTree('/clickhouse-tables/test_concurrent_move', 'r2') +PARTITION BY CounterID +ORDER BY (CounterID, EventDate) +SETTINGS index_granularity = 8192, storage_policy = 's3'""" + ) + partitions = range(10) + + for i in partitions: + node1.query( + f"INSERT INTO test_concurrent_move SELECT toDate('2023-01-01') + toIntervalDay(number), {i} from system.numbers limit 20" + ) + node1.query( + f"INSERT INTO test_concurrent_move SELECT toDate('2023-01-01') + toIntervalDay(number) + rand(), {i} from system.numbers limit 20" + ) + node1.query( + f"INSERT INTO test_concurrent_move SELECT toDate('2023-01-01') + toIntervalDay(number) + rand(), {i} from system.numbers limit 20" + ) + node1.query( + f"INSERT INTO test_concurrent_move SELECT toDate('2023-01-01') + toIntervalDay(number) + rand(), {i} from system.numbers limit 20" + ) + + node2.query("SYSTEM SYNC REPLICA test_concurrent_move") + + # check that we can move parts concurrently without exceptions + p = Pool(3) + for i in partitions: + + def move_partition_to_s3(node): + node.query( + f"ALTER TABLE test_concurrent_move MOVE PARTITION '{i}' TO DISK 's3'" + ) + + j1 = p.apply_async(move_partition_to_s3, (node1,)) + j2 = p.apply_async(move_partition_to_s3, (node2,)) + j1.get() + j2.get() + + def get_part_to_disk(query_result): + part_to_disk = {} + for row in query_result.strip().split("\n"): + disk, part = row.split("\t") + part_to_disk[part] = disk + return part_to_disk + + part_to_disk = get_part_to_disk( + node1.query( + "SELECT disk_name, name FROM system.parts where table = 'test_concurrent_move' and active" + ) + ) + + assert all([value == "s3" for value in part_to_disk.values()]) + + part_to_disk = get_part_to_disk( + node2.query( + "SELECT disk_name, name FROM system.parts where table = 'test_concurrent_move' and active" + ) + ) + assert all([value == "s3" for value in part_to_disk.values()]) + + +def test_zero_copy_mutation(started_cluster): + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + + node1.query( + """ +CREATE TABLE test_zero_copy_mutation (EventDate Date, CounterID UInt32) +ENGINE = ReplicatedMergeTree('/clickhouse-tables/test_zero_copy_mutation', 'r1') +ORDER BY (CounterID, EventDate) +SETTINGS index_granularity = 8192, storage_policy = 's3_only'""" + ) + + node2.query( + """ +CREATE TABLE test_zero_copy_mutation (EventDate Date, CounterID UInt32) +ENGINE = ReplicatedMergeTree('/clickhouse-tables/test_zero_copy_mutation', 'r2') +ORDER BY (CounterID, EventDate) +SETTINGS index_granularity = 8192, storage_policy = 's3_only'""" + ) + + node1.query( + "INSERT INTO test_zero_copy_mutation SELECT toDate('2023-01-01') + toIntervalDay(number) + rand(), number * number from system.numbers limit 10" + ) + + node2.query("SYSTEM STOP REPLICATION QUEUES test_zero_copy_mutation") + p = Pool(3) + + def run_long_mutation(node): + node1.query( + "ALTER TABLE test_zero_copy_mutation DELETE WHERE sleepEachRow(1) == 1" + ) + + job = p.apply_async(run_long_mutation, (node1,)) + + for i in range(30): + count = node1.query( + "SELECT count() FROM system.replication_queue WHERE type = 'MUTATE_PART'" + ).strip() + if int(count) > 0: + break + else: + time.sleep(0.1) + + node2.query("SYSTEM START REPLICATION QUEUES test_zero_copy_mutation") + + node2.query("SYSTEM SYNC REPLICA test_zero_copy_mutation") + + job.get() + + assert node2.contains_in_log("all_0_0_0_1/part_exclusive_lock exists") + assert node2.contains_in_log("Removing zero-copy lock on") + assert node2.contains_in_log("all_0_0_0_1/part_exclusive_lock doesn't exist") diff --git a/tests/queries/0_stateless/00534_functions_bad_arguments5.sh b/tests/queries/0_stateless/00534_functions_bad_arguments5.sh index 7b180870443..a8b0ce77677 100755 --- a/tests/queries/0_stateless/00534_functions_bad_arguments5.sh +++ b/tests/queries/0_stateless/00534_functions_bad_arguments5.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-tsan, no-debug +# Tags: no-tsan, no-debug, no-fasttest # Tag no-tsan: Too long for TSan # shellcheck disable=SC2016 diff --git a/tests/queries/0_stateless/00534_functions_bad_arguments7.sh b/tests/queries/0_stateless/00534_functions_bad_arguments7.sh index 8358d2b80d4..383e5a1b434 100755 --- a/tests/queries/0_stateless/00534_functions_bad_arguments7.sh +++ b/tests/queries/0_stateless/00534_functions_bad_arguments7.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-tsan, no-debug +# Tags: no-tsan, no-debug, no-fasttest # Tag no-tsan: Too long for TSan # shellcheck disable=SC2016 diff --git a/tests/queries/0_stateless/00719_parallel_ddl_table.sh b/tests/queries/0_stateless/00719_parallel_ddl_table.sh index 2a542ea21f6..fdc994aec33 100755 --- a/tests/queries/0_stateless/00719_parallel_ddl_table.sh +++ b/tests/queries/0_stateless/00719_parallel_ddl_table.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest set -e CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) diff --git a/tests/queries/0_stateless/00746_sql_fuzzy.sh b/tests/queries/0_stateless/00746_sql_fuzzy.sh index b534b1820ba..c0741beea12 100755 --- a/tests/queries/0_stateless/00746_sql_fuzzy.sh +++ b/tests/queries/0_stateless/00746_sql_fuzzy.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01054_cache_dictionary_bunch_update.sh b/tests/queries/0_stateless/01054_cache_dictionary_bunch_update.sh index 04b1f8b65ce..02ea1fa699c 100755 --- a/tests/queries/0_stateless/01054_cache_dictionary_bunch_update.sh +++ b/tests/queries/0_stateless/01054_cache_dictionary_bunch_update.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-parallel +# Tags: no-parallel, no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py b/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py index d38e7738deb..ff15f14cbc3 100755 --- a/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py +++ b/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Tags: no-parallel +# Tags: no-parallel, no-fasttest import os import sys diff --git a/tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.sh b/tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.sh index 2e3b7dd9785..3deb16fa439 100755 --- a/tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.sh +++ b/tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-random-settings, no-parallel +# Tags: no-random-settings, no-parallel, no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01195_formats_diagnostic_info.sh b/tests/queries/0_stateless/01195_formats_diagnostic_info.sh index e75780a4520..b146d65fc58 100755 --- a/tests/queries/0_stateless/01195_formats_diagnostic_info.sh +++ b/tests/queries/0_stateless/01195_formats_diagnostic_info.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest # shellcheck disable=SC2206 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) diff --git a/tests/queries/0_stateless/01246_buffer_flush.sql b/tests/queries/0_stateless/01246_buffer_flush.sql index 47891a7f00e..ac507d94b69 100644 --- a/tests/queries/0_stateless/01246_buffer_flush.sql +++ b/tests/queries/0_stateless/01246_buffer_flush.sql @@ -1,3 +1,5 @@ +-- Tags: no-fasttest + drop table if exists data_01256; drop table if exists buffer_01256; diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index c061eb95a65..abebc35d072 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -50,7 +50,8 @@ CREATE DATABASE [] DATABASE CREATE CREATE TABLE [] TABLE CREATE CREATE VIEW [] VIEW CREATE CREATE DICTIONARY [] DICTIONARY CREATE -CREATE TEMPORARY TABLE [] GLOBAL CREATE +CREATE TEMPORARY TABLE [] GLOBAL CREATE ARBITRARY TEMPORARY TABLE +CREATE ARBITRARY TEMPORARY TABLE [] GLOBAL CREATE CREATE FUNCTION [] GLOBAL CREATE CREATE NAMED COLLECTION [] GLOBAL CREATE CREATE [] \N ALL diff --git a/tests/queries/0_stateless/01293_optimize_final_force.sh b/tests/queries/0_stateless/01293_optimize_final_force.sh index 60d45f87385..994d5952dbc 100755 --- a/tests/queries/0_stateless/01293_optimize_final_force.sh +++ b/tests/queries/0_stateless/01293_optimize_final_force.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01300_client_save_history_when_terminated_long.expect b/tests/queries/0_stateless/01300_client_save_history_when_terminated_long.expect index a593075bb9a..c897d7e9772 100755 --- a/tests/queries/0_stateless/01300_client_save_history_when_terminated_long.expect +++ b/tests/queries/0_stateless/01300_client_save_history_when_terminated_long.expect @@ -28,7 +28,7 @@ exec kill -9 [exp_pid] close # Run client one more time and press "up" to see the last recorded query -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --history_file=$history_file" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --history_file=$history_file" expect ":) " send -- "\[A" expect "for the history" diff --git a/tests/queries/0_stateless/01380_coded_delta_exception_code.sql b/tests/queries/0_stateless/01380_coded_delta_exception_code.sql index 587fac958cd..f4b88a93904 100644 --- a/tests/queries/0_stateless/01380_coded_delta_exception_code.sql +++ b/tests/queries/0_stateless/01380_coded_delta_exception_code.sql @@ -2,5 +2,5 @@ CREATE TABLE delta_codec_synthetic (`id` Decimal(38, 10) CODEC(Delta, ZSTD(22))) CREATE TABLE delta_codec_synthetic (`id` Decimal(38, 10) CODEC(DoubleDelta, ZSTD(22))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 36 } CREATE TABLE delta_codec_synthetic (`id` Decimal(38, 10) CODEC(Gorilla, ZSTD(22))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 36 } -CREATE TABLE delta_codec_synthetic (`id` UInt64 CODEC(DoubleDelta(3), ZSTD(22))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 36 } -CREATE TABLE delta_codec_synthetic (`id` UInt64 CODEC(Gorilla('hello, world'), ZSTD(22))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 36 } +CREATE TABLE delta_codec_synthetic (`id` UInt64 CODEC(DoubleDelta(3), ZSTD(22))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_CODEC_PARAMETER } +CREATE TABLE delta_codec_synthetic (`id` UInt64 CODEC(Gorilla('hello, world'), ZSTD(22))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_CODEC_PARAMETER } diff --git a/tests/queries/0_stateless/01395_limit_more_cases.sh b/tests/queries/0_stateless/01395_limit_more_cases.sh index 32c854e53fb..177147d2142 100755 --- a/tests/queries/0_stateless/01395_limit_more_cases.sh +++ b/tests/queries/0_stateless/01395_limit_more_cases.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01647_clickhouse_local_hung.sh b/tests/queries/0_stateless/01647_clickhouse_local_hung.sh index 04f32055ab6..4789db18b2e 100755 --- a/tests/queries/0_stateless/01647_clickhouse_local_hung.sh +++ b/tests/queries/0_stateless/01647_clickhouse_local_hung.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest set -e diff --git a/tests/queries/0_stateless/02015_async_inserts_4.sh b/tests/queries/0_stateless/02015_async_inserts_4.sh index 65598923b96..28f0e250630 100755 --- a/tests/queries/0_stateless/02015_async_inserts_4.sh +++ b/tests/queries/0_stateless/02015_async_inserts_4.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02015_async_inserts_7.sh b/tests/queries/0_stateless/02015_async_inserts_7.sh index c8cbbc48a29..29f908cdc90 100755 --- a/tests/queries/0_stateless/02015_async_inserts_7.sh +++ b/tests/queries/0_stateless/02015_async_inserts_7.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index f77076bcd5c..c13de3faec3 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -289,7 +289,7 @@ CREATE TABLE system.grants ( `user_name` Nullable(String), `role_name` Nullable(String), - `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'SHOW NAMED COLLECTIONS SECRETS' = 92, 'ACCESS MANAGEMENT' = 93, 'SYSTEM SHUTDOWN' = 94, 'SYSTEM DROP DNS CACHE' = 95, 'SYSTEM DROP MARK CACHE' = 96, 'SYSTEM DROP UNCOMPRESSED CACHE' = 97, 'SYSTEM DROP MMAP CACHE' = 98, 'SYSTEM DROP QUERY CACHE' = 99, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 100, 'SYSTEM DROP FILESYSTEM CACHE' = 101, 'SYSTEM DROP SCHEMA CACHE' = 102, 'SYSTEM DROP S3 CLIENT CACHE' = 103, 'SYSTEM DROP CACHE' = 104, 'SYSTEM RELOAD CONFIG' = 105, 'SYSTEM RELOAD USERS' = 106, 'SYSTEM RELOAD SYMBOLS' = 107, 'SYSTEM RELOAD DICTIONARY' = 108, 'SYSTEM RELOAD MODEL' = 109, 'SYSTEM RELOAD FUNCTION' = 110, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 111, 'SYSTEM RELOAD' = 112, 'SYSTEM RESTART DISK' = 113, 'SYSTEM MERGES' = 114, 'SYSTEM TTL MERGES' = 115, 'SYSTEM FETCHES' = 116, 'SYSTEM MOVES' = 117, 'SYSTEM DISTRIBUTED SENDS' = 118, 'SYSTEM REPLICATED SENDS' = 119, 'SYSTEM SENDS' = 120, 'SYSTEM REPLICATION QUEUES' = 121, 'SYSTEM DROP REPLICA' = 122, 'SYSTEM SYNC REPLICA' = 123, 'SYSTEM RESTART REPLICA' = 124, 'SYSTEM RESTORE REPLICA' = 125, 'SYSTEM WAIT LOADING PARTS' = 126, 'SYSTEM SYNC DATABASE REPLICA' = 127, 'SYSTEM SYNC TRANSACTION LOG' = 128, 'SYSTEM SYNC FILE CACHE' = 129, 'SYSTEM FLUSH DISTRIBUTED' = 130, 'SYSTEM FLUSH LOGS' = 131, 'SYSTEM FLUSH' = 132, 'SYSTEM THREAD FUZZER' = 133, 'SYSTEM UNFREEZE' = 134, 'SYSTEM' = 135, 'dictGet' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158), + `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'TRUNCATE' = 64, 'OPTIMIZE' = 65, 'BACKUP' = 66, 'KILL QUERY' = 67, 'KILL TRANSACTION' = 68, 'MOVE PARTITION BETWEEN SHARDS' = 69, 'CREATE USER' = 70, 'ALTER USER' = 71, 'DROP USER' = 72, 'CREATE ROLE' = 73, 'ALTER ROLE' = 74, 'DROP ROLE' = 75, 'ROLE ADMIN' = 76, 'CREATE ROW POLICY' = 77, 'ALTER ROW POLICY' = 78, 'DROP ROW POLICY' = 79, 'CREATE QUOTA' = 80, 'ALTER QUOTA' = 81, 'DROP QUOTA' = 82, 'CREATE SETTINGS PROFILE' = 83, 'ALTER SETTINGS PROFILE' = 84, 'DROP SETTINGS PROFILE' = 85, 'SHOW USERS' = 86, 'SHOW ROLES' = 87, 'SHOW ROW POLICIES' = 88, 'SHOW QUOTAS' = 89, 'SHOW SETTINGS PROFILES' = 90, 'SHOW ACCESS' = 91, 'SHOW NAMED COLLECTIONS' = 92, 'SHOW NAMED COLLECTIONS SECRETS' = 93, 'ACCESS MANAGEMENT' = 94, 'SYSTEM SHUTDOWN' = 95, 'SYSTEM DROP DNS CACHE' = 96, 'SYSTEM DROP MARK CACHE' = 97, 'SYSTEM DROP UNCOMPRESSED CACHE' = 98, 'SYSTEM DROP MMAP CACHE' = 99, 'SYSTEM DROP QUERY CACHE' = 100, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 101, 'SYSTEM DROP FILESYSTEM CACHE' = 102, 'SYSTEM DROP SCHEMA CACHE' = 103, 'SYSTEM DROP S3 CLIENT CACHE' = 104, 'SYSTEM DROP CACHE' = 105, 'SYSTEM RELOAD CONFIG' = 106, 'SYSTEM RELOAD USERS' = 107, 'SYSTEM RELOAD SYMBOLS' = 108, 'SYSTEM RELOAD DICTIONARY' = 109, 'SYSTEM RELOAD MODEL' = 110, 'SYSTEM RELOAD FUNCTION' = 111, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 112, 'SYSTEM RELOAD' = 113, 'SYSTEM RESTART DISK' = 114, 'SYSTEM MERGES' = 115, 'SYSTEM TTL MERGES' = 116, 'SYSTEM FETCHES' = 117, 'SYSTEM MOVES' = 118, 'SYSTEM DISTRIBUTED SENDS' = 119, 'SYSTEM REPLICATED SENDS' = 120, 'SYSTEM SENDS' = 121, 'SYSTEM REPLICATION QUEUES' = 122, 'SYSTEM DROP REPLICA' = 123, 'SYSTEM SYNC REPLICA' = 124, 'SYSTEM RESTART REPLICA' = 125, 'SYSTEM RESTORE REPLICA' = 126, 'SYSTEM WAIT LOADING PARTS' = 127, 'SYSTEM SYNC DATABASE REPLICA' = 128, 'SYSTEM SYNC TRANSACTION LOG' = 129, 'SYSTEM SYNC FILE CACHE' = 130, 'SYSTEM FLUSH DISTRIBUTED' = 131, 'SYSTEM FLUSH LOGS' = 132, 'SYSTEM FLUSH' = 133, 'SYSTEM THREAD FUZZER' = 134, 'SYSTEM UNFREEZE' = 135, 'SYSTEM' = 136, 'dictGet' = 137, 'addressToLine' = 138, 'addressToLineWithInlines' = 139, 'addressToSymbol' = 140, 'demangle' = 141, 'INTROSPECTION' = 142, 'FILE' = 143, 'URL' = 144, 'REMOTE' = 145, 'MONGO' = 146, 'MEILISEARCH' = 147, 'MYSQL' = 148, 'POSTGRES' = 149, 'SQLITE' = 150, 'ODBC' = 151, 'JDBC' = 152, 'HDFS' = 153, 'S3' = 154, 'HIVE' = 155, 'SOURCES' = 156, 'CLUSTER' = 157, 'ALL' = 158, 'NONE' = 159), `database` Nullable(String), `table` Nullable(String), `column` Nullable(String), @@ -570,10 +570,10 @@ ENGINE = SystemPartsColumns COMMENT 'SYSTEM TABLE is built on the fly.' CREATE TABLE system.privileges ( - `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'SHOW NAMED COLLECTIONS SECRETS' = 92, 'ACCESS MANAGEMENT' = 93, 'SYSTEM SHUTDOWN' = 94, 'SYSTEM DROP DNS CACHE' = 95, 'SYSTEM DROP MARK CACHE' = 96, 'SYSTEM DROP UNCOMPRESSED CACHE' = 97, 'SYSTEM DROP MMAP CACHE' = 98, 'SYSTEM DROP QUERY CACHE' = 99, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 100, 'SYSTEM DROP FILESYSTEM CACHE' = 101, 'SYSTEM DROP SCHEMA CACHE' = 102, 'SYSTEM DROP S3 CLIENT CACHE' = 103, 'SYSTEM DROP CACHE' = 104, 'SYSTEM RELOAD CONFIG' = 105, 'SYSTEM RELOAD USERS' = 106, 'SYSTEM RELOAD SYMBOLS' = 107, 'SYSTEM RELOAD DICTIONARY' = 108, 'SYSTEM RELOAD MODEL' = 109, 'SYSTEM RELOAD FUNCTION' = 110, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 111, 'SYSTEM RELOAD' = 112, 'SYSTEM RESTART DISK' = 113, 'SYSTEM MERGES' = 114, 'SYSTEM TTL MERGES' = 115, 'SYSTEM FETCHES' = 116, 'SYSTEM MOVES' = 117, 'SYSTEM DISTRIBUTED SENDS' = 118, 'SYSTEM REPLICATED SENDS' = 119, 'SYSTEM SENDS' = 120, 'SYSTEM REPLICATION QUEUES' = 121, 'SYSTEM DROP REPLICA' = 122, 'SYSTEM SYNC REPLICA' = 123, 'SYSTEM RESTART REPLICA' = 124, 'SYSTEM RESTORE REPLICA' = 125, 'SYSTEM WAIT LOADING PARTS' = 126, 'SYSTEM SYNC DATABASE REPLICA' = 127, 'SYSTEM SYNC TRANSACTION LOG' = 128, 'SYSTEM SYNC FILE CACHE' = 129, 'SYSTEM FLUSH DISTRIBUTED' = 130, 'SYSTEM FLUSH LOGS' = 131, 'SYSTEM FLUSH' = 132, 'SYSTEM THREAD FUZZER' = 133, 'SYSTEM UNFREEZE' = 134, 'SYSTEM' = 135, 'dictGet' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158), + `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'TRUNCATE' = 64, 'OPTIMIZE' = 65, 'BACKUP' = 66, 'KILL QUERY' = 67, 'KILL TRANSACTION' = 68, 'MOVE PARTITION BETWEEN SHARDS' = 69, 'CREATE USER' = 70, 'ALTER USER' = 71, 'DROP USER' = 72, 'CREATE ROLE' = 73, 'ALTER ROLE' = 74, 'DROP ROLE' = 75, 'ROLE ADMIN' = 76, 'CREATE ROW POLICY' = 77, 'ALTER ROW POLICY' = 78, 'DROP ROW POLICY' = 79, 'CREATE QUOTA' = 80, 'ALTER QUOTA' = 81, 'DROP QUOTA' = 82, 'CREATE SETTINGS PROFILE' = 83, 'ALTER SETTINGS PROFILE' = 84, 'DROP SETTINGS PROFILE' = 85, 'SHOW USERS' = 86, 'SHOW ROLES' = 87, 'SHOW ROW POLICIES' = 88, 'SHOW QUOTAS' = 89, 'SHOW SETTINGS PROFILES' = 90, 'SHOW ACCESS' = 91, 'SHOW NAMED COLLECTIONS' = 92, 'SHOW NAMED COLLECTIONS SECRETS' = 93, 'ACCESS MANAGEMENT' = 94, 'SYSTEM SHUTDOWN' = 95, 'SYSTEM DROP DNS CACHE' = 96, 'SYSTEM DROP MARK CACHE' = 97, 'SYSTEM DROP UNCOMPRESSED CACHE' = 98, 'SYSTEM DROP MMAP CACHE' = 99, 'SYSTEM DROP QUERY CACHE' = 100, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 101, 'SYSTEM DROP FILESYSTEM CACHE' = 102, 'SYSTEM DROP SCHEMA CACHE' = 103, 'SYSTEM DROP S3 CLIENT CACHE' = 104, 'SYSTEM DROP CACHE' = 105, 'SYSTEM RELOAD CONFIG' = 106, 'SYSTEM RELOAD USERS' = 107, 'SYSTEM RELOAD SYMBOLS' = 108, 'SYSTEM RELOAD DICTIONARY' = 109, 'SYSTEM RELOAD MODEL' = 110, 'SYSTEM RELOAD FUNCTION' = 111, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 112, 'SYSTEM RELOAD' = 113, 'SYSTEM RESTART DISK' = 114, 'SYSTEM MERGES' = 115, 'SYSTEM TTL MERGES' = 116, 'SYSTEM FETCHES' = 117, 'SYSTEM MOVES' = 118, 'SYSTEM DISTRIBUTED SENDS' = 119, 'SYSTEM REPLICATED SENDS' = 120, 'SYSTEM SENDS' = 121, 'SYSTEM REPLICATION QUEUES' = 122, 'SYSTEM DROP REPLICA' = 123, 'SYSTEM SYNC REPLICA' = 124, 'SYSTEM RESTART REPLICA' = 125, 'SYSTEM RESTORE REPLICA' = 126, 'SYSTEM WAIT LOADING PARTS' = 127, 'SYSTEM SYNC DATABASE REPLICA' = 128, 'SYSTEM SYNC TRANSACTION LOG' = 129, 'SYSTEM SYNC FILE CACHE' = 130, 'SYSTEM FLUSH DISTRIBUTED' = 131, 'SYSTEM FLUSH LOGS' = 132, 'SYSTEM FLUSH' = 133, 'SYSTEM THREAD FUZZER' = 134, 'SYSTEM UNFREEZE' = 135, 'SYSTEM' = 136, 'dictGet' = 137, 'addressToLine' = 138, 'addressToLineWithInlines' = 139, 'addressToSymbol' = 140, 'demangle' = 141, 'INTROSPECTION' = 142, 'FILE' = 143, 'URL' = 144, 'REMOTE' = 145, 'MONGO' = 146, 'MEILISEARCH' = 147, 'MYSQL' = 148, 'POSTGRES' = 149, 'SQLITE' = 150, 'ODBC' = 151, 'JDBC' = 152, 'HDFS' = 153, 'S3' = 154, 'HIVE' = 155, 'SOURCES' = 156, 'CLUSTER' = 157, 'ALL' = 158, 'NONE' = 159), `aliases` Array(String), `level` Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2, 'DICTIONARY' = 3, 'VIEW' = 4, 'COLUMN' = 5)), - `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE FUNCTION' = 53, 'CREATE NAMED COLLECTION' = 54, 'CREATE' = 55, 'DROP DATABASE' = 56, 'DROP TABLE' = 57, 'DROP VIEW' = 58, 'DROP DICTIONARY' = 59, 'DROP FUNCTION' = 60, 'DROP NAMED COLLECTION' = 61, 'DROP' = 62, 'TRUNCATE' = 63, 'OPTIMIZE' = 64, 'BACKUP' = 65, 'KILL QUERY' = 66, 'KILL TRANSACTION' = 67, 'MOVE PARTITION BETWEEN SHARDS' = 68, 'CREATE USER' = 69, 'ALTER USER' = 70, 'DROP USER' = 71, 'CREATE ROLE' = 72, 'ALTER ROLE' = 73, 'DROP ROLE' = 74, 'ROLE ADMIN' = 75, 'CREATE ROW POLICY' = 76, 'ALTER ROW POLICY' = 77, 'DROP ROW POLICY' = 78, 'CREATE QUOTA' = 79, 'ALTER QUOTA' = 80, 'DROP QUOTA' = 81, 'CREATE SETTINGS PROFILE' = 82, 'ALTER SETTINGS PROFILE' = 83, 'DROP SETTINGS PROFILE' = 84, 'SHOW USERS' = 85, 'SHOW ROLES' = 86, 'SHOW ROW POLICIES' = 87, 'SHOW QUOTAS' = 88, 'SHOW SETTINGS PROFILES' = 89, 'SHOW ACCESS' = 90, 'SHOW NAMED COLLECTIONS' = 91, 'SHOW NAMED COLLECTIONS SECRETS' = 92, 'ACCESS MANAGEMENT' = 93, 'SYSTEM SHUTDOWN' = 94, 'SYSTEM DROP DNS CACHE' = 95, 'SYSTEM DROP MARK CACHE' = 96, 'SYSTEM DROP UNCOMPRESSED CACHE' = 97, 'SYSTEM DROP MMAP CACHE' = 98, 'SYSTEM DROP QUERY CACHE' = 99, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 100, 'SYSTEM DROP FILESYSTEM CACHE' = 101, 'SYSTEM DROP SCHEMA CACHE' = 102, 'SYSTEM DROP S3 CLIENT CACHE' = 103, 'SYSTEM DROP CACHE' = 104, 'SYSTEM RELOAD CONFIG' = 105, 'SYSTEM RELOAD USERS' = 106, 'SYSTEM RELOAD SYMBOLS' = 107, 'SYSTEM RELOAD DICTIONARY' = 108, 'SYSTEM RELOAD MODEL' = 109, 'SYSTEM RELOAD FUNCTION' = 110, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 111, 'SYSTEM RELOAD' = 112, 'SYSTEM RESTART DISK' = 113, 'SYSTEM MERGES' = 114, 'SYSTEM TTL MERGES' = 115, 'SYSTEM FETCHES' = 116, 'SYSTEM MOVES' = 117, 'SYSTEM DISTRIBUTED SENDS' = 118, 'SYSTEM REPLICATED SENDS' = 119, 'SYSTEM SENDS' = 120, 'SYSTEM REPLICATION QUEUES' = 121, 'SYSTEM DROP REPLICA' = 122, 'SYSTEM SYNC REPLICA' = 123, 'SYSTEM RESTART REPLICA' = 124, 'SYSTEM RESTORE REPLICA' = 125, 'SYSTEM WAIT LOADING PARTS' = 126, 'SYSTEM SYNC DATABASE REPLICA' = 127, 'SYSTEM SYNC TRANSACTION LOG' = 128, 'SYSTEM SYNC FILE CACHE' = 129, 'SYSTEM FLUSH DISTRIBUTED' = 130, 'SYSTEM FLUSH LOGS' = 131, 'SYSTEM FLUSH' = 132, 'SYSTEM THREAD FUZZER' = 133, 'SYSTEM UNFREEZE' = 134, 'SYSTEM' = 135, 'dictGet' = 136, 'addressToLine' = 137, 'addressToLineWithInlines' = 138, 'addressToSymbol' = 139, 'demangle' = 140, 'INTROSPECTION' = 141, 'FILE' = 142, 'URL' = 143, 'REMOTE' = 144, 'MONGO' = 145, 'MEILISEARCH' = 146, 'MYSQL' = 147, 'POSTGRES' = 148, 'SQLITE' = 149, 'ODBC' = 150, 'JDBC' = 151, 'HDFS' = 152, 'S3' = 153, 'HIVE' = 154, 'SOURCES' = 155, 'CLUSTER' = 156, 'ALL' = 157, 'NONE' = 158)) + `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'TRUNCATE' = 64, 'OPTIMIZE' = 65, 'BACKUP' = 66, 'KILL QUERY' = 67, 'KILL TRANSACTION' = 68, 'MOVE PARTITION BETWEEN SHARDS' = 69, 'CREATE USER' = 70, 'ALTER USER' = 71, 'DROP USER' = 72, 'CREATE ROLE' = 73, 'ALTER ROLE' = 74, 'DROP ROLE' = 75, 'ROLE ADMIN' = 76, 'CREATE ROW POLICY' = 77, 'ALTER ROW POLICY' = 78, 'DROP ROW POLICY' = 79, 'CREATE QUOTA' = 80, 'ALTER QUOTA' = 81, 'DROP QUOTA' = 82, 'CREATE SETTINGS PROFILE' = 83, 'ALTER SETTINGS PROFILE' = 84, 'DROP SETTINGS PROFILE' = 85, 'SHOW USERS' = 86, 'SHOW ROLES' = 87, 'SHOW ROW POLICIES' = 88, 'SHOW QUOTAS' = 89, 'SHOW SETTINGS PROFILES' = 90, 'SHOW ACCESS' = 91, 'SHOW NAMED COLLECTIONS' = 92, 'SHOW NAMED COLLECTIONS SECRETS' = 93, 'ACCESS MANAGEMENT' = 94, 'SYSTEM SHUTDOWN' = 95, 'SYSTEM DROP DNS CACHE' = 96, 'SYSTEM DROP MARK CACHE' = 97, 'SYSTEM DROP UNCOMPRESSED CACHE' = 98, 'SYSTEM DROP MMAP CACHE' = 99, 'SYSTEM DROP QUERY CACHE' = 100, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 101, 'SYSTEM DROP FILESYSTEM CACHE' = 102, 'SYSTEM DROP SCHEMA CACHE' = 103, 'SYSTEM DROP S3 CLIENT CACHE' = 104, 'SYSTEM DROP CACHE' = 105, 'SYSTEM RELOAD CONFIG' = 106, 'SYSTEM RELOAD USERS' = 107, 'SYSTEM RELOAD SYMBOLS' = 108, 'SYSTEM RELOAD DICTIONARY' = 109, 'SYSTEM RELOAD MODEL' = 110, 'SYSTEM RELOAD FUNCTION' = 111, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 112, 'SYSTEM RELOAD' = 113, 'SYSTEM RESTART DISK' = 114, 'SYSTEM MERGES' = 115, 'SYSTEM TTL MERGES' = 116, 'SYSTEM FETCHES' = 117, 'SYSTEM MOVES' = 118, 'SYSTEM DISTRIBUTED SENDS' = 119, 'SYSTEM REPLICATED SENDS' = 120, 'SYSTEM SENDS' = 121, 'SYSTEM REPLICATION QUEUES' = 122, 'SYSTEM DROP REPLICA' = 123, 'SYSTEM SYNC REPLICA' = 124, 'SYSTEM RESTART REPLICA' = 125, 'SYSTEM RESTORE REPLICA' = 126, 'SYSTEM WAIT LOADING PARTS' = 127, 'SYSTEM SYNC DATABASE REPLICA' = 128, 'SYSTEM SYNC TRANSACTION LOG' = 129, 'SYSTEM SYNC FILE CACHE' = 130, 'SYSTEM FLUSH DISTRIBUTED' = 131, 'SYSTEM FLUSH LOGS' = 132, 'SYSTEM FLUSH' = 133, 'SYSTEM THREAD FUZZER' = 134, 'SYSTEM UNFREEZE' = 135, 'SYSTEM' = 136, 'dictGet' = 137, 'addressToLine' = 138, 'addressToLineWithInlines' = 139, 'addressToSymbol' = 140, 'demangle' = 141, 'INTROSPECTION' = 142, 'FILE' = 143, 'URL' = 144, 'REMOTE' = 145, 'MONGO' = 146, 'MEILISEARCH' = 147, 'MYSQL' = 148, 'POSTGRES' = 149, 'SQLITE' = 150, 'ODBC' = 151, 'JDBC' = 152, 'HDFS' = 153, 'S3' = 154, 'HIVE' = 155, 'SOURCES' = 156, 'CLUSTER' = 157, 'ALL' = 158, 'NONE' = 159)) ) ENGINE = SystemPrivileges COMMENT 'SYSTEM TABLE is built on the fly.' diff --git a/tests/queries/0_stateless/02184_default_table_engine.sql b/tests/queries/0_stateless/02184_default_table_engine.sql index 4b5ad6c008c..109875d53a5 100644 --- a/tests/queries/0_stateless/02184_default_table_engine.sql +++ b/tests/queries/0_stateless/02184_default_table_engine.sql @@ -82,7 +82,7 @@ SET default_table_engine = 'Log'; CREATE TEMPORARY TABLE tmp (n int); SHOW CREATE TEMPORARY TABLE tmp; CREATE TEMPORARY TABLE tmp1 (n int) ENGINE=Memory; -CREATE TEMPORARY TABLE tmp2 (n int) ENGINE=Log; -- {serverError 80} +CREATE TEMPORARY TABLE tmp2 (n int) ENGINE=Log; CREATE TEMPORARY TABLE tmp2 (n int) ORDER BY n; -- {serverError 80} CREATE TEMPORARY TABLE tmp2 (n int, PRIMARY KEY (n)); -- {serverError 80} diff --git a/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh b/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh index 171dcc52c9c..e5d00bc1a1c 100755 --- a/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh +++ b/tests/queries/0_stateless/02229_client_stop_multiquery_in_SIGINT.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02293_grouping_function.sql b/tests/queries/0_stateless/02293_grouping_function.sql index cf076c8e51c..c858eae269d 100644 --- a/tests/queries/0_stateless/02293_grouping_function.sql +++ b/tests/queries/0_stateless/02293_grouping_function.sql @@ -1,3 +1,5 @@ +set optimize_group_by_function_keys=0; + SELECT number, grouping(number, number % 2, number % 3) AS gr diff --git a/tests/queries/0_stateless/02293_grouping_function_group_by.sql b/tests/queries/0_stateless/02293_grouping_function_group_by.sql index d438a8a5277..da6477a1822 100644 --- a/tests/queries/0_stateless/02293_grouping_function_group_by.sql +++ b/tests/queries/0_stateless/02293_grouping_function_group_by.sql @@ -1,3 +1,5 @@ +set optimize_group_by_function_keys=0; + SELECT number, grouping(number, number % 2, number % 3) = 6 diff --git a/tests/queries/0_stateless/02315_grouping_constant_folding.reference b/tests/queries/0_stateless/02315_grouping_constant_folding.reference index 6e591de2661..31816318a42 100644 --- a/tests/queries/0_stateless/02315_grouping_constant_folding.reference +++ b/tests/queries/0_stateless/02315_grouping_constant_folding.reference @@ -27,3 +27,17 @@ SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY ROLLUP(a, 5 0 0 2 5 1 0 2 10 0 0 0 +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING SETS ((a, b), (a, a), ()) ORDER BY (amount, a, b) SETTINGS force_grouping_standard_compatibility=0, allow_experimental_analyzer=1; +1 0 0 3 +1 0 2 3 +1 0 4 3 +1 0 6 3 +1 0 8 3 +1 1 1 3 +1 1 3 3 +1 1 5 3 +1 1 7 3 +1 1 9 3 +5 0 0 2 +5 1 0 2 +10 0 0 0 diff --git a/tests/queries/0_stateless/02315_grouping_constant_folding.sql b/tests/queries/0_stateless/02315_grouping_constant_folding.sql index ff259b7be79..f992aa0da32 100644 --- a/tests/queries/0_stateless/02315_grouping_constant_folding.sql +++ b/tests/queries/0_stateless/02315_grouping_constant_folding.sql @@ -9,5 +9,7 @@ SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY ROLLUP(a, b) ORDER BY (amount, a, b) SETTINGS force_grouping_standard_compatibility=0; +SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING SETS ((a, b), (a, a), ()) ORDER BY (amount, a, b) SETTINGS force_grouping_standard_compatibility=0, allow_experimental_analyzer=1; + -- { echoOff } DROP TABLE test02315; diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference index 32ddab4886c..2e049dbc936 100644 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.reference +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.reference @@ -464,3 +464,16 @@ Expression ((Projection + (Before ORDER BY + (Projection + Before ORDER BY)))) 1 0 +-- DISTINCT COUNT() with GROUP BY => do _not_ remove DISTINCT +-- query +select distinct count() from numbers(10) group by number +-- explain +Expression (Projection) + Distinct + Distinct (Preliminary DISTINCT) + Expression (Before ORDER BY) + Aggregating + Expression (Before GROUP BY) + ReadFromStorage (SystemNumbers) +-- execute +1 diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.sh b/tests/queries/0_stateless/02500_remove_redundant_distinct.sh index 879cc776fe1..d550b057853 100755 --- a/tests/queries/0_stateless/02500_remove_redundant_distinct.sh +++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.sh @@ -256,3 +256,7 @@ FROM GROUP BY a WITH TOTALS )" run_query "$query" + +echo "-- DISTINCT COUNT() with GROUP BY => do _not_ remove DISTINCT" +query="select distinct count() from numbers(10) group by number" +run_query "$query" diff --git a/tests/queries/0_stateless/02525_different_engines_in_temporary_tables.reference b/tests/queries/0_stateless/02525_different_engines_in_temporary_tables.reference new file mode 100644 index 00000000000..3d1916b29f6 --- /dev/null +++ b/tests/queries/0_stateless/02525_different_engines_in_temporary_tables.reference @@ -0,0 +1,14 @@ +1 a +2 b +3 c +0 +0 +1 a +2 b +3 c +1 a +2 b +3 c +1 a +2 b +3 c diff --git a/tests/queries/0_stateless/02525_different_engines_in_temporary_tables.sql b/tests/queries/0_stateless/02525_different_engines_in_temporary_tables.sql new file mode 100644 index 00000000000..7ebc05dfece --- /dev/null +++ b/tests/queries/0_stateless/02525_different_engines_in_temporary_tables.sql @@ -0,0 +1,66 @@ +DROP TEMPORARY TABLE IF EXISTS table_merge_tree_02525; +CREATE TEMPORARY TABLE table_merge_tree_02525 +( + id UInt64, + info String +) +ENGINE = MergeTree +ORDER BY id +PRIMARY KEY id; +INSERT INTO table_merge_tree_02525 VALUES (1, 'a'), (2, 'b'), (3, 'c'); +SELECT * FROM table_merge_tree_02525; +-- Check that temporary table with MergeTree is not sent to remote servers +-- The query with remote() should not fail +SELECT dummy FROM remote('127.0.0.{1,2}', system, one); +DROP TEMPORARY TABLE table_merge_tree_02525; + +DROP TEMPORARY TABLE IF EXISTS table_log_02525; +CREATE TEMPORARY TABLE table_log_02525 +( + id UInt64, + info String +) +ENGINE = Log; +INSERT INTO table_log_02525 VALUES (1, 'a'), (2, 'b'), (3, 'c'); +SELECT * FROM table_log_02525; +DROP TEMPORARY TABLE table_log_02525; + +DROP TEMPORARY TABLE IF EXISTS table_stripe_log_02525; +CREATE TEMPORARY TABLE table_stripe_log_02525 +( + id UInt64, + info String +) +ENGINE = StripeLog; +INSERT INTO table_stripe_log_02525 VALUES (1, 'a'), (2, 'b'), (3, 'c'); +SELECT * FROM table_stripe_log_02525; +DROP TEMPORARY TABLE table_stripe_log_02525; + +DROP TEMPORARY TABLE IF EXISTS table_tiny_log_02525; +CREATE TEMPORARY TABLE table_tiny_log_02525 +( + id UInt64, + info String +) +ENGINE = TinyLog; +INSERT INTO table_tiny_log_02525 VALUES (1, 'a'), (2, 'b'), (3, 'c'); +SELECT * FROM table_tiny_log_02525; +DROP TEMPORARY TABLE table_tiny_log_02525; + +DROP TEMPORARY TABLE IF EXISTS table_replicated_merge_tree_02525; +CREATE TEMPORARY TABLE table_replicated_merge_tree_02525 +( + id UInt64, + info String +) +ENGINE ReplicatedMergeTree('/clickhouse/tables/{database}/test_02525/table_replicated_merge_tree_02525', 'r1') +ORDER BY id +PRIMARY KEY id; -- { serverError INCORRECT_QUERY } + +DROP TEMPORARY TABLE IF EXISTS table_keeper_map_02525; +CREATE TEMPORARY TABLE table_keeper_map_02525 +( + key String, + value UInt32 +) Engine=KeeperMap('/' || currentDatabase() || '/test02525') +PRIMARY KEY(key); -- { serverError INCORRECT_QUERY } diff --git a/tests/queries/0_stateless/02534_keyed_siphash.reference b/tests/queries/0_stateless/02534_keyed_siphash.reference index 52e92f37720..3606b9a41db 100644 --- a/tests/queries/0_stateless/02534_keyed_siphash.reference +++ b/tests/queries/0_stateless/02534_keyed_siphash.reference @@ -191,6 +191,6 @@ E51B38608EF25F57 1 1 E28DBDE7FE22E41C -1CE422FEE7BD8DE20000000000000000 +1 E28DBDE7FE22E41C -1CE422FEE7BD8DE20000000000000000 +1 diff --git a/tests/queries/0_stateless/02534_keyed_siphash.sql b/tests/queries/0_stateless/02534_keyed_siphash.sql index 3c41efd7d58..9c914f586f0 100644 --- a/tests/queries/0_stateless/02534_keyed_siphash.sql +++ b/tests/queries/0_stateless/02534_keyed_siphash.sql @@ -269,6 +269,6 @@ select sipHash64Keyed(toUInt64(0), '1'); -- { serverError 48 } select sipHash128Keyed(toUInt64(0), '1'); -- { serverError 48 } select hex(sipHash64()); -select hex(sipHash128()); +SELECT hex(sipHash128()) = hex(reverse(unhex('1CE422FEE7BD8DE20000000000000000'))) or hex(sipHash128()) = '1CE422FEE7BD8DE20000000000000000'; select hex(sipHash64Keyed()); -select hex(sipHash128Keyed()); +SELECT hex(sipHash128Keyed()) = hex(reverse(unhex('1CE422FEE7BD8DE20000000000000000'))) or hex(sipHash128Keyed()) = '1CE422FEE7BD8DE20000000000000000'; diff --git a/tests/queries/0_stateless/02552_siphash128_reference.reference b/tests/queries/0_stateless/02552_siphash128_reference.reference index a831c691ce7..452e9910660 100644 --- a/tests/queries/0_stateless/02552_siphash128_reference.reference +++ b/tests/queries/0_stateless/02552_siphash128_reference.reference @@ -126,5 +126,5 @@ E3040C00EB28F15366CA73CBD872E740 1 1 1 -1CE422FEE7BD8DE20000000000000000 -1CE422FEE7BD8DE20000000000000000 +1 +1 diff --git a/tests/queries/0_stateless/02552_siphash128_reference.sql b/tests/queries/0_stateless/02552_siphash128_reference.sql index 323561654b9..c238e51b690 100644 --- a/tests/queries/0_stateless/02552_siphash128_reference.sql +++ b/tests/queries/0_stateless/02552_siphash128_reference.sql @@ -203,5 +203,5 @@ select sipHash128ReferenceKeyed((toUInt64(0),toUInt64(0)),char(0, 1, 2, 3, 4, 5, select sipHash128ReferenceKeyed((0, 0), '1'); -- { serverError 48 } select sipHash128ReferenceKeyed(toUInt64(0), '1'); -- { serverError 48 } -select hex(sipHash128Reference()); -select hex(sipHash128ReferenceKeyed()); +SELECT hex(sipHash128Reference()) = hex(reverse(unhex('1CE422FEE7BD8DE20000000000000000'))) or hex(sipHash128()) = '1CE422FEE7BD8DE20000000000000000'; +SELECT hex(sipHash128ReferenceKeyed()) = hex(reverse(unhex('1CE422FEE7BD8DE20000000000000000'))) or hex(sipHash128Keyed()) = '1CE422FEE7BD8DE20000000000000000'; diff --git a/tests/queries/0_stateless/02561_temporary_table_grants.reference b/tests/queries/0_stateless/02561_temporary_table_grants.reference new file mode 100644 index 00000000000..b462a5a7baa --- /dev/null +++ b/tests/queries/0_stateless/02561_temporary_table_grants.reference @@ -0,0 +1,4 @@ +OK +OK +OK +OK diff --git a/tests/queries/0_stateless/02561_temporary_table_grants.sh b/tests/queries/0_stateless/02561_temporary_table_grants.sh new file mode 100755 index 00000000000..6e0c96786e8 --- /dev/null +++ b/tests/queries/0_stateless/02561_temporary_table_grants.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +set -e + +user=user_$CLICKHOUSE_TEST_UNIQUE_NAME +$CLICKHOUSE_CLIENT --query "DROP USER IF EXISTS $user" +$CLICKHOUSE_CLIENT --query "CREATE USER $user IDENTIFIED WITH PLAINTEXT_PASSWORD BY 'hello'" + +$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_memory_02561(name String)" 2>&1 | grep -F "Not enough privileges. To execute this query it's necessary to have grant CREATE TEMPORARY TABLE" > /dev/null && echo "OK" + +$CLICKHOUSE_CLIENT --query "GRANT CREATE TEMPORARY TABLE ON *.* TO $user" +$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_memory_02561(name String)" + +$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_merge_tree_02561(name String) ENGINE = MergeTree() ORDER BY name" 2>&1 | grep -F "Not enough privileges. To execute this query it's necessary to have grant CREATE ARBITRARY TEMPORARY TABLE" > /dev/null && echo "OK" + +$CLICKHOUSE_CLIENT --query "GRANT CREATE ARBITRARY TEMPORARY TABLE ON *.* TO $user" + +$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_merge_tree_02561(name String) ENGINE = MergeTree() ORDER BY name" + +$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_file_02561(name String) ENGINE = File(TabSeparated)" 2>&1 | grep -F "Not enough privileges. To execute this query it's necessary to have grant FILE" > /dev/null && echo "OK" + +$CLICKHOUSE_CLIENT --query "GRANT FILE ON *.* TO $user" + +$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_file_02561(name String) ENGINE = File(TabSeparated)" + +$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_url_02561(name String) ENGINE = URL('http://127.0.0.1:8123?query=select+12', 'RawBLOB')" 2>&1 | grep -F "Not enough privileges. To execute this query it's necessary to have grant URL" > /dev/null && echo "OK" + +$CLICKHOUSE_CLIENT --query "GRANT URL ON *.* TO $user" + +$CLICKHOUSE_CLIENT --user $user --password hello --query "CREATE TEMPORARY TABLE table_url_02561(name String) ENGINE = URL('http://127.0.0.1:8123?query=select+12', 'RawBLOB')" + +$CLICKHOUSE_CLIENT --query "DROP USER $user" diff --git a/tests/queries/0_stateless/02561_temporary_table_sessions.reference b/tests/queries/0_stateless/02561_temporary_table_sessions.reference new file mode 100644 index 00000000000..b3890873523 --- /dev/null +++ b/tests/queries/0_stateless/02561_temporary_table_sessions.reference @@ -0,0 +1,7 @@ +OK +1 d +2 e +3 f +1 a +2 b +3 c diff --git a/tests/queries/0_stateless/02561_temporary_table_sessions.sh b/tests/queries/0_stateless/02561_temporary_table_sessions.sh new file mode 100755 index 00000000000..a810a48cdf3 --- /dev/null +++ b/tests/queries/0_stateless/02561_temporary_table_sessions.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Tags: no-parallel + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +SESSION_ID_A="$RANDOM$RANDOM$RANDOM" +SESSION_ID_B="$RANDOM$RANDOM$RANDOM" + +# Create temporary table and insert in SESSION_ID_A +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${SESSION_ID_A}" -d 'CREATE TEMPORARY TABLE table_merge_tree_02561 (id UInt64, info String) ENGINE = MergeTree ORDER BY id' +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${SESSION_ID_A}" -d "INSERT INTO table_merge_tree_02561 VALUES (1, 'a'), (2, 'b'), (3, 'c')" + +# Select from SESSION_ID_B +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${SESSION_ID_B}" -d "SELECT * FROM table_merge_tree_02561" | tr -d '\n' | grep -F 'UNKNOWN_TABLE' > /dev/null && echo "OK" + +# Create temporary table, insert and select in SESSION_ID_B +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${SESSION_ID_B}" -d 'CREATE TEMPORARY TABLE table_merge_tree_02561 (id UInt64, info String) ENGINE = MergeTree ORDER BY id' +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${SESSION_ID_B}" -d "INSERT INTO table_merge_tree_02561 VALUES (1, 'd'), (2, 'e'), (3, 'f')" +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${SESSION_ID_B}" -d "SELECT * FROM table_merge_tree_02561" + +# Select from SESSION_ID_A +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${SESSION_ID_A}" -d "SELECT * FROM table_merge_tree_02561" + +# Drop tables in both sessions +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${SESSION_ID_A}" -d "DROP TEMPORARY TABLE table_merge_tree_02561" +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${SESSION_ID_B}" -d "DROP TEMPORARY TABLE table_merge_tree_02561" diff --git a/tests/queries/0_stateless/02572_system_logs_materialized_views_ignore_errors.reference b/tests/queries/0_stateless/02572_system_logs_materialized_views_ignore_errors.reference index d5446e756a3..029f80b46b0 100644 --- a/tests/queries/0_stateless/02572_system_logs_materialized_views_ignore_errors.reference +++ b/tests/queries/0_stateless/02572_system_logs_materialized_views_ignore_errors.reference @@ -1,2 +1,2 @@ -10 querystart OK -10 queryfinish OK +11 queryfinish OK +11 querystart OK diff --git a/tests/queries/0_stateless/02572_system_logs_materialized_views_ignore_errors.sql b/tests/queries/0_stateless/02572_system_logs_materialized_views_ignore_errors.sql index 9568bc7af1a..a7a74190821 100644 --- a/tests/queries/0_stateless/02572_system_logs_materialized_views_ignore_errors.sql +++ b/tests/queries/0_stateless/02572_system_logs_materialized_views_ignore_errors.sql @@ -9,6 +9,8 @@ set log_queries=1; drop table if exists log_proxy_02572; drop table if exists push_to_logs_proxy_mv_02572; +-- create log tables +system flush logs; create table log_proxy_02572 as system.query_log engine=Distributed('test_shard_localhost', currentDatabase(), 'receiver_02572'); create materialized view push_to_logs_proxy_mv_02572 to log_proxy_02572 as select * from system.query_log; @@ -23,4 +25,6 @@ system flush logs; -- lower() to pass through clickhouse-test "exception" check select count(), lower(type::String), errorCodeToName(exception_code) from system.query_log - where current_database = currentDatabase() group by 2, 3; + where current_database = currentDatabase() + group by 2, 3 + order by 2; diff --git a/tests/queries/0_stateless/02584_compressor_codecs.reference b/tests/queries/0_stateless/02584_compressor_codecs.reference new file mode 100644 index 00000000000..bb0850568bb --- /dev/null +++ b/tests/queries/0_stateless/02584_compressor_codecs.reference @@ -0,0 +1,9 @@ +1 +1 +1 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/02584_compressor_codecs.sh b/tests/queries/0_stateless/02584_compressor_codecs.sh new file mode 100755 index 00000000000..fad6847b792 --- /dev/null +++ b/tests/queries/0_stateless/02584_compressor_codecs.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +echo "Hello, World!" > 02584_test_data + +$CLICKHOUSE_COMPRESSOR --codec 'Delta' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' +$CLICKHOUSE_COMPRESSOR --codec 'Delta(5)' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' 2>&1 | grep -c "ILLEGAL_CODEC_PARAMETER"; +$CLICKHOUSE_COMPRESSOR --codec 'Delta([1,2])' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' 2>&1 | grep -c "ILLEGAL_CODEC_PARAMETER"; +$CLICKHOUSE_COMPRESSOR --codec 'Delta(4)' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out'; + +$CLICKHOUSE_COMPRESSOR --codec 'DoubleDelta' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' +$CLICKHOUSE_COMPRESSOR --codec 'DoubleDelta(5)' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' 2>&1 | grep -c "ILLEGAL_CODEC_PARAMETER"; +$CLICKHOUSE_COMPRESSOR --codec 'DoubleDelta([1,2])' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' 2>&1 | grep -c "ILLEGAL_CODEC_PARAMETER"; +$CLICKHOUSE_COMPRESSOR --codec 'DoubleDelta(4)' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out'; + +$CLICKHOUSE_COMPRESSOR --codec 'Gorilla' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' +$CLICKHOUSE_COMPRESSOR --codec 'Gorilla(5)' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' 2>&1 | grep -c "ILLEGAL_CODEC_PARAMETER"; +$CLICKHOUSE_COMPRESSOR --codec 'Gorilla([1,2])' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' 2>&1 | grep -c "ILLEGAL_CODEC_PARAMETER"; +$CLICKHOUSE_COMPRESSOR --codec 'Gorilla(4)' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out'; + +$CLICKHOUSE_COMPRESSOR --codec 'FPC' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out'; +$CLICKHOUSE_COMPRESSOR --codec 'FPC(5)' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out'; +$CLICKHOUSE_COMPRESSOR --codec 'FPC(5, 1)' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' 2>&1 | grep -c "ILLEGAL_CODEC_PARAMETER"; +$CLICKHOUSE_COMPRESSOR --codec 'FPC([1,2,3])' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' 2>&1 | grep -c "ILLEGAL_CODEC_PARAMETER"; +$CLICKHOUSE_COMPRESSOR --codec 'FPC(5, 4)' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out'; + + +$CLICKHOUSE_COMPRESSOR --codec 'T64' --codec 'LZ4' --input '02584_test_data' --output '02584_test_out' 2>&1 | grep -c "CANNOT_COMPRESS"; + +rm 02584_test_data 02584_test_out + diff --git a/tests/queries/0_stateless/02676_distinct_reading_in_order_analyzer.reference b/tests/queries/0_stateless/02676_distinct_reading_in_order_analyzer.reference new file mode 100644 index 00000000000..016202cfb66 --- /dev/null +++ b/tests/queries/0_stateless/02676_distinct_reading_in_order_analyzer.reference @@ -0,0 +1 @@ +MergeTreeInOrder diff --git a/tests/queries/0_stateless/02676_distinct_reading_in_order_analyzer.sql b/tests/queries/0_stateless/02676_distinct_reading_in_order_analyzer.sql new file mode 100644 index 00000000000..f00c1322e1d --- /dev/null +++ b/tests/queries/0_stateless/02676_distinct_reading_in_order_analyzer.sql @@ -0,0 +1,8 @@ +drop table if exists t; + +set allow_experimental_analyzer=1; + +create table t (a UInt64, b UInt64) engine=MergeTree() order by (a); +insert into t select number % 2, number from numbers(10); + +select splitByChar(' ', trimBoth(explain))[1] from (explain pipeline select distinct a from t) where explain like '%MergeTreeInOrder%'; diff --git a/tests/queries/0_stateless/02677_analyzer_bitmap_has_any.reference b/tests/queries/0_stateless/02677_analyzer_bitmap_has_any.reference new file mode 100644 index 00000000000..16d7e43ecb3 --- /dev/null +++ b/tests/queries/0_stateless/02677_analyzer_bitmap_has_any.reference @@ -0,0 +1,4 @@ +1 0 +-------------- +-------------- +1 0 diff --git a/tests/queries/0_stateless/02677_analyzer_bitmap_has_any.sql b/tests/queries/0_stateless/02677_analyzer_bitmap_has_any.sql new file mode 100644 index 00000000000..4af06634c66 --- /dev/null +++ b/tests/queries/0_stateless/02677_analyzer_bitmap_has_any.sql @@ -0,0 +1,36 @@ +SELECT + bitmapHasAny(bitmapBuild([toUInt8(1)]), ( + SELECT groupBitmapState(toUInt8(1)) + )) has1, + bitmapHasAny(bitmapBuild([toUInt64(1)]), ( + SELECT groupBitmapState(toUInt64(2)) + )) has2; + +SELECT '--------------'; + +SELECT * +FROM +( + SELECT + bitmapHasAny(bitmapBuild([toUInt8(1)]), ( + SELECT groupBitmapState(toUInt8(1)) + )) has1, + bitmapHasAny(bitmapBuild([toUInt64(1)]), ( + SELECT groupBitmapState(toUInt64(2)) + )) has2 +); -- { serverError 43 } + +SELECT '--------------'; + +SELECT * +FROM +( + SELECT + bitmapHasAny(bitmapBuild([toUInt8(1)]), ( + SELECT groupBitmapState(toUInt8(1)) + )) has1, + bitmapHasAny(bitmapBuild([toUInt64(1)]), ( + SELECT groupBitmapState(toUInt64(2)) + )) has2 +) SETTINGS allow_experimental_analyzer = 1; + diff --git a/tests/queries/0_stateless/02679_explain_merge_tree_prewhere_row_policy.reference b/tests/queries/0_stateless/02679_explain_merge_tree_prewhere_row_policy.reference new file mode 100644 index 00000000000..2f3b59e5530 --- /dev/null +++ b/tests/queries/0_stateless/02679_explain_merge_tree_prewhere_row_policy.reference @@ -0,0 +1,26 @@ +Expression ((Projection + Before ORDER BY)) +Header: id UInt64 + value String +Actions: INPUT :: 0 -> id UInt64 : 0 + INPUT :: 1 -> value String : 1 +Positions: 0 1 + ReadFromMergeTree (default.test_table) + Header: id UInt64 + value String + ReadType: Default + Parts: 0 + Granules: 0 + Prewhere info + Need filter: 1 + Prewhere filter + Prewhere filter column: equals(id, 5) (removed) + Actions: INPUT : 0 -> id UInt64 : 0 + COLUMN Const(UInt8) -> 5 UInt8 : 1 + FUNCTION equals(id : 0, 5 :: 1) -> equals(id, 5) UInt8 : 2 + Positions: 2 0 + Row level filter + Row level filter column: greaterOrEquals(id, 5) + Actions: INPUT : 0 -> id UInt64 : 0 + COLUMN Const(UInt8) -> 5 UInt8 : 1 + FUNCTION greaterOrEquals(id : 0, 5 :: 1) -> greaterOrEquals(id, 5) UInt8 : 2 + Positions: 2 0 diff --git a/tests/queries/0_stateless/02679_explain_merge_tree_prewhere_row_policy.sql b/tests/queries/0_stateless/02679_explain_merge_tree_prewhere_row_policy.sql new file mode 100644 index 00000000000..8099ccc0b0d --- /dev/null +++ b/tests/queries/0_stateless/02679_explain_merge_tree_prewhere_row_policy.sql @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS test_table; +CREATE TABLE test_table +( + id UInt64, + value String +) ENGINE=MergeTree ORDER BY id; + +INSERT INTO test_table VALUES (0, 'Value'); + +DROP ROW POLICY IF EXISTS test_row_policy ON test_table; +CREATE ROW POLICY test_row_policy ON test_table USING id >= 5 TO ALL; + +EXPLAIN header = 1, actions = 1 SELECT id, value FROM test_table PREWHERE id = 5; + +DROP ROW POLICY test_row_policy ON test_table; +DROP TABLE test_table; diff --git a/tests/queries/0_stateless/02680_instr_alias_for_position_case_insensitive.reference b/tests/queries/0_stateless/02680_instr_alias_for_position_case_insensitive.reference new file mode 100644 index 00000000000..4792e70f333 --- /dev/null +++ b/tests/queries/0_stateless/02680_instr_alias_for_position_case_insensitive.reference @@ -0,0 +1,2 @@ +2 +3 diff --git a/tests/queries/0_stateless/02680_instr_alias_for_position_case_insensitive.sql b/tests/queries/0_stateless/02680_instr_alias_for_position_case_insensitive.sql new file mode 100644 index 00000000000..c1c55c2c982 --- /dev/null +++ b/tests/queries/0_stateless/02680_instr_alias_for_position_case_insensitive.sql @@ -0,0 +1,2 @@ +select INSTR('hello', 'e'); +select INSTR('hELlo', 'L'); diff --git a/tests/queries/0_stateless/25340_storage_join_insert_select_deadlock.reference b/tests/queries/0_stateless/25340_storage_join_insert_select_deadlock.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/25340_storage_join_insert_select_deadlock.sql b/tests/queries/0_stateless/25340_storage_join_insert_select_deadlock.sql new file mode 100644 index 00000000000..59528511357 --- /dev/null +++ b/tests/queries/0_stateless/25340_storage_join_insert_select_deadlock.sql @@ -0,0 +1,16 @@ +DROP TABLE IF EXISTS test_table_join; + +CREATE TABLE test_table_join +( + id UInt64, + value String +) ENGINE = Join(Any, Left, id); + +INSERT INTO test_table_join VALUES (1, 'q'); + +INSERT INTO test_table_join SELECT * from test_table_join; -- { serverError DEADLOCK_AVOIDED } + +INSERT INTO test_table_join SELECT * FROM (SELECT 1 as id) AS t1 ANY LEFT JOIN test_table_join USING (id); -- { serverError DEADLOCK_AVOIDED } +INSERT INTO test_table_join SELECT id, toString(id) FROM (SELECT 1 as id) AS t1 ANY LEFT JOIN (SELECT id FROM test_table_join) AS t2 USING (id); -- { serverError DEADLOCK_AVOIDED } + +DROP TABLE IF EXISTS test_table_join; diff --git a/tests/queries/0_stateless/25341_inverted_idx_checksums.reference b/tests/queries/0_stateless/25341_inverted_idx_checksums.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/25341_inverted_idx_checksums.sql b/tests/queries/0_stateless/25341_inverted_idx_checksums.sql new file mode 100644 index 00000000000..92ffa7a6196 --- /dev/null +++ b/tests/queries/0_stateless/25341_inverted_idx_checksums.sql @@ -0,0 +1,16 @@ +SET allow_experimental_inverted_index = 1; + +CREATE TABLE t +( + `key` UInt64, + `str` String, + INDEX inv_idx str TYPE inverted(0) GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY key; + +INSERT INTO t VALUES (1, 'Hello World'); + +ALTER TABLE t DETACH PART 'all_1_1_0'; + +ALTER TABLE t ATTACH PART 'all_1_1_0'; \ No newline at end of file diff --git a/tests/queries/0_stateless/25400_marked_dropped_tables.reference b/tests/queries/0_stateless/25400_marked_dropped_tables.reference new file mode 100644 index 00000000000..6fc5caff0cb --- /dev/null +++ b/tests/queries/0_stateless/25400_marked_dropped_tables.reference @@ -0,0 +1,8 @@ +25400_marked_dropped_tables MergeTree +index UInt32 +database String +table String +uuid UUID +engine String +metadata_dropped_path String +table_dropped_time DateTime diff --git a/tests/queries/0_stateless/25400_marked_dropped_tables.sql b/tests/queries/0_stateless/25400_marked_dropped_tables.sql new file mode 100644 index 00000000000..101642fa779 --- /dev/null +++ b/tests/queries/0_stateless/25400_marked_dropped_tables.sql @@ -0,0 +1,11 @@ +-- Tags: no-ordinary-database + +SET database_atomic_wait_for_drop_and_detach_synchronously = 0; +DROP TABLE IF EXISTS 25400_marked_dropped_tables; + +CREATE TABLE 25400_marked_dropped_tables (id Int32) Engine=MergeTree() ORDER BY id; +DROP TABLE 25400_marked_dropped_tables; + +SELECT table, engine FROM system.marked_dropped_tables WHERE database = currentDatabase() LIMIT 1; +DESCRIBE TABLE system.marked_dropped_tables; + diff --git a/utils/check-style/check-style b/utils/check-style/check-style index 53165d14f96..f9b7a9bbbb7 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -392,3 +392,6 @@ find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | # Check for existence of __init__.py files for i in "${ROOT_PATH}"/tests/integration/test_*; do FILE="${i}/__init__.py"; [ ! -f "${FILE}" ] && echo "${FILE} should exist for every integration test"; done + +# A small typo can lead to debug code in release builds, see https://github.com/ClickHouse/ClickHouse/pull/47647 +find $ROOT_PATH/{src,programs,utils} -name '*.h' -or -name '*.cpp' | xargs grep -l -F '#ifdef NDEBUG' | xargs -I@FILE awk '/#ifdef NDEBUG/ { inside = 1; dirty = 1 } /#endif/ { if (inside && dirty) { print "File @FILE has suspicious #ifdef NDEBUG, possibly confused with #ifndef NDEBUG" }; inside = 0 } /#else/ { dirty = 0 }' @FILE