diff --git a/.clang-tidy b/.clang-tidy index 5da1d309f62..e9451272681 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -16,6 +16,7 @@ Checks: '*, -android-*, + -bugprone-assignment-in-if-condition, -bugprone-branch-clone, -bugprone-easily-swappable-parameters, -bugprone-exception-escape, @@ -23,7 +24,6 @@ Checks: '*, -bugprone-narrowing-conversions, -bugprone-not-null-terminated-result, -bugprone-unchecked-optional-access, - -bugprone-assignment-in-if-condition, -cert-dcl16-c, -cert-err58-cpp, @@ -34,7 +34,6 @@ Checks: '*, -clang-analyzer-optin.performance.Padding, -clang-analyzer-optin.portability.UnixAPI, - -clang-analyzer-security.insecureAPI.bzero, -clang-analyzer-security.insecureAPI.strcpy, @@ -103,12 +102,13 @@ Checks: '*, -openmp-*, + -misc-const-correctness, -misc-no-recursion, -misc-non-private-member-variables-in-classes, - -misc-const-correctness, -modernize-avoid-c-arrays, -modernize-concat-nested-namespaces, + -modernize-macro-to-enum, -modernize-pass-by-value, -modernize-return-braced-init-list, -modernize-use-auto, @@ -117,7 +117,6 @@ Checks: '*, -modernize-use-nodiscard, -modernize-use-override, -modernize-use-trailing-return-type, - -modernize-macro-to-enum, -performance-inefficient-string-concatenation, -performance-no-int-to-ptr, @@ -135,17 +134,35 @@ Checks: '*, -readability-magic-numbers, -readability-named-parameter, -readability-redundant-declaration, + -readability-simplify-boolean-expr, -readability-static-accessed-through-instance, -readability-suspicious-call-argument, -readability-uppercase-literal-suffix, -readability-use-anyofallof, - -readability-simplify-boolean-expr, -zirkon-*, + + -misc-*, # temporarily disabled due to being too slow + # also disable checks in other categories which are aliases of checks in misc-*: + # https://releases.llvm.org/15.0.0/tools/clang/tools/extra/docs/clang-tidy/checks/list.html + -cert-dcl54-cpp, # alias of misc-new-delete-overloads + -hicpp-new-delete-operators, # alias of misc-new-delete-overloads + -cert-fio38-c, # alias of misc-non-copyable-objects + -cert-dcl03-c, # alias of misc-static-assert + -hicpp-static-assert, # alias of misc-static-assert + -cert-err09-cpp, # alias of misc-throw-by-value-catch-by-reference + -cert-err61-cpp, # alias of misc-throw-by-value-catch-by-reference + -cppcoreguidelines-c-copy-assignment-signature, # alias of misc-unconventional-assign-operator + -cppcoreguidelines-non-private-member-variables-in-classes, # alias of misc-non-private-member-variables-in-classes ' WarningsAsErrors: '*' +# TODO: use dictionary syntax for CheckOptions when minimum clang-tidy level rose to 15 +# some-check.SomeOption: 'some value' +# instead of +# - key: some-check.SomeOption +# value: 'some value' CheckOptions: - key: readability-identifier-naming.ClassCase value: CamelCase diff --git a/.github/workflows/tags_stable.yml b/.github/workflows/tags_stable.yml index 1b03b6fde3c..f8cfa1137cc 100644 --- a/.github/workflows/tags_stable.yml +++ b/.github/workflows/tags_stable.yml @@ -61,6 +61,7 @@ jobs: committer: "robot-clickhouse " commit-message: Update version_date.tsv and changelogs after ${{ env.GITHUB_TAG }} branch: auto/${{ env.GITHUB_TAG }} + assignees: ${{ github.event.sender.login }} # assign the PR to the tag pusher delete-branch: true title: Update version_date.tsv and changelogs after ${{ env.GITHUB_TAG }} labels: do not test diff --git a/.gitmodules b/.gitmodules index ebeef312ae8..a4cfcc91485 100644 --- a/.gitmodules +++ b/.gitmodules @@ -287,3 +287,6 @@ [submodule "contrib/xxHash"] path = contrib/xxHash url = https://github.com/Cyan4973/xxHash.git +[submodule "contrib/google-benchmark"] + path = contrib/google-benchmark + url = https://github.com/google/benchmark.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 06e6f943fd3..e121559d4e0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,6 +111,7 @@ if (ENABLE_FUZZING) set (ENABLE_JEMALLOC 0) set (ENABLE_CHECK_HEAVY_BUILDS 1) set (GLIBC_COMPATIBILITY OFF) + set (ENABLE_BENCHMARKS 0) # For codegen_select_fuzzer set (ENABLE_PROTOBUF 1) @@ -168,6 +169,7 @@ endif () option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" ON) option(ENABLE_EXAMPLES "Build all example programs in 'examples' subdirectories" OFF) +option(ENABLE_BENCHMARKS "Build all benchmark programs in 'benchmarks' subdirectories" OFF) if (OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64) AND USE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND NOT USE_MUSL) # Only for Linux, x86_64 or aarch64. diff --git a/README.md b/README.md index 4f2483097d6..59c9c180c90 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,6 @@ ClickHouse® is an open-source column-oriented database management system that a * [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any. ## Upcoming events -* [**v22.11 Release Webinar**](https://clickhouse.com/company/events/v22-11-release-webinar) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap. -* [**ClickHosue Meetup at the RELEX Solutions office in Stockholm**](https://www.meetup.com/clickhouse-stockholm-user-group/events/289492084/) - Dec 1 - Formulate by RELEX is a Swedish promotion planning and analytics company. They will share why they chose ClickHouse for their real time analytics and forecasting solution. The ClickHouse team will then present how ClickHouse is used for real time financial data analytics, including tick data, trade analytics and risk management. -* [**ClickHouse Meetup at the Deutsche Bank office in Berlin**](https://www.meetup.com/clickhouse-berlin-user-group/events/289311596/) - Dec 5 - Hear from Deutsche Bank on why they chose ClickHouse for big sensitive data in a regulated environment. The ClickHouse team will then present how ClickHouse is used for real time financial data analytics, including tick data, trade analytics and risk management. -* [**ClickHouse Meetup at the Rokt offices in Manhattan**](https://www.meetup.com/clickhouse-new-york-user-group/events/289403909/) - Dec 6 - We are very excited to be holding our next in-person ClickHouse meetup at the Rokt offices in Manhattan. Featuring talks from Bloomberg, Disney Streaming, Prequel, Rokt, and ClickHouse - +* [**v22.12 Release Webinar**](https://clickhouse.com/company/events/v22-12-release-webinar) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap. +* [**ClickHouse Meetup at the CHEQ office in Tel Aviv**](https://www.meetup.com/clickhouse-tel-aviv-user-group/events/289599423/) - Jan 16 - We are very excited to be holding our next in-person ClickHouse meetup at the CHEQ office in Tel Aviv! Hear from CHEQ, ServiceNow and Contentsquare, as well as a deep dive presentation from ClickHouse CTO Alexey Milovidov. Join us for a fun evening of talks, food and discussion! +* **ClickHouse Meetup in Seattle* - Keep an eye on this space as we will be announcing a January meetup in Seattle soon! diff --git a/base/base/ReplxxLineReader.cpp b/base/base/ReplxxLineReader.cpp index e0dc81af5b0..b86746365b7 100644 --- a/base/base/ReplxxLineReader.cpp +++ b/base/base/ReplxxLineReader.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include /// is_any_of namespace @@ -38,7 +39,7 @@ std::string getEditor() return editor; } -std::string getFuzzyFinder() +std::pair getFuzzyFinder() { const char * env_path = std::getenv("PATH"); // NOLINT(concurrency-mt-unsafe) @@ -52,14 +53,20 @@ std::string getFuzzyFinder() std::filesystem::path path(path_str); std::filesystem::path sk_bin_path = path / "sk"; if (!access(sk_bin_path.c_str(), X_OK)) - return sk_bin_path; + return {sk_bin_path, FUZZY_FINDER_SKIM}; std::filesystem::path fzf_bin_path = path / "fzf"; if (!access(fzf_bin_path.c_str(), X_OK)) - return fzf_bin_path; + return {fzf_bin_path, FUZZY_FINDER_FZF}; } - return {}; + return {"", FUZZY_FINDER_NONE}; +} + +String escapeShellArgument(std::string arg) +{ + boost::replace_all(arg, "'", "'\\''"); + return fmt::format("'{}'", arg); } /// See comments in ShellCommand::executeImpl() @@ -305,11 +312,12 @@ ReplxxLineReader::ReplxxLineReader( replxx::Replxx::highlighter_callback_t highlighter_) : LineReader(history_file_path_, multiline_, std::move(extenders_), std::move(delimiters_)), highlighter(std::move(highlighter_)) , editor(getEditor()) - , fuzzy_finder(getFuzzyFinder()) { using namespace std::placeholders; using Replxx = replxx::Replxx; + std::tie(fuzzy_finder, fuzzy_finder_type) = getFuzzyFinder(); + if (!history_file_path.empty()) { history_file_fd = open(history_file_path.c_str(), O_RDWR); @@ -415,11 +423,12 @@ ReplxxLineReader::ReplxxLineReader( rx.bind_key(Replxx::KEY::meta('#'), insert_comment_action); /// interactive search in history (requires fzf/sk) - if (!fuzzy_finder.empty()) + if (fuzzy_finder_type != FUZZY_FINDER_NONE) { auto interactive_history_search = [this](char32_t code) { openInteractiveHistorySearch(); + rx.invoke(Replxx::ACTION::CLEAR_SELF, code); return rx.invoke(Replxx::ACTION::REPAINT, code); }; rx.bind_key(Replxx::KEY::control('R'), interactive_history_search); @@ -515,9 +524,22 @@ void ReplxxLineReader::openInteractiveHistorySearch() /// /// And also note, that fzf and skim is 95% compatible (at least option /// that is used here) - std::string fuzzy_finder_command = fmt::format( - "{} --read0 --tac --no-sort --tiebreak=index --bind=ctrl-r:toggle-sort --height=30% < {} > {}", - fuzzy_finder, history_file.getPath(), output_file.getPath()); + std::string fuzzy_finder_command = fmt::format("{} --read0 --height=30%", fuzzy_finder); + switch (fuzzy_finder_type) + { + case FUZZY_FINDER_SKIM: + fuzzy_finder_command += " --tac --tiebreak=-score"; + break; + case FUZZY_FINDER_FZF: + fuzzy_finder_command += " --tac --tiebreak=index"; + break; + case FUZZY_FINDER_NONE: + /// assertion for !fuzzy_finder.empty() is enough + break; + } + fuzzy_finder_command += fmt::format(" < {} > {}", + escapeShellArgument(history_file.getPath()), + escapeShellArgument(output_file.getPath())); char * const argv[] = {sh, sh_c, fuzzy_finder_command.data(), nullptr}; try diff --git a/base/base/ReplxxLineReader.h b/base/base/ReplxxLineReader.h index fea1405a208..9be3b3aa993 100644 --- a/base/base/ReplxxLineReader.h +++ b/base/base/ReplxxLineReader.h @@ -4,6 +4,14 @@ #include +enum FuzzyFinderType +{ + FUZZY_FINDER_NONE, + /// Use https://github.com/junegunn/fzf + FUZZY_FINDER_FZF, + /// Use https://github.com/lotabout/skim + FUZZY_FINDER_SKIM, +}; class ReplxxLineReader : public LineReader { @@ -38,4 +46,5 @@ private: std::string editor; std::string fuzzy_finder; + FuzzyFinderType fuzzy_finder_type = FUZZY_FINDER_NONE; }; diff --git a/base/base/wide_integer_impl.h b/base/base/wide_integer_impl.h index 1b5f502722c..f5b30cbab55 100644 --- a/base/base/wide_integer_impl.h +++ b/base/base/wide_integer_impl.h @@ -187,8 +187,20 @@ struct integer::_impl static_assert(Bits % base_bits == 0); /// Simple iteration in both directions - static constexpr unsigned little(unsigned idx) { return idx; } - static constexpr unsigned big(unsigned idx) { return item_count - 1 - idx; } + static constexpr unsigned little(unsigned idx) + { + if constexpr (std::endian::native == std::endian::little) + return idx; + else + return item_count - 1 - idx; + } + static constexpr unsigned big(unsigned idx) + { + if constexpr (std::endian::native == std::endian::little) + return item_count - 1 - idx; + else + return idx; + } static constexpr unsigned any(unsigned idx) { return idx; } template @@ -240,20 +252,20 @@ struct integer::_impl { static_assert(sizeof(Integral) <= sizeof(base_type)); - self.items[0] = _impl::to_Integral(rhs); + self.items[little(0)] = _impl::to_Integral(rhs); if constexpr (std::is_signed_v) { if (rhs < 0) { - for (size_t i = 1; i < item_count; ++i) - self.items[i] = -1; + for (unsigned i = 1; i < item_count; ++i) + self.items[little(i)] = -1; return; } } - for (size_t i = 1; i < item_count; ++i) - self.items[i] = 0; + for (unsigned i = 1; i < item_count; ++i) + self.items[little(i)] = 0; } template @@ -348,7 +360,7 @@ struct integer::_impl constexpr const unsigned to_copy = min_bits / base_bits; for (unsigned i = 0; i < to_copy; ++i) - self.items[i] = rhs.items[i]; + self.items[little(i)] = rhs.items[little(i)]; if constexpr (Bits > Bits2) { @@ -357,13 +369,13 @@ struct integer::_impl if (rhs < 0) { for (unsigned i = to_copy; i < item_count; ++i) - self.items[i] = -1; + self.items[little(i)] = -1; return; } } for (unsigned i = to_copy; i < item_count; ++i) - self.items[i] = 0; + self.items[little(i)] = 0; } } @@ -454,7 +466,7 @@ private: { if constexpr (sizeof(T) <= sizeof(base_type)) { - if (0 == idx) + if (little(0) == idx) return static_cast(x); } else if (idx * sizeof(base_type) < sizeof(T)) @@ -475,7 +487,7 @@ private: for (unsigned i = 0; i < op_items; ++i) { - base_type rhs_item = get_item(rhs, i); + base_type rhs_item = get_item(rhs, little(i)); base_type & res_item = res.items[little(i)]; underflows[i] = res_item < rhs_item; @@ -508,7 +520,7 @@ private: for (unsigned i = 0; i < op_items; ++i) { - base_type rhs_item = get_item(rhs, i); + base_type rhs_item = get_item(rhs, little(i)); base_type & res_item = res.items[little(i)]; res_item += rhs_item; @@ -580,12 +592,12 @@ private: else if constexpr (Bits == 128 && sizeof(base_type) == 8) { using CompilerUInt128 = unsigned __int128; - CompilerUInt128 a = (CompilerUInt128(lhs.items[1]) << 64) + lhs.items[0]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult) - CompilerUInt128 b = (CompilerUInt128(rhs.items[1]) << 64) + rhs.items[0]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult) + CompilerUInt128 a = (CompilerUInt128(lhs.items[little(1)]) << 64) + lhs.items[little(0)]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult) + CompilerUInt128 b = (CompilerUInt128(rhs.items[little(1)]) << 64) + rhs.items[little(0)]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult) CompilerUInt128 c = a * b; integer res; - res.items[0] = c; - res.items[1] = c >> 64; + res.items[little(0)] = c; + res.items[little(1)] = c >> 64; return res; } else @@ -597,7 +609,7 @@ private: #endif for (unsigned i = 0; i < item_count; ++i) { - base_type rhs_item = get_item(rhs, i); + base_type rhs_item = get_item(rhs, little(i)); unsigned pos = i * base_bits; while (rhs_item) @@ -792,7 +804,7 @@ public: integer res; for (unsigned i = 0; i < item_count; ++i) - res.items[little(i)] = lhs.items[little(i)] | get_item(rhs, i); + res.items[little(i)] = lhs.items[little(i)] | get_item(rhs, little(i)); return res; } else @@ -810,7 +822,7 @@ public: integer res; for (unsigned i = 0; i < item_count; ++i) - res.items[little(i)] = lhs.items[little(i)] & get_item(rhs, i); + res.items[little(i)] = lhs.items[little(i)] & get_item(rhs, little(i)); return res; } else @@ -845,17 +857,17 @@ public: { using CompilerUInt128 = unsigned __int128; - CompilerUInt128 a = (CompilerUInt128(numerator.items[1]) << 64) + numerator.items[0]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult) - CompilerUInt128 b = (CompilerUInt128(denominator.items[1]) << 64) + denominator.items[0]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult) + CompilerUInt128 a = (CompilerUInt128(numerator.items[little(1)]) << 64) + numerator.items[little(0)]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult) + CompilerUInt128 b = (CompilerUInt128(denominator.items[little(1)]) << 64) + denominator.items[little(0)]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult) CompilerUInt128 c = a / b; // NOLINT integer res; - res.items[0] = c; - res.items[1] = c >> 64; + res.items[little(0)] = c; + res.items[little(1)] = c >> 64; CompilerUInt128 remainder = a - b * c; - numerator.items[0] = remainder; - numerator.items[1] = remainder >> 64; + numerator.items[little(0)] = remainder; + numerator.items[little(1)] = remainder >> 64; return res; } @@ -1039,15 +1051,15 @@ constexpr integer::integer(std::initializer_list il) noexcept else { auto it = il.begin(); - for (size_t i = 0; i < _impl::item_count; ++i) + for (unsigned i = 0; i < _impl::item_count; ++i) { if (it < il.end()) { - items[i] = *it; + items[_impl::little(i)] = *it; ++it; } else - items[i] = 0; + items[_impl::little(i)] = 0; } } } @@ -1208,7 +1220,7 @@ constexpr integer::operator T() const noexcept UnsignedT res{}; for (unsigned i = 0; i < _impl::item_count && i < (sizeof(T) + sizeof(base_type) - 1) / sizeof(base_type); ++i) - res += UnsignedT(items[i]) << (sizeof(base_type) * 8 * i); // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult) + res += UnsignedT(items[_impl::little(i)]) << (sizeof(base_type) * 8 * i); // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult) return res; } diff --git a/cmake/clang_tidy.cmake b/cmake/clang_tidy.cmake index 57295682487..ceaafdaa9aa 100644 --- a/cmake/clang_tidy.cmake +++ b/cmake/clang_tidy.cmake @@ -5,21 +5,21 @@ if (ENABLE_CLANG_TIDY) find_program (CLANG_TIDY_CACHE_PATH NAMES "clang-tidy-cache") if (CLANG_TIDY_CACHE_PATH) - find_program (_CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-15" "clang-tidy-14" "clang-tidy-13" "clang-tidy-12") + find_program (_CLANG_TIDY_PATH NAMES "clang-tidy-15" "clang-tidy-14" "clang-tidy-13" "clang-tidy-12" "clang-tidy") # Why do we use ';' here? # It's a cmake black magic: https://cmake.org/cmake/help/latest/prop_tgt/LANG_CLANG_TIDY.html#prop_tgt:%3CLANG%3E_CLANG_TIDY # The CLANG_TIDY_PATH is passed to CMAKE_CXX_CLANG_TIDY, which follows CXX_CLANG_TIDY syntax. set (CLANG_TIDY_PATH "${CLANG_TIDY_CACHE_PATH};${_CLANG_TIDY_PATH}" CACHE STRING "A combined command to run clang-tidy with caching wrapper") else () - find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-15" "clang-tidy-14" "clang-tidy-13" "clang-tidy-12") + find_program (CLANG_TIDY_PATH NAMES "clang-tidy-15" "clang-tidy-14" "clang-tidy-13" "clang-tidy-12" "clang-tidy") endif () if (CLANG_TIDY_PATH) message (STATUS "Using clang-tidy: ${CLANG_TIDY_PATH}. - The checks will be run during build process. - See the .clang-tidy file at the root directory to configure the checks.") + The checks will be run during the build process. + See the .clang-tidy file in the root directory to configure the checks.") set (USE_CLANG_TIDY ON) diff --git a/cmake/tools.cmake b/cmake/tools.cmake index e8fecd9f30b..3ddf8a869be 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -21,12 +21,12 @@ set (APPLE_CLANG_MINIMUM_VERSION 12.0.0) set (GCC_MINIMUM_VERSION 11) if (COMPILER_GCC) + message (FATAL_ERROR "Compilation with GCC is unsupported. Please use Clang instead.") + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${GCC_MINIMUM_VERSION}) message (FATAL_ERROR "Compilation with GCC version ${CMAKE_CXX_COMPILER_VERSION} is unsupported, the minimum required version is ${GCC_MINIMUM_VERSION}.") endif () - message (WARNING "Compilation with GCC is unsupported. Please use Clang instead.") - elseif (COMPILER_CLANG) if (CMAKE_CXX_COMPILER_ID MATCHES "AppleClang") # (Experimental!) Specify "-DALLOW_APPLECLANG=ON" when running CMake configuration step, if you want to experiment with using it. @@ -83,7 +83,7 @@ if ((OS_LINUX OR OS_DARWIN) AND NOT LINKER_NAME) if (NOT LINKER_NAME) if (GOLD_PATH) - message (WARNING "Linking with gold is not recommended. Please use lld.") + message (FATAL_ERROR "Linking with gold is unsupported. Please use lld.") if (COMPILER_GCC) set (LINKER_NAME "gold") else () diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index ec7382846c2..c7419d74aac 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -171,6 +171,8 @@ add_contrib (annoy-cmake annoy) add_contrib (xxHash-cmake xxHash) +add_contrib (google-benchmark-cmake google-benchmark) + # Put all targets defined here and in subdirectories under "contrib/" folders in GUI-based IDEs. # Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear # in "contrib/..." as originally planned, so we workaround this by fixing FOLDER properties of all targets manually, diff --git a/contrib/google-benchmark b/contrib/google-benchmark new file mode 160000 index 00000000000..2257fa4d6af --- /dev/null +++ b/contrib/google-benchmark @@ -0,0 +1 @@ +Subproject commit 2257fa4d6afb8e5a2ccd510a70f38fe7fcdf1edf diff --git a/contrib/google-benchmark-cmake/CMakeLists.txt b/contrib/google-benchmark-cmake/CMakeLists.txt new file mode 100644 index 00000000000..5d8fa7b838b --- /dev/null +++ b/contrib/google-benchmark-cmake/CMakeLists.txt @@ -0,0 +1,34 @@ +set (SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/google-benchmark/src") + +set (SRCS + "${SRC_DIR}/benchmark.cc" + "${SRC_DIR}/benchmark_api_internal.cc" + "${SRC_DIR}/benchmark_name.cc" + "${SRC_DIR}/benchmark_register.cc" + "${SRC_DIR}/benchmark_runner.cc" + "${SRC_DIR}/check.cc" + "${SRC_DIR}/colorprint.cc" + "${SRC_DIR}/commandlineflags.cc" + "${SRC_DIR}/complexity.cc" + "${SRC_DIR}/console_reporter.cc" + "${SRC_DIR}/counter.cc" + "${SRC_DIR}/csv_reporter.cc" + "${SRC_DIR}/json_reporter.cc" + "${SRC_DIR}/perf_counters.cc" + "${SRC_DIR}/reporter.cc" + "${SRC_DIR}/sleep.cc" + "${SRC_DIR}/statistics.cc" + "${SRC_DIR}/string_util.cc" + "${SRC_DIR}/sysinfo.cc" + "${SRC_DIR}/timers.cc") + +add_library(google_benchmark "${SRCS}") +target_include_directories(google_benchmark SYSTEM PUBLIC "${SRC_DIR}/../include") + +add_library(google_benchmark_main "${SRC_DIR}/benchmark_main.cc") +target_link_libraries(google_benchmark_main PUBLIC google_benchmark) + +add_library(google_benchmark_all INTERFACE) +target_link_libraries(google_benchmark_all INTERFACE google_benchmark google_benchmark_main) + +add_library(ch_contrib::gbenchmark_all ALIAS google_benchmark_all) diff --git a/contrib/qpl b/contrib/qpl index cdc8442f7a5..becb7a1b15b 160000 --- a/contrib/qpl +++ b/contrib/qpl @@ -1 +1 @@ -Subproject commit cdc8442f7a5e7a6ff6eea39c69665e0c5034d85d +Subproject commit becb7a1b15bdb4845ec3721a550707ffa51d029d diff --git a/contrib/qpl-cmake/CMakeLists.txt b/contrib/qpl-cmake/CMakeLists.txt index dc90f07a9bc..beef8432e7a 100644 --- a/contrib/qpl-cmake/CMakeLists.txt +++ b/contrib/qpl-cmake/CMakeLists.txt @@ -15,7 +15,7 @@ set (QPL_SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl/sources") set (QPL_BINARY_DIR "${ClickHouse_BINARY_DIR}/build/contrib/qpl") set (UUID_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl-cmake") -set (EFFICIENT_WAIT ON) +set (EFFICIENT_WAIT OFF) set (BLOCK_ON_FAULT ON) set (LOG_HW_INIT OFF) set (SANITIZE_MEMORY OFF) @@ -42,7 +42,7 @@ include("${QPL_PROJECT_DIR}/cmake/CompileOptions.cmake") include(CheckLanguage) check_language(ASM_NASM) if(NOT CMAKE_ASM_NASM_COMPILER) - message(FATAL_ERROR "Please install NASM from 'https://www.nasm.us/' because NASM compiler can not be found!") + message(FATAL_ERROR "Please install NASM from 'https://www.nasm.us/' because NASM compiler can not be found!") endif() # [SUBDIR]isal @@ -110,18 +110,18 @@ target_compile_options(isal PRIVATE "$<$:>" "$<$:>") -target_compile_options(isal_asm PUBLIC "-I${QPL_SRC_DIR}/isal/include/" - PUBLIC "-I${QPL_SRC_DIR}/isal/igzip/" - PUBLIC "-I${QPL_SRC_DIR}/isal/crc/" - PUBLIC "-DQPL_LIB") +target_compile_options(isal_asm PRIVATE "-I${QPL_SRC_DIR}/isal/include/" + PRIVATE "-I${QPL_SRC_DIR}/isal/igzip/" + PRIVATE "-I${QPL_SRC_DIR}/isal/crc/" + PRIVATE "-DQPL_LIB") # AS_FEATURE_LEVEL=10 means "Check SIMD capabilities of the target system at runtime and use up to AVX512 if available". # AS_FEATURE_LEVEL=5 means "Check SIMD capabilities of the target system at runtime and use up to AVX2 if available". # HAVE_KNOWS_AVX512 means rely on AVX512 being available on the target system. if (ENABLE_AVX512) - target_compile_options(isal_asm PUBLIC "-DHAVE_AS_KNOWS_AVX512" "-DAS_FEATURE_LEVEL=10") + target_compile_options(isal_asm PRIVATE "-DHAVE_AS_KNOWS_AVX512" "-DAS_FEATURE_LEVEL=10") else() - target_compile_options(isal_asm PUBLIC "-DAS_FEATURE_LEVEL=5") + target_compile_options(isal_asm PRIVATE "-DAS_FEATURE_LEVEL=5") endif() # Here must remove "-fno-sanitize=undefined" from COMPILE_OPTIONS. @@ -315,7 +315,13 @@ target_compile_definitions(_qpl PRIVATE -DQPL_BADARG_CHECK PUBLIC -DENABLE_QPL_COMPRESSION) +find_library(LIBACCEL accel-config) +if(NOT LIBACCEL) + message(FATAL_ERROR "Please install QPL dependency library:libaccel-config from https://github.com/intel/idxd-config") +endif() + target_link_libraries(_qpl + PRIVATE ${LIBACCEL} PRIVATE ${CMAKE_DL_LIBS}) add_library (ch_contrib::qpl ALIAS _qpl) diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index b717cec2d33..305fc279414 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -33,7 +33,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="22.11.1.1360" +ARG VERSION="22.11.2.30" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 8a5dc04681e..f1c4dd097aa 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -21,7 +21,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="22.11.1.1360" +ARG VERSION="22.11.2.30" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docs/changelogs/v22.8.11.15-lts.md b/docs/changelogs/v22.8.11.15-lts.md new file mode 100644 index 00000000000..b0c4a7cc168 --- /dev/null +++ b/docs/changelogs/v22.8.11.15-lts.md @@ -0,0 +1,23 @@ +--- +sidebar_position: 1 +sidebar_label: 2022 +--- + +# 2022 Changelog + +### ClickHouse release v22.8.11.15-lts (65c9506d161) FIXME as compared to v22.8.10.29-lts (d568a57f7af) + +#### Bug Fix +* Backported in [#43098](https://github.com/ClickHouse/ClickHouse/issues/43098): Updated normaliser to clone the alias ast. resolves [#42452](https://github.com/ClickHouse/ClickHouse/issues/42452) Implementation: * Updated QueryNormalizer to clone alias ast, when its replaced. Previously just assigning the same leads to exception in LogicalExpressinsOptimizer as it would be the same parent being inserted again. * This bug is not seen with new analyser (allow_experimental_analyzer), so no changes for it. I added a test for the same. [#42827](https://github.com/ClickHouse/ClickHouse/pull/42827) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). + +#### Bug Fix (user-visible misbehavior in official stable or prestable release) + +* Backported in [#43751](https://github.com/ClickHouse/ClickHouse/issues/43751): An issue with the following exception has been reported while trying to read a Parquet file from S3 into ClickHouse:. [#43297](https://github.com/ClickHouse/ClickHouse/pull/43297) ([Arthur Passos](https://github.com/arthurpassos)). +* Backported in [#43617](https://github.com/ClickHouse/ClickHouse/issues/43617): Fix sumMap() for Nullable(Decimal()). [#43414](https://github.com/ClickHouse/ClickHouse/pull/43414) ([Azat Khuzhin](https://github.com/azat)). +* Backported in [#43886](https://github.com/ClickHouse/ClickHouse/issues/43886): Fixed `ALTER ... RESET SETTING` with `ON CLUSTER`. It could be applied to one replica only. Fixes [#43843](https://github.com/ClickHouse/ClickHouse/issues/43843). [#43848](https://github.com/ClickHouse/ClickHouse/pull/43848) ([Elena Torró](https://github.com/elenatorro)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Use only PRs to our repository in pr_info on push [#43895](https://github.com/ClickHouse/ClickHouse/pull/43895) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix tags workflow [#43942](https://github.com/ClickHouse/ClickHouse/pull/43942) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index 1ed93f7a1cb..391d3a3f59a 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -9,7 +9,7 @@ slug: /en/install You have two options for getting up and running with ClickHouse: - **[ClickHouse Cloud](https://clickhouse.com/cloud/):** the official ClickHouse as a service, - built by, maintained, and supported by the creators of ClickHouse -- **Self-managed ClickHouse:** ClickHouse can run on any Linux, FreeBSD, or Mac OS X with x86_64, AArch64, or PowerPC64LE CPU architecture +- **[Self-managed ClickHouse](https://github.com/ClickHouse/ClickHouse):** ClickHouse can run on any Linux, FreeBSD, or Mac OS X with x86_64, AArch64, or PowerPC64LE CPU architecture ## ClickHouse Cloud diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index 731348abfe7..58998a6f491 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -1202,6 +1202,7 @@ SELECT * FROM json_each_row_nested - [input_format_import_nested_json](../operations/settings/settings.md#input_format_import_nested_json) - map nested JSON data to nested tables (it works for JSONEachRow format). Default value - `false`. - [input_format_json_read_bools_as_numbers](../operations/settings/settings.md#input_format_json_read_bools_as_numbers) - allow to parse bools as numbers in JSON input formats. Default value - `true`. - [input_format_json_read_numbers_as_strings](../operations/settings/settings.md#input_format_json_read_numbers_as_strings) - allow to parse numbers as strings in JSON input formats. Default value - `false`. +- [input_format_json_read_objects_as_strings](../operations/settings/settings.md#input_format_json_read_objects_as_strings) - allow to parse JSON objects as strings in JSON input formats. Default value - `false`. - [output_format_json_quote_64bit_integers](../operations/settings/settings.md#output_format_json_quote_64bit_integers) - controls quoting of 64-bit integers in JSON output format. Default value - `true`. - [output_format_json_quote_64bit_floats](../operations/settings/settings.md#output_format_json_quote_64bit_floats) - controls quoting of 64-bit floats in JSON output format. Default value - `false`. - [output_format_json_quote_denormals](../operations/settings/settings.md#output_format_json_quote_denormals) - enables '+nan', '-nan', '+inf', '-inf' outputs in JSON output format. Default value - `false`. diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md index 2c8044d38f7..9af6df0c87d 100644 --- a/docs/en/interfaces/http.md +++ b/docs/en/interfaces/http.md @@ -244,7 +244,7 @@ The username and password can be indicated in one of three ways: $ echo 'SELECT 1' | curl 'http://user:password@localhost:8123/' -d @- ``` -1. In the ‘user’ and ‘password’ URL parameters. Example: +2. In the ‘user’ and ‘password’ URL parameters (*We do not recommend using this method as the parameter might be logged by web proxy and cached in the browser*). Example: @@ -252,7 +252,7 @@ $ echo 'SELECT 1' | curl 'http://user:password@localhost:8123/' -d @- $ echo 'SELECT 1' | curl 'http://localhost:8123/?user=user&password=password' -d @- ``` -1. Using ‘X-ClickHouse-User’ and ‘X-ClickHouse-Key’ headers. Example: +3. Using ‘X-ClickHouse-User’ and ‘X-ClickHouse-Key’ headers. Example: diff --git a/docs/en/operations/settings/constraints-on-settings.md b/docs/en/operations/settings/constraints-on-settings.md index 651b6465f7e..bb015f80834 100644 --- a/docs/en/operations/settings/constraints-on-settings.md +++ b/docs/en/operations/settings/constraints-on-settings.md @@ -91,4 +91,21 @@ Code: 452, e.displayText() = DB::Exception: Setting force_index_by_date should n **Note:** the `default` profile has special handling: all the constraints defined for the `default` profile become the default constraints, so they restrict all the users until they’re overridden explicitly for these users. +## Constraints on Merge Tree Settings +It is possible to set constraints for [merge tree settings](merge-tree-settings.md). There constraints are applied when table with merge tree engine is created or its storage settings are altered. Name of merge tree setting must be prepended by `merge_tree_` prefix when referenced in `` section. + +**Example:** Forbid to create new tables with explicitly specified `storage_policy` + +``` xml + + + + + + + + + +``` + [Original article](https://clickhouse.com/docs/en/operations/settings/constraints_on_settings/) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 2fc6e64b7eb..ddfaab02159 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -3753,6 +3753,29 @@ Allow parsing numbers as strings in JSON input formats. Disabled by default. +### input_format_json_read_objects_as_strings {#input_format_json_read_objects_as_strings} + +Allow parsing JSON objects as strings in JSON input formats. + +Example: + +```sql +SET input_format_json_read_objects_as_strings = 1; +CREATE TABLE test (id UInt64, obj String, date Date) ENGINE=Memory(); +INSERT INTO test FORMAT JSONEachRow {"id" : 1, "obj" : {"a" : 1, "b" : "Hello"}, "date" : "2020-01-01"}; +SELECT * FROM test; +``` + +Result: + +``` +┌─id─┬─obj──────────────────────┬───────date─┐ +│ 1 │ {"a" : 1, "b" : "Hello"} │ 2020-01-01 │ +└────┴──────────────────────────┴────────────┘ +``` + +Disabled by default. + ### input_format_json_validate_types_from_metadata {#input_format_json_validate_types_from_metadata} For JSON/JSONCompact/JSONColumnsWithMetadata input formats, if this setting is set to 1, diff --git a/docs/en/operations/update.md b/docs/en/operations/update.md index 6a880bb78b6..58d4b690cb9 100644 --- a/docs/en/operations/update.md +++ b/docs/en/operations/update.md @@ -1,10 +1,82 @@ --- slug: /en/operations/update +sidebar_title: Self-managed Upgrade +title: Self-managed Upgrade --- -# Update +## ClickHouse upgrade overview -## Self-managed ClickHouse Upgrade +This document contains: +- general guidelines +- a recommended plan +- specifics for upgrading the binaries on your systems + +## General guidelines + +These notes should help you with planning, and to understand why we make the recommendations that we do later in the document. + +### Upgrade ClickHouse server separately from ClickHouse Keeper or ZooKeeper +Unless there is a security fix needed for ClickHouse Keeper or Apache ZooKeeper it is not necessary to upgrade Keeper when you upgrade ClickHouse server. Keeper stability is required during the upgrade process, so complete the ClickHouse server upgrades before considering an upgrade of Keeper. + +### Minor version upgrades should be adopted often +It is highly recommended to always upgrade to the newest minor version as soon as it is released. Minor releases do not have breaking changes but do have important bug fixes (and may have security fixes). + + +### Test experimental features on a separate ClickHouse server running the target version + +The compatibility of experimental features can be broken at any moment in any way. If you are using experimental features, then check the changelogs and consider setting up a separate ClickHouse server with the target version installed and test your use of the experimental features there. + +### Downgrades +If you upgrade and then realize that the new version is not compatible with some feature that you depend on you may be able to downgrade to a recent (less than one year old) version if you have not started to use any of the new features. Once the new features are used the downgrade will not work. + +### Multiple ClickHouse server versions in a cluster + +We make an effort to maintain a one-year compatibility window (which includes 2 LTS versions). This means that any two versions should be able to work together in a cluster if the difference between them is less than one year (or if there are less than two LTS versions between them). However, it is recommended to upgrade all members of a cluster to the same version as quickly as possible, as some minor issues are possible (like slowdown of distributed queries, retriable errors in some background operations in ReplicatedMergeTree, etc). + +We never recommend running different versions in the same cluster when the release dates are more than one year. While we do not expect that you will have data loss, the cluster may become unusable. The issues that you should expect if you have more than one year difference in versions include: + +- the cluster may not work +- some (or even all) queries may fail with arbitrary errors +- arbitrary errors/warnings may appear in the logs +- it may be impossible to downgrade + +### Incremental upgrades + +If the difference between the current version and the target version is more than one year, then it is recommended to either: +- Upgrade with downtime (stop all servers, upgrade all servers, run all servers). +- Or to upgrade through an intermediate version (a version less than one year more recent than the current version). + + + +## Recommended plan + +These are the recommended steps for a zero-downtime ClickHouse upgrade: + +1. Make sure that your configuration changes are not in the default `/etc/clickhouse-server/config.xml` file and that they are instead in `/etc/clickhouse-server/config.d/`, as `/etc/clickhouse-server/config.xml` could be overwritten during an upgrade. +2. Read through the [changelogs](/docs/en/whats-new/changelog/index.md) for breaking changes (going back from the target release to the release you are currently on). +3. Make any updates identified in the breaking changes that can be made before upgrading, and a list of the changes that will need to be made after the upgrade. +4. Identify one or more replicas for each shard to keep up while the rest of the replicas for each shard are upgraded. +5. On the replicas that will be upgraded, one at a time: + - shutdown ClickHouse server + - upgrade the server to the target version + - bring ClickHouse server up + - wait for the Keeper messages to indicate that the system is stable + - continue to the next replica +6. Check for errors in the Keeper log and the ClickHouse log +7. Upgrade the replicas identified in step 4 to the new version +8. Refer to the list of changes made in steps 1 through 3 and make the changes that need to be made after the upgrade. + +:::note +This error message is expected when there are multiple versions of ClickHouse running in a replicated environment. You will stop seeing these when all replicas are upgraded to the same version. +``` +MergeFromLogEntryTask: Code: 40. DB::Exception: Checksums of parts don't match: +hash of uncompressed files doesn't match. (CHECKSUM_DOESNT_MATCH) Data after merge is not +byte-identical to data on another replicas. +``` +::: + + +## ClickHouse server binary upgrade process If ClickHouse was installed from `deb` packages, execute the following commands on the server: diff --git a/docs/en/sql-reference/aggregate-functions/reference/exponentialmovingaverage.md b/docs/en/sql-reference/aggregate-functions/reference/exponentialmovingaverage.md index 7b1709e6d5c..2587bc5533f 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/exponentialmovingaverage.md +++ b/docs/en/sql-reference/aggregate-functions/reference/exponentialmovingaverage.md @@ -1,6 +1,7 @@ --- slug: /en/sql-reference/aggregate-functions/reference/exponentialmovingaverage sidebar_position: 108 +sidebar_title: exponentialMovingAverage --- ## exponentialMovingAverage diff --git a/docs/en/sql-reference/statements/alter/update.md b/docs/en/sql-reference/statements/alter/update.md index 5d27c382982..234812f6ed4 100644 --- a/docs/en/sql-reference/statements/alter/update.md +++ b/docs/en/sql-reference/statements/alter/update.md @@ -7,7 +7,7 @@ sidebar_label: UPDATE # ALTER TABLE … UPDATE Statements ``` sql -ALTER TABLE [db.]table [ON CLUSTER cluster] UPDATE column1 = expr1 [, ...] WHERE filter_expr +ALTER TABLE [db.]table [ON CLUSTER cluster] UPDATE column1 = expr1 [, ...] [IN PARTITION partition_id] WHERE filter_expr ``` Manipulates data matching the specified filtering expression. Implemented as a [mutation](/docs/en/sql-reference/statements/alter/index.md#mutations). diff --git a/docs/en/sql-reference/statements/create/settings-profile.md b/docs/en/sql-reference/statements/create/settings-profile.md index 8883b22896b..c4ca89f3284 100644 --- a/docs/en/sql-reference/statements/create/settings-profile.md +++ b/docs/en/sql-reference/statements/create/settings-profile.md @@ -10,7 +10,7 @@ Creates [settings profiles](../../../operations/access-rights.md#settings-profil Syntax: ``` sql -CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] TO name1 [ON CLUSTER cluster_name1] +CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1] [, name2 [ON CLUSTER cluster_name2] ...] [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...] ``` diff --git a/docs/ru/sql-reference/statements/create/settings-profile.md b/docs/ru/sql-reference/statements/create/settings-profile.md index 9aa77e4c241..d37b975e096 100644 --- a/docs/ru/sql-reference/statements/create/settings-profile.md +++ b/docs/ru/sql-reference/statements/create/settings-profile.md @@ -11,7 +11,7 @@ sidebar_label: "Профиль настроек" Синтаксис: ``` sql -CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] TO name1 [ON CLUSTER cluster_name1] +CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1] [, name2 [ON CLUSTER cluster_name2] ...] [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [CONST|READONLY|WRITABLE|CHANGEABLE_IN_READONLY] | INHERIT 'profile_name'] [,...] ``` @@ -26,4 +26,4 @@ CREATE SETTINGS PROFILE [IF NOT EXISTS | OR REPLACE] TO name1 [ON CLUSTER cluste CREATE SETTINGS PROFILE max_memory_usage_profile SETTINGS max_memory_usage = 100000001 MIN 90000000 MAX 110000000 TO robin ``` - \ No newline at end of file + diff --git a/packages/build b/packages/build index 531e068338d..c8fb77e9371 100755 --- a/packages/build +++ b/packages/build @@ -111,6 +111,8 @@ EOF tar -czf "$TARBALL" -C "$OUTPUT_DIR" "$PKG_DIR" fi + sha512sum "$TARBALL" > "$TARBALL".sha512 + rm -r "$PKG_PATH" } diff --git a/programs/benchmark/Benchmark.cpp b/programs/benchmark/Benchmark.cpp index 4d55a67f0f8..017b28fe082 100644 --- a/programs/benchmark/Benchmark.cpp +++ b/programs/benchmark/Benchmark.cpp @@ -58,22 +58,52 @@ namespace ErrorCodes class Benchmark : public Poco::Util::Application { public: - Benchmark(unsigned concurrency_, double delay_, - Strings && hosts_, Ports && ports_, bool round_robin_, - bool cumulative_, bool secure_, const String & default_database_, - const String & user_, const String & password_, const String & quota_key_, const String & stage, - bool randomize_, size_t max_iterations_, double max_time_, - const String & json_path_, size_t confidence_, - const String & query_id_, const String & query_to_execute_, bool continue_on_errors_, - bool reconnect_, bool display_client_side_time_, bool print_stacktrace_, const Settings & settings_) + Benchmark(unsigned concurrency_, + double delay_, + Strings && hosts_, + Ports && ports_, + bool round_robin_, + bool cumulative_, + bool secure_, + const String & default_database_, + const String & user_, + const String & password_, + const String & quota_key_, + const String & stage, + bool randomize_, + size_t max_iterations_, + double max_time_, + const String & json_path_, + size_t confidence_, + const String & query_id_, + const String & query_to_execute_, + size_t max_consecutive_errors_, + bool continue_on_errors_, + bool reconnect_, + bool display_client_side_time_, + bool print_stacktrace_, + const Settings & settings_) : - round_robin(round_robin_), concurrency(concurrency_), delay(delay_), queue(concurrency), randomize(randomize_), - cumulative(cumulative_), max_iterations(max_iterations_), max_time(max_time_), - json_path(json_path_), confidence(confidence_), query_id(query_id_), - query_to_execute(query_to_execute_), continue_on_errors(continue_on_errors_), reconnect(reconnect_), + round_robin(round_robin_), + concurrency(concurrency_), + delay(delay_), + queue(concurrency), + randomize(randomize_), + cumulative(cumulative_), + max_iterations(max_iterations_), + max_time(max_time_), + json_path(json_path_), + confidence(confidence_), + query_id(query_id_), + query_to_execute(query_to_execute_), + continue_on_errors(continue_on_errors_), + max_consecutive_errors(max_consecutive_errors_), + reconnect(reconnect_), display_client_side_time(display_client_side_time_), - print_stacktrace(print_stacktrace_), settings(settings_), - shared_context(Context::createShared()), global_context(Context::createGlobal(shared_context.get())), + print_stacktrace(print_stacktrace_), + settings(settings_), + shared_context(Context::createShared()), + global_context(Context::createGlobal(shared_context.get())), pool(concurrency) { const auto secure = secure_ ? Protocol::Secure::Enable : Protocol::Secure::Disable; @@ -166,6 +196,7 @@ private: String query_id; String query_to_execute; bool continue_on_errors; + size_t max_consecutive_errors; bool reconnect; bool display_client_side_time; bool print_stacktrace; @@ -174,6 +205,8 @@ private: ContextMutablePtr global_context; QueryProcessingStage::Enum query_processing_stage; + std::atomic consecutive_errors{0}; + /// Don't execute new queries after timelimit or SIGINT or exception std::atomic shutdown{false}; @@ -393,13 +426,14 @@ private: try { execute(connection_entries, query, connection_index); + consecutive_errors = 0; } catch (...) { std::lock_guard lock(mutex); std::cerr << "An error occurred while processing the query " << "'" << query << "'" << ": " << getCurrentExceptionMessage(false) << std::endl; - if (!continue_on_errors) + if (!(continue_on_errors || max_consecutive_errors > ++consecutive_errors)) { shutdown = true; throw; @@ -648,6 +682,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) ("stacktrace", "print stack traces of exceptions") ("confidence", value()->default_value(5), "set the level of confidence for T-test [0=80%, 1=90%, 2=95%, 3=98%, 4=99%, 5=99.5%(default)") ("query_id", value()->default_value(""), "") + ("max-consecutive-errors", value()->default_value(0), "set number of allowed consecutive errors") ("continue_on_errors", "continue testing even if a query fails") ("reconnect", "establish new connection for every query") ("client-side-time", "display the time including network communication instead of server-side time; note that for server versions before 22.8 we always display client-side time") @@ -702,6 +737,7 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv) options["confidence"].as(), options["query_id"].as(), options["query"].as(), + options["max-consecutive-errors"].as(), options.count("continue_on_errors"), options.count("reconnect"), options.count("client-side-time"), diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 6e289b57845..115f76174bd 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -348,17 +348,9 @@ void Client::connect() } catch (const Exception & e) { - /// It is typical when users install ClickHouse, type some password and instantly forget it. - /// This problem can't be fixed with reconnection so it is not attempted - if ((connection_parameters.user.empty() || connection_parameters.user == "default") - && e.code() == DB::ErrorCodes::AUTHENTICATION_FAILED) + if (e.code() == DB::ErrorCodes::AUTHENTICATION_FAILED) { - std::cerr << std::endl - << "If you have installed ClickHouse and forgot password you can reset it in the configuration file." << std::endl - << "The password for default user is typically located at /etc/clickhouse-server/users.d/default-password.xml" << std::endl - << "and deleting this file will reset the password." << std::endl - << "See also /etc/clickhouse-server/users.xml on the server where ClickHouse is installed." << std::endl - << std::endl; + /// This problem can't be fixed with reconnection so it is not attempted throw; } else diff --git a/programs/disks/CMakeLists.txt b/programs/disks/CMakeLists.txt index 58e1c958f68..9477854a58b 100644 --- a/programs/disks/CMakeLists.txt +++ b/programs/disks/CMakeLists.txt @@ -1,4 +1,15 @@ -set (CLICKHOUSE_DISKS_SOURCES DisksApp.cpp ICommand.cpp) +set (CLICKHOUSE_DISKS_SOURCES + DisksApp.cpp + ICommand.cpp + CommandCopy.cpp + CommandLink.cpp + CommandList.cpp + CommandListDisks.cpp + CommandMkDir.cpp + CommandMove.cpp + CommandRead.cpp + CommandRemove.cpp + CommandWrite.cpp) set (CLICKHOUSE_DISKS_LINK PRIVATE diff --git a/programs/disks/CommandCopy.cpp b/programs/disks/CommandCopy.cpp index 1e5852fe651..1e4a3ba6908 100644 --- a/programs/disks/CommandCopy.cpp +++ b/programs/disks/CommandCopy.cpp @@ -1,7 +1,6 @@ -#pragma once - #include "ICommand.h" #include +#include namespace DB { @@ -11,7 +10,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -class CommandCopy : public ICommand +class CommandCopy final : public ICommand { public: CommandCopy() @@ -51,16 +50,16 @@ public: String disk_name_from = config.getString("diskFrom", config.getString("disk", "default")); String disk_name_to = config.getString("diskTo", config.getString("disk", "default")); - String path_from = command_arguments[0]; - String path_to = command_arguments[1]; + const String & path_from = command_arguments[0]; + const String & path_to = command_arguments[1]; DiskPtr disk_from = global_context->getDisk(disk_name_from); DiskPtr disk_to = global_context->getDisk(disk_name_to); - String full_path_from = fullPathWithValidate(disk_from, path_from); - String full_path_to = fullPathWithValidate(disk_to, path_to); + String relative_path_from = validatePathAndGetAsRelative(path_from); + String relative_path_to = validatePathAndGetAsRelative(path_to); - disk_from->copy(full_path_from, disk_to, full_path_to); + disk_from->copy(relative_path_from, disk_to, relative_path_to); } }; } diff --git a/programs/disks/CommandLink.cpp b/programs/disks/CommandLink.cpp index af48f0de097..766d03a0b6b 100644 --- a/programs/disks/CommandLink.cpp +++ b/programs/disks/CommandLink.cpp @@ -1,5 +1,3 @@ -#pragma once - #include "ICommand.h" #include @@ -11,7 +9,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -class CommandLink : public ICommand +class CommandLink final : public ICommand { public: CommandLink() @@ -40,15 +38,15 @@ public: String disk_name = config.getString("disk", "default"); - String path_from = command_arguments[0]; - String path_to = command_arguments[1]; + const String & path_from = command_arguments[0]; + const String & path_to = command_arguments[1]; DiskPtr disk = global_context->getDisk(disk_name); - String full_path_from = fullPathWithValidate(disk, path_from); - String full_path_to = fullPathWithValidate(disk, path_to); + String relative_path_from = validatePathAndGetAsRelative(path_from); + String relative_path_to = validatePathAndGetAsRelative(path_to); - disk->createHardLink(full_path_from, full_path_to); + disk->createHardLink(relative_path_from, relative_path_to); } }; } diff --git a/programs/disks/CommandList.cpp b/programs/disks/CommandList.cpp index e76bb9e65fb..a1d41316b9d 100644 --- a/programs/disks/CommandList.cpp +++ b/programs/disks/CommandList.cpp @@ -1,7 +1,6 @@ -#pragma once - #include "ICommand.h" #include +#include namespace DB { @@ -11,7 +10,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -class CommandList : public ICommand +class CommandList final : public ICommand { public: CommandList() @@ -46,43 +45,47 @@ public: String disk_name = config.getString("disk", "default"); - String path = command_arguments[0]; + const String & path = command_arguments[0]; DiskPtr disk = global_context->getDisk(disk_name); - String full_path = fullPathWithValidate(disk, path); + String relative_path = validatePathAndGetAsRelative(path); bool recursive = config.getBool("recursive", false); if (recursive) - listRecursive(disk, full_path); + listRecursive(disk, relative_path); else - list(disk, full_path); + list(disk, relative_path); } private: - static void list(const DiskPtr & disk, const std::string & full_path) + static void list(const DiskPtr & disk, const std::string & relative_path) { std::vector file_names; - disk->listFiles(full_path, file_names); + disk->listFiles(relative_path, file_names); for (const auto & file_name : file_names) std::cout << file_name << '\n'; } - static void listRecursive(const DiskPtr & disk, const std::string & full_path) + static void listRecursive(const DiskPtr & disk, const std::string & relative_path) { std::vector file_names; - disk->listFiles(full_path, file_names); + disk->listFiles(relative_path, file_names); - std::cout << full_path << ":\n"; - for (const auto & file_name : file_names) - std::cout << file_name << '\n'; - std::cout << "\n"; + std::cout << relative_path << ":\n"; + + if (!file_names.empty()) + { + for (const auto & file_name : file_names) + std::cout << file_name << '\n'; + std::cout << "\n"; + } for (const auto & file_name : file_names) { - auto path = full_path + "/" + file_name; + auto path = relative_path + "/" + file_name; if (disk->isDirectory(path)) listRecursive(disk, path); } diff --git a/programs/disks/CommandListDisks.cpp b/programs/disks/CommandListDisks.cpp index 22cffdd21fd..a6b38f60a67 100644 --- a/programs/disks/CommandListDisks.cpp +++ b/programs/disks/CommandListDisks.cpp @@ -1,5 +1,3 @@ -#pragma once - #include "ICommand.h" #include @@ -11,7 +9,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -class CommandListDisks : public ICommand +class CommandListDisks final : public ICommand { public: CommandListDisks() diff --git a/programs/disks/CommandMkDir.cpp b/programs/disks/CommandMkDir.cpp index 11a940028a3..b4b08391663 100644 --- a/programs/disks/CommandMkDir.cpp +++ b/programs/disks/CommandMkDir.cpp @@ -1,7 +1,7 @@ -#pragma once - #include "ICommand.h" + #include +#include namespace DB { @@ -11,7 +11,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -class CommandMkDir : public ICommand +class CommandMkDir final : public ICommand { public: CommandMkDir() @@ -46,17 +46,17 @@ public: String disk_name = config.getString("disk", "default"); - String path = command_arguments[0]; + const String & path = command_arguments[0]; DiskPtr disk = global_context->getDisk(disk_name); - String full_path = fullPathWithValidate(disk, path); + String relative_path = validatePathAndGetAsRelative(path); bool recursive = config.getBool("recursive", false); if (recursive) - disk->createDirectories(full_path); + disk->createDirectories(relative_path); else - disk->createDirectory(full_path); + disk->createDirectory(relative_path); } }; } diff --git a/programs/disks/CommandMove.cpp b/programs/disks/CommandMove.cpp index 6322cf4b47d..085a0e2d5eb 100644 --- a/programs/disks/CommandMove.cpp +++ b/programs/disks/CommandMove.cpp @@ -1,5 +1,3 @@ -#pragma once - #include "ICommand.h" #include @@ -11,7 +9,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -class CommandMove : public ICommand +class CommandMove final : public ICommand { public: CommandMove() @@ -39,18 +37,18 @@ public: String disk_name = config.getString("disk", "default"); - String path_from = command_arguments[0]; - String path_to = command_arguments[1]; + const String & path_from = command_arguments[0]; + const String & path_to = command_arguments[1]; DiskPtr disk = global_context->getDisk(disk_name); - String full_path_from = fullPathWithValidate(disk, path_from); - String full_path_to = fullPathWithValidate(disk, path_to); + String relative_path_from = validatePathAndGetAsRelative(path_from); + String relative_path_to = validatePathAndGetAsRelative(path_to); - if (disk->isFile(full_path_from)) - disk->moveFile(full_path_from, full_path_to); + if (disk->isFile(relative_path_from)) + disk->moveFile(relative_path_from, relative_path_to); else - disk->moveDirectory(full_path_from, full_path_to); + disk->moveDirectory(relative_path_from, relative_path_to); } }; } diff --git a/programs/disks/CommandRead.cpp b/programs/disks/CommandRead.cpp index 6b77a27e918..2a04dd7a902 100644 --- a/programs/disks/CommandRead.cpp +++ b/programs/disks/CommandRead.cpp @@ -1,7 +1,9 @@ -#pragma once - #include "ICommand.h" #include +#include +#include +#include +#include namespace DB { @@ -11,7 +13,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -class CommandRead : public ICommand +class CommandRead final : public ICommand { public: CommandRead() @@ -46,27 +48,25 @@ public: String disk_name = config.getString("disk", "default"); - String path = command_arguments[0]; - DiskPtr disk = global_context->getDisk(disk_name); - String full_path = fullPathWithValidate(disk, path); + String relative_path = validatePathAndGetAsRelative(command_arguments[0]); String path_output = config.getString("output", ""); if (!path_output.empty()) { - String full_path_output = fullPathWithValidate(disk, path_output); + String relative_path_output = validatePathAndGetAsRelative(path_output); - auto in = disk->readFile(full_path); - auto out = disk->writeFile(full_path_output); + auto in = disk->readFile(relative_path); + auto out = disk->writeFile(relative_path_output); copyData(*in, *out); out->finalize(); return; } else { - auto in = disk->readFile(full_path); + auto in = disk->readFile(relative_path); std::unique_ptr out = std::make_unique(STDOUT_FILENO); copyData(*in, *out); } diff --git a/programs/disks/CommandRemove.cpp b/programs/disks/CommandRemove.cpp index c1d3129bb8d..c742cdec042 100644 --- a/programs/disks/CommandRemove.cpp +++ b/programs/disks/CommandRemove.cpp @@ -1,5 +1,3 @@ -#pragma once - #include "ICommand.h" #include @@ -11,7 +9,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -class CommandRemove : public ICommand +class CommandRemove final : public ICommand { public: CommandRemove() @@ -39,13 +37,13 @@ public: String disk_name = config.getString("disk", "default"); - String path = command_arguments[0]; + const String & path = command_arguments[0]; DiskPtr disk = global_context->getDisk(disk_name); - String full_path = fullPathWithValidate(disk, path); + String relative_path = validatePathAndGetAsRelative(path); - disk->removeRecursive(full_path); + disk->removeRecursive(relative_path); } }; } diff --git a/programs/disks/CommandWrite.cpp b/programs/disks/CommandWrite.cpp index 0b1c5823c81..152cb33c458 100644 --- a/programs/disks/CommandWrite.cpp +++ b/programs/disks/CommandWrite.cpp @@ -1,8 +1,11 @@ -#pragma once - #include "ICommand.h" #include +#include +#include +#include +#include + namespace DB { @@ -11,7 +14,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -class CommandWrite : public ICommand +class CommandWrite final : public ICommand { public: CommandWrite() @@ -46,11 +49,11 @@ public: String disk_name = config.getString("disk", "default"); - String path = command_arguments[0]; + const String & path = command_arguments[0]; DiskPtr disk = global_context->getDisk(disk_name); - String full_path = fullPathWithValidate(disk, path); + String relative_path = validatePathAndGetAsRelative(path); String path_input = config.getString("input", ""); std::unique_ptr in; @@ -60,11 +63,11 @@ public: } else { - String full_path_input = fullPathWithValidate(disk, path_input); - in = disk->readFile(full_path_input); + String relative_path_input = validatePathAndGetAsRelative(path_input); + in = disk->readFile(relative_path_input); } - auto out = disk->writeFile(full_path); + auto out = disk->writeFile(relative_path); copyData(*in, *out); out->finalize(); } diff --git a/programs/disks/DisksApp.cpp b/programs/disks/DisksApp.cpp index ea46bb1ba8d..0e0e34f7d10 100644 --- a/programs/disks/DisksApp.cpp +++ b/programs/disks/DisksApp.cpp @@ -1,11 +1,12 @@ #include "DisksApp.h" +#include "ICommand.h" #include -#include - +#include #include + namespace DB { diff --git a/programs/disks/DisksApp.h b/programs/disks/DisksApp.h index 24fa9c3d9eb..0b596921707 100644 --- a/programs/disks/DisksApp.h +++ b/programs/disks/DisksApp.h @@ -1,28 +1,22 @@ #pragma once -#include "CommandCopy.cpp" -#include "CommandLink.cpp" -#include "CommandList.cpp" -#include "CommandListDisks.cpp" -#include "CommandMkDir.cpp" -#include "CommandMove.cpp" -#include "CommandRead.cpp" -#include "CommandRemove.cpp" -#include "CommandWrite.cpp" - #include -#include -#include -#include -#include #include +#include + +#include namespace DB { +class ICommand; using CommandPtr = std::unique_ptr; +namespace po = boost::program_options; +using ProgramOptionsDescription = boost::program_options::options_description; +using CommandLineOptions = boost::program_options::variables_map; + class DisksApp : public Poco::Util::Application, public Loggers { public: diff --git a/programs/disks/ICommand.cpp b/programs/disks/ICommand.cpp index 093ebb93cb2..52d1a2196a9 100644 --- a/programs/disks/ICommand.cpp +++ b/programs/disks/ICommand.cpp @@ -30,19 +30,21 @@ void ICommand::addOptions(ProgramOptionsDescription & options_description) options_description.add(*command_option_description); } -String ICommand::fullPathWithValidate(const DiskPtr & disk, const String & path) +String ICommand::validatePathAndGetAsRelative(const String & path) { - if (fs::path(path).lexically_normal().string() != path) + /// If path contain non-normalized symbols like . we will normalized them. If the resulting normalized path + /// still contain '..' it can be dangerous, disallow such paths. Also since clickhouse-disks + /// is not an interactive program (don't track you current path) it's OK to disallow .. paths. + String lexically_normal_path = fs::path(path).lexically_normal(); + if (lexically_normal_path.find("..") != std::string::npos) throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Path {} is not normalized", path); - String disk_path = fs::canonical(fs::path(disk->getPath())) / ""; - String full_path = (fs::absolute(disk_path) / path).lexically_normal(); + /// If path is absolute we should keep it as relative inside disk, so disk will look like + /// an ordinary filesystem with root. + if (fs::path(lexically_normal_path).is_absolute()) + return lexically_normal_path.substr(1); - if (!full_path.starts_with(disk_path)) - throw DB::Exception( - DB::ErrorCodes::BAD_ARGUMENTS, "Path {} must be inside disk path {}", full_path, disk_path); - - return path; + return lexically_normal_path; } } diff --git a/programs/disks/ICommand.h b/programs/disks/ICommand.h index f57f74a880e..de41eedec35 100644 --- a/programs/disks/ICommand.h +++ b/programs/disks/ICommand.h @@ -2,16 +2,10 @@ #include -#include - -#include -#include -#include - #include -#include #include +#include #include @@ -43,7 +37,7 @@ public: protected: void printHelpMessage() const; - static String fullPathWithValidate(const DiskPtr & disk, const String & path); + static String validatePathAndGetAsRelative(const String & path); public: String command_name; @@ -55,14 +49,16 @@ protected: po::positional_options_description positional_options_description; }; +using CommandPtr = std::unique_ptr; + } -std::unique_ptr makeCommandCopy(); -std::unique_ptr makeCommandLink(); -std::unique_ptr makeCommandList(); -std::unique_ptr makeCommandListDisks(); -std::unique_ptr makeCommandMove(); -std::unique_ptr makeCommandRead(); -std::unique_ptr makeCommandRemove(); -std::unique_ptr makeCommandWrite(); -std::unique_ptr makeCommandMkDir(); +DB::CommandPtr makeCommandCopy(); +DB::CommandPtr makeCommandLink(); +DB::CommandPtr makeCommandList(); +DB::CommandPtr makeCommandListDisks(); +DB::CommandPtr makeCommandMove(); +DB::CommandPtr makeCommandRead(); +DB::CommandPtr makeCommandRemove(); +DB::CommandPtr makeCommandWrite(); +DB::CommandPtr makeCommandMkDir(); diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index 04642df6f40..063d48b1d70 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -16,11 +16,13 @@ #include #include #include +#include #include #include #include +#include #include -#include +#include #include #include #include @@ -38,7 +40,6 @@ namespace ErrorCodes extern const int AUTHENTICATION_FAILED; } - namespace { void checkForUsersNotInMainConfig( @@ -103,7 +104,7 @@ public: bool isSettingNameAllowed(std::string_view setting_name) const { - if (Settings::hasBuiltin(setting_name)) + if (settingIsBuiltin(setting_name)) return true; std::lock_guard lock{mutex}; @@ -454,9 +455,21 @@ UUID AccessControl::authenticate(const Credentials & credentials, const Poco::Ne { tryLogCurrentException(getLogger(), "from: " + address.toString() + ", user: " + credentials.getUserName() + ": Authentication failed"); + WriteBufferFromOwnString message; + message << credentials.getUserName() << ": Authentication failed: password is incorrect, or there is no user with such name."; + + /// Better exception message for usability. + /// It is typical when users install ClickHouse, type some password and instantly forget it. + if (credentials.getUserName().empty() || credentials.getUserName() == "default") + message << "\n\n" + << "If you have installed ClickHouse and forgot password you can reset it in the configuration file.\n" + << "The password for default user is typically located at /etc/clickhouse-server/users.d/default-password.xml\n" + << "and deleting this file will reset the password.\n" + << "See also /etc/clickhouse-server/users.xml on the server where ClickHouse is installed.\n\n"; + /// We use the same message for all authentication failures because we don't want to give away any unnecessary information for security reasons, /// only the log will show the exact reason. - throw Exception(credentials.getUserName() + ": Authentication failed: password is incorrect or there is no user with such name", ErrorCodes::AUTHENTICATION_FAILED); + throw Exception(message.str(), ErrorCodes::AUTHENTICATION_FAILED); } } diff --git a/src/Access/ContextAccess.h b/src/Access/ContextAccess.h index 84ef0ab722d..63604a03b4e 100644 --- a/src/Access/ContextAccess.h +++ b/src/Access/ContextAccess.h @@ -174,7 +174,6 @@ private: void initialize(); void setUser(const UserPtr & user_) const; void setRolesInfo(const std::shared_ptr & roles_info_) const; - void setSettingsAndConstraints() const; void calculateAccessRights() const; template diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index 0317e43f8d1..9983adcb417 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -1,13 +1,15 @@ +#include #include +#include #include #include +#include #include #include #include #include #include - namespace DB { namespace ErrorCodes @@ -18,7 +20,6 @@ namespace ErrorCodes extern const int UNKNOWN_SETTING; } - SettingsConstraints::SettingsConstraints(const AccessControl & access_control_) : access_control(&access_control_) { } @@ -35,19 +36,28 @@ void SettingsConstraints::clear() constraints.clear(); } -void SettingsConstraints::set(const String & setting_name, const Field & min_value, const Field & max_value, SettingConstraintWritability writability) +void SettingsConstraints::set(const String & full_name, const Field & min_value, const Field & max_value, SettingConstraintWritability writability) { - auto & constraint = constraints[setting_name]; + auto & constraint = constraints[full_name]; if (!min_value.isNull()) - constraint.min_value = Settings::castValueUtil(setting_name, min_value); + constraint.min_value = settingCastValueUtil(full_name, min_value); if (!max_value.isNull()) - constraint.max_value = Settings::castValueUtil(setting_name, max_value); + constraint.max_value = settingCastValueUtil(full_name, max_value); constraint.writability = writability; } -void SettingsConstraints::get(const Settings & current_settings, std::string_view setting_name, Field & min_value, Field & max_value, SettingConstraintWritability & writability) const +void SettingsConstraints::get(const Settings & current_settings, std::string_view short_name, Field & min_value, Field & max_value, SettingConstraintWritability & writability) const { - auto checker = getChecker(current_settings, setting_name); + // NOTE: for `Settings` short name is equal to full name + auto checker = getChecker(current_settings, short_name); + min_value = checker.constraint.min_value; + max_value = checker.constraint.max_value; + writability = checker.constraint.writability; +} + +void SettingsConstraints::get(const MergeTreeSettings &, std::string_view short_name, Field & min_value, Field & max_value, SettingConstraintWritability & writability) const +{ + auto checker = getMergeTreeChecker(short_name); min_value = checker.constraint.min_value; max_value = checker.constraint.max_value; writability = checker.constraint.writability; @@ -97,6 +107,17 @@ void SettingsConstraints::check(const Settings & current_settings, SettingsChang }); } +void SettingsConstraints::check(const MergeTreeSettings & current_settings, const SettingChange & change) const +{ + checkImpl(current_settings, const_cast(change), THROW_ON_VIOLATION); +} + +void SettingsConstraints::check(const MergeTreeSettings & current_settings, const SettingsChanges & changes) const +{ + for (const auto & change : changes) + check(current_settings, change); +} + void SettingsConstraints::clamp(const Settings & current_settings, SettingsChanges & changes) const { boost::range::remove_erase_if( @@ -107,6 +128,36 @@ void SettingsConstraints::clamp(const Settings & current_settings, SettingsChang }); } +template +bool getNewValueToCheck(const T & current_settings, SettingChange & change, Field & new_value, bool throw_on_failure) +{ + Field current_value; + bool has_current_value = current_settings.tryGet(change.name, current_value); + + /// Setting isn't checked if value has not changed. + if (has_current_value && change.value == current_value) + return false; + + if (throw_on_failure) + new_value = T::castValueUtil(change.name, change.value); + else + { + try + { + new_value = T::castValueUtil(change.name, change.value); + } + catch (...) + { + return false; + } + } + + /// Setting isn't checked if value has not changed. + if (has_current_value && new_value == current_value) + return false; + + return true; +} bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingChange & change, ReactionOnViolation reaction) const { @@ -115,26 +166,6 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh if (setting_name == "profile") return true; - bool cannot_cast; - auto cast_value = [&](const Field & x) -> Field - { - cannot_cast = false; - if (reaction == THROW_ON_VIOLATION) - return Settings::castValueUtil(setting_name, x); - else - { - try - { - return Settings::castValueUtil(setting_name, x); - } - catch (...) - { - cannot_cast = true; - return {}; - } - } - }; - if (reaction == THROW_ON_VIOLATION) { try @@ -156,27 +187,21 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, SettingCh else if (!access_control->isSettingNameAllowed(setting_name)) return false; - Field current_value, new_value; - if (current_settings.tryGet(setting_name, current_value)) - { - /// Setting isn't checked if value has not changed. - if (change.value == current_value) - return false; - - new_value = cast_value(change.value); - if ((new_value == current_value) || cannot_cast) - return false; - } - else - { - new_value = cast_value(change.value); - if (cannot_cast) - return false; - } + Field new_value; + if (!getNewValueToCheck(current_settings, change, new_value, reaction == THROW_ON_VIOLATION)) + return false; return getChecker(current_settings, setting_name).check(change, new_value, reaction); } +bool SettingsConstraints::checkImpl(const MergeTreeSettings & current_settings, SettingChange & change, ReactionOnViolation reaction) const +{ + Field new_value; + if (!getNewValueToCheck(current_settings, change, new_value, reaction == THROW_ON_VIOLATION)) + return false; + return getMergeTreeChecker(change.name).check(change, new_value, reaction); +} + bool SettingsConstraints::Checker::check(SettingChange & change, const Field & new_value, ReactionOnViolation reaction) const { const String & setting_name = change.name; @@ -185,16 +210,13 @@ bool SettingsConstraints::Checker::check(SettingChange & change, const Field & n { if (reaction == THROW_ON_VIOLATION) return applyVisitor(FieldVisitorAccurateLess{}, left, right); - else + try { - try - { - return applyVisitor(FieldVisitorAccurateLess{}, left, right); - } - catch (...) - { - return true; - } + return applyVisitor(FieldVisitorAccurateLess{}, left, right); + } + catch (...) + { + return true; } }; @@ -280,6 +302,14 @@ SettingsConstraints::Checker SettingsConstraints::getChecker(const Settings & cu return Checker(it->second); } +SettingsConstraints::Checker SettingsConstraints::getMergeTreeChecker(std::string_view short_name) const +{ + auto it = constraints.find(settingFullName(short_name)); + if (it == constraints.end()) + return Checker(); // Allowed + return Checker(it->second); +} + bool SettingsConstraints::Constraint::operator==(const Constraint & other) const { return writability == other.writability && min_value == other.min_value && max_value == other.max_value; diff --git a/src/Access/SettingsConstraints.h b/src/Access/SettingsConstraints.h index 822bf42861b..ed671774281 100644 --- a/src/Access/SettingsConstraints.h +++ b/src/Access/SettingsConstraints.h @@ -12,6 +12,7 @@ namespace Poco::Util namespace DB { struct Settings; +struct MergeTreeSettings; struct SettingChange; class SettingsChanges; class AccessControl; @@ -65,8 +66,9 @@ public: void clear(); bool empty() const { return constraints.empty(); } - void set(const String & setting_name, const Field & min_value, const Field & max_value, SettingConstraintWritability writability); - void get(const Settings & current_settings, std::string_view setting_name, Field & min_value, Field & max_value, SettingConstraintWritability & writability) const; + void set(const String & full_name, const Field & min_value, const Field & max_value, SettingConstraintWritability writability); + void get(const Settings & current_settings, std::string_view short_name, Field & min_value, Field & max_value, SettingConstraintWritability & writability) const; + void get(const MergeTreeSettings & current_settings, std::string_view short_name, Field & min_value, Field & max_value, SettingConstraintWritability & writability) const; void merge(const SettingsConstraints & other); @@ -75,6 +77,10 @@ public: void check(const Settings & current_settings, const SettingsChanges & changes) const; void check(const Settings & current_settings, SettingsChanges & changes) const; + /// Checks whether `change` violates these constraints and throws an exception if so. (setting short name is expected inside `changes`) + void check(const MergeTreeSettings & current_settings, const SettingChange & change) const; + void check(const MergeTreeSettings & current_settings, const SettingsChanges & changes) const; + /// Checks whether `change` violates these and clamps the `change` if so. void clamp(const Settings & current_settings, SettingsChanges & changes) const; @@ -137,8 +143,10 @@ private: }; bool checkImpl(const Settings & current_settings, SettingChange & change, ReactionOnViolation reaction) const; + bool checkImpl(const MergeTreeSettings & current_settings, SettingChange & change, ReactionOnViolation reaction) const; Checker getChecker(const Settings & current_settings, std::string_view setting_name) const; + Checker getMergeTreeChecker(std::string_view short_name) const; // Special container for heterogeneous lookups: to avoid `String` construction during `find(std::string_view)` using Constraints = std::unordered_map>; diff --git a/src/Access/UsersConfigAccessStorage.cpp b/src/Access/UsersConfigAccessStorage.cpp index e3690f36cc3..1feb687d1d3 100644 --- a/src/Access/UsersConfigAccessStorage.cpp +++ b/src/Access/UsersConfigAccessStorage.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -451,9 +452,9 @@ namespace for (const String & constraint_type : constraint_types) { if (constraint_type == "min") - profile_element.min_value = Settings::stringToValueUtil(setting_name, config.getString(path_to_name + "." + constraint_type)); + profile_element.min_value = settingStringToValueUtil(setting_name, config.getString(path_to_name + "." + constraint_type)); else if (constraint_type == "max") - profile_element.max_value = Settings::stringToValueUtil(setting_name, config.getString(path_to_name + "." + constraint_type)); + profile_element.max_value = settingStringToValueUtil(setting_name, config.getString(path_to_name + "." + constraint_type)); else if (constraint_type == "readonly" || constraint_type == "const") { writability_count++; @@ -517,7 +518,7 @@ namespace SettingsProfileElement profile_element; profile_element.setting_name = setting_name; - profile_element.value = Settings::stringToValueUtil(setting_name, config.getString(profile_config + "." + key)); + profile_element.value = settingStringToValueUtil(setting_name, config.getString(profile_config + "." + key)); profile->elements.emplace_back(std::move(profile_element)); } diff --git a/src/Access/resolveSetting.h b/src/Access/resolveSetting.h new file mode 100644 index 00000000000..8469c60bbe3 --- /dev/null +++ b/src/Access/resolveSetting.h @@ -0,0 +1,90 @@ +#pragma once + +#include +#include + +// +// Settings from different classes (Settings, MergeTreeSettings) can coexist in the same "namespace". +// This is, for example, required to define settings constraints inside user profiles. +// `resolveSetting(full_name)` is used to resolve setting name and choose which class is to be used. +// Templated lambda syntax should be used: +// +// return resolveSetting(name, [] (std::string_view name, SettingsType) +// { +// return T::castValueUtil(name, value); // T will be deduced into `Settings`, `MergeTreeSettings`, ... +// }); +// + +namespace DB +{ + +constexpr std::string_view MERGE_TREE_SETTINGS_PREFIX = "merge_tree_"; + +template struct SettingsType {}; + +// Resolve setting name and call function `f` back with short name and class +template +auto resolveSetting(std::string_view full_name, F && f) +{ + if (full_name.starts_with(MERGE_TREE_SETTINGS_PREFIX)) + { + std::string_view short_name = static_cast(full_name).substr(MERGE_TREE_SETTINGS_PREFIX.size()); + if (MergeTreeSettings::hasBuiltin(short_name)) // Check is required because `Settings` also contain names starting with 'merge_tree_' prefix + return f(short_name, SettingsType()); + } + // NOTE: other setting name resolution rules are to be added here + + // If no rule works - use global namespace + return f(full_name, SettingsType()); +} + +inline Field settingCastValueUtil(std::string_view full_name, const Field & value) +{ + return resolveSetting(full_name, [&] (std::string_view short_name, SettingsType) + { + return T::castValueUtil(short_name, value); + }); +} + +inline String settingValueToStringUtil(std::string_view full_name, const Field & value) +{ + return resolveSetting(full_name, [&] (std::string_view short_name, SettingsType) + { + return T::valueToStringUtil(short_name, value); + }); +} + +inline Field settingStringToValueUtil(std::string_view full_name, const String & str) +{ + return resolveSetting(full_name, [&] (std::string_view short_name, SettingsType) + { + return T::stringToValueUtil(short_name, str); + }); +} + +inline bool settingIsBuiltin(std::string_view full_name) +{ + return resolveSetting(full_name, [&] (std::string_view short_name, SettingsType) + { + return T::hasBuiltin(short_name); + }); +} + +template +inline String settingFullName(std::string_view short_name); + +template <> +inline String settingFullName(std::string_view short_name) +{ + return String(short_name); +} + +template <> +inline String settingFullName(std::string_view short_name) +{ + String full_name(MERGE_TREE_SETTINGS_PREFIX); + full_name += short_name; // Just because you cannot concatenate `std::string_view` and `std::string` using operator+ in C++20 yet + return full_name; +} + +} diff --git a/src/AggregateFunctions/AggregateFunctionNull.h b/src/AggregateFunctions/AggregateFunctionNull.h index 64f48ac2987..26d36b84860 100644 --- a/src/AggregateFunctions/AggregateFunctionNull.h +++ b/src/AggregateFunctions/AggregateFunctionNull.h @@ -477,7 +477,7 @@ public: final_flags = std::make_unique(row_end); final_flags_ptr = final_flags.get(); - bool included_elements = 0; + size_t included_elements = 0; const auto & flags = assert_cast(*columns[if_argument_pos]).getData(); for (size_t i = row_begin; i < row_end; i++) { diff --git a/src/Analyzer/Passes/CountDistinctPass.cpp b/src/Analyzer/Passes/CountDistinctPass.cpp index 2b55efa3552..05c31ec28ba 100644 --- a/src/Analyzer/Passes/CountDistinctPass.cpp +++ b/src/Analyzer/Passes/CountDistinctPass.cpp @@ -3,6 +3,8 @@ #include #include +#include + #include #include #include @@ -56,7 +58,7 @@ public: auto & count_distinct_argument_column_typed = count_distinct_argument_column->as(); /// Build subquery SELECT count_distinct_argument_column FROM table_expression GROUP BY count_distinct_argument_column - auto subquery = std::make_shared(); + auto subquery = std::make_shared(Context::createCopy(query_node->getContext())); subquery->getJoinTree() = query_node->getJoinTree(); subquery->getProjection().getNodes().push_back(count_distinct_argument_column); subquery->getGroupBy().getNodes().push_back(count_distinct_argument_column); diff --git a/src/Analyzer/Passes/NormalizeCountVariantsPass.cpp b/src/Analyzer/Passes/NormalizeCountVariantsPass.cpp index 28e5af3f5db..cd6aa4d76f4 100644 --- a/src/Analyzer/Passes/NormalizeCountVariantsPass.cpp +++ b/src/Analyzer/Passes/NormalizeCountVariantsPass.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB { @@ -16,7 +17,8 @@ namespace class NormalizeCountVariantsVisitor : public InDepthQueryTreeVisitor { public: - static void visitImpl(QueryTreeNodePtr & node) + explicit NormalizeCountVariantsVisitor(ContextPtr context_) : context(std::move(context_)) {} + void visitImpl(QueryTreeNodePtr & node) { auto * function_node = node->as(); if (!function_node || !function_node->isAggregateFunction() || (function_node->getFunctionName() != "count" && function_node->getFunctionName() != "sum")) @@ -39,13 +41,16 @@ public: } else if (function_node->getFunctionName() == "sum" && first_argument_constant_literal.getType() == Field::Types::UInt64 && - first_argument_constant_literal.get() == 1) + first_argument_constant_literal.get() == 1 && + !context->getSettingsRef().aggregate_functions_null_for_empty) { resolveAsCountAggregateFunction(*function_node); function_node->getArguments().getNodes().clear(); } } private: + ContextPtr context; + static inline void resolveAsCountAggregateFunction(FunctionNode & function_node) { auto function_result_type = function_node.getResultType(); @@ -59,9 +64,9 @@ private: } -void NormalizeCountVariantsPass::run(QueryTreeNodePtr query_tree_node, ContextPtr) +void NormalizeCountVariantsPass::run(QueryTreeNodePtr query_tree_node, ContextPtr context) { - NormalizeCountVariantsVisitor visitor; + NormalizeCountVariantsVisitor visitor(context); visitor.visit(query_tree_node); } diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index a1fc09bac39..904d104950c 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -647,6 +647,11 @@ struct IdentifierResolveScope subquery_depth = parent_scope->subquery_depth; context = parent_scope->context; } + + if (auto * union_node = scope_node->as()) + context = union_node->getContext(); + else if (auto * query_node = scope_node->as()) + context = query_node->getContext(); } QueryTreeNodePtr scope_node; @@ -974,7 +979,9 @@ public: void resolve(QueryTreeNodePtr node, const QueryTreeNodePtr & table_expression, ContextPtr context) { IdentifierResolveScope scope(node, nullptr /*parent_scope*/); - scope.context = context; + + if (!scope.context) + scope.context = context; auto node_type = node->getNodeType(); @@ -4050,7 +4057,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi auto constant_data_type = std::make_shared(); - auto in_subquery = std::make_shared(); + auto in_subquery = std::make_shared(Context::createCopy(scope.context)); in_subquery->getProjection().getNodes().push_back(std::make_shared(1UL, constant_data_type)); in_subquery->getJoinTree() = exists_subquery_argument; in_subquery->getLimit() = std::make_shared(1UL, constant_data_type); @@ -4103,7 +4110,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi projection_columns.emplace_back(column.name, column.type); } - auto in_second_argument_query_node = std::make_shared(); + auto in_second_argument_query_node = std::make_shared(Context::createCopy(scope.context)); in_second_argument_query_node->setIsSubquery(true); in_second_argument_query_node->getProjectionNode() = std::move(column_nodes_to_select); in_second_argument_query_node->getJoinTree() = std::move(in_second_argument); @@ -5764,14 +5771,6 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier max_subquery_depth); auto & query_node_typed = query_node->as(); - - if (query_node_typed.hasSettingsChanges()) - { - auto updated_scope_context = Context::createCopy(scope.context); - updated_scope_context->applySettingsChanges(query_node_typed.getSettingsChanges()); - scope.context = std::move(updated_scope_context); - } - const auto & settings = scope.context->getSettingsRef(); if (settings.group_by_use_nulls) diff --git a/src/Analyzer/Passes/SumIfToCountIfPass.cpp b/src/Analyzer/Passes/SumIfToCountIfPass.cpp index e40ba25a965..91c277d35b3 100644 --- a/src/Analyzer/Passes/SumIfToCountIfPass.cpp +++ b/src/Analyzer/Passes/SumIfToCountIfPass.cpp @@ -56,7 +56,7 @@ public: if (!isInt64OrUInt64FieldType(constant_value_literal.getType())) return; - if (constant_value_literal.get() != 1) + if (constant_value_literal.get() != 1 || context->getSettingsRef().aggregate_functions_null_for_empty) return; function_node_arguments_nodes[0] = std::move(function_node_arguments_nodes[1]); diff --git a/src/Analyzer/QueryNode.cpp b/src/Analyzer/QueryNode.cpp index 0fc2edc3cce..618cbd6d9e8 100644 --- a/src/Analyzer/QueryNode.cpp +++ b/src/Analyzer/QueryNode.cpp @@ -21,8 +21,10 @@ namespace DB { -QueryNode::QueryNode() +QueryNode::QueryNode(ContextMutablePtr context_, SettingsChanges settings_changes_) : IQueryTreeNode(children_size) + , context(std::move(context_)) + , settings_changes(std::move(settings_changes_)) { children[with_child_index] = std::make_shared(); children[projection_child_index] = std::make_shared(); @@ -32,6 +34,10 @@ QueryNode::QueryNode() children[limit_by_child_index] = std::make_shared(); } +QueryNode::QueryNode(ContextMutablePtr context_) + : QueryNode(context_, {} /*settings_changes*/) +{} + void QueryNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const { buffer << std::string(indent, ' ') << "QUERY id: " << format_state.getNodeId(this); @@ -222,7 +228,7 @@ void QueryNode::updateTreeHashImpl(HashState & state) const QueryTreeNodePtr QueryNode::cloneImpl() const { - auto result_query_node = std::make_shared(); + auto result_query_node = std::make_shared(context); result_query_node->is_subquery = is_subquery; result_query_node->is_cte = is_cte; diff --git a/src/Analyzer/QueryNode.h b/src/Analyzer/QueryNode.h index 37c4f60fa9e..54154e1e353 100644 --- a/src/Analyzer/QueryNode.h +++ b/src/Analyzer/QueryNode.h @@ -10,6 +10,8 @@ #include #include +#include + namespace DB { @@ -61,7 +63,41 @@ using QueryNodePtr = std::shared_ptr; class QueryNode final : public IQueryTreeNode { public: - explicit QueryNode(); + /// Construct query node with context and changed settings + explicit QueryNode(ContextMutablePtr context_, SettingsChanges settings_changes_); + + /// Construct query node with context + explicit QueryNode(ContextMutablePtr context_); + + /// Get context + ContextPtr getContext() const + { + return context; + } + + /// Get mutable context + const ContextMutablePtr & getMutableContext() const + { + return context; + } + + /// Get mutable context + ContextMutablePtr & getMutableContext() + { + return context; + } + + /// Returns true if query node has settings changes, false otherwise + bool hasSettingsChanges() const + { + return !settings_changes.empty(); + } + + /// Get query node settings changes + const SettingsChanges & getSettingsChanges() const + { + return settings_changes; + } /// Returns true if query node is subquery, false otherwise bool isSubquery() const @@ -513,24 +549,6 @@ public: return children[offset_child_index]; } - /// Returns true if query node has settings changes specified, false otherwise - bool hasSettingsChanges() const - { - return !settings_changes.empty(); - } - - /// Get query node settings changes - const SettingsChanges & getSettingsChanges() const - { - return settings_changes; - } - - /// Set query node settings changes value - void setSettingsChanges(SettingsChanges settings_changes_value) - { - settings_changes = std::move(settings_changes_value); - } - /// Get query node projection columns const NamesAndTypes & getProjectionColumns() const { @@ -572,6 +590,7 @@ private: std::string cte_name; NamesAndTypes projection_columns; + ContextMutablePtr context; SettingsChanges settings_changes; static constexpr size_t with_child_index = 0; diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index 01ecd4ece30..2b2326badfa 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -77,75 +77,90 @@ public: } private: - QueryTreeNodePtr buildSelectOrUnionExpression(const ASTPtr & select_or_union_query, bool is_subquery, const std::string & cte_name) const; + QueryTreeNodePtr buildSelectOrUnionExpression(const ASTPtr & select_or_union_query, + bool is_subquery, + const std::string & cte_name, + const ContextPtr & context) const; - QueryTreeNodePtr buildSelectWithUnionExpression(const ASTPtr & select_with_union_query, bool is_subquery, const std::string & cte_name) const; + QueryTreeNodePtr buildSelectWithUnionExpression(const ASTPtr & select_with_union_query, + bool is_subquery, + const std::string & cte_name, + const ContextPtr & context) const; - QueryTreeNodePtr buildSelectIntersectExceptQuery(const ASTPtr & select_intersect_except_query, bool is_subquery, const std::string & cte_name) const; + QueryTreeNodePtr buildSelectIntersectExceptQuery(const ASTPtr & select_intersect_except_query, + bool is_subquery, + const std::string & cte_name, + const ContextPtr & context) const; - QueryTreeNodePtr buildSelectExpression(const ASTPtr & select_query, bool is_subquery, const std::string & cte_name) const; + QueryTreeNodePtr buildSelectExpression(const ASTPtr & select_query, + bool is_subquery, + const std::string & cte_name, + const ContextPtr & context) const; - QueryTreeNodePtr buildSortList(const ASTPtr & order_by_expression_list) const; + QueryTreeNodePtr buildSortList(const ASTPtr & order_by_expression_list, const ContextPtr & context) const; - QueryTreeNodePtr buildInterpolateList(const ASTPtr & interpolate_expression_list) const; + QueryTreeNodePtr buildInterpolateList(const ASTPtr & interpolate_expression_list, const ContextPtr & context) const; - QueryTreeNodePtr buildWindowList(const ASTPtr & window_definition_list) const; + QueryTreeNodePtr buildWindowList(const ASTPtr & window_definition_list, const ContextPtr & context) const; - QueryTreeNodePtr buildExpressionList(const ASTPtr & expression_list) const; + QueryTreeNodePtr buildExpressionList(const ASTPtr & expression_list, const ContextPtr & context) const; - QueryTreeNodePtr buildExpression(const ASTPtr & expression) const; + QueryTreeNodePtr buildExpression(const ASTPtr & expression, const ContextPtr & context) const; - QueryTreeNodePtr buildWindow(const ASTPtr & window_definition) const; + QueryTreeNodePtr buildWindow(const ASTPtr & window_definition, const ContextPtr & context) const; - QueryTreeNodePtr buildJoinTree(const ASTPtr & tables_in_select_query) const; + QueryTreeNodePtr buildJoinTree(const ASTPtr & tables_in_select_query, const ContextPtr & context) const; - ColumnTransformersNodes buildColumnTransformers(const ASTPtr & matcher_expression, size_t start_child_index) const; + ColumnTransformersNodes buildColumnTransformers(const ASTPtr & matcher_expression, size_t start_child_index, const ContextPtr & context) const; ASTPtr query; - ContextPtr context; QueryTreeNodePtr query_tree_node; - }; QueryTreeBuilder::QueryTreeBuilder(ASTPtr query_, ContextPtr context_) : query(query_->clone()) - , context(std::move(context_)) { if (query->as() || query->as() || query->as()) - query_tree_node = buildSelectOrUnionExpression(query, false /*is_subquery*/, {} /*cte_name*/); + query_tree_node = buildSelectOrUnionExpression(query, false /*is_subquery*/, {} /*cte_name*/, context_); else if (query->as()) - query_tree_node = buildExpressionList(query); + query_tree_node = buildExpressionList(query, context_); else - query_tree_node = buildExpression(query); + query_tree_node = buildExpression(query, context_); } -QueryTreeNodePtr QueryTreeBuilder::buildSelectOrUnionExpression(const ASTPtr & select_or_union_query, bool is_subquery, const std::string & cte_name) const +QueryTreeNodePtr QueryTreeBuilder::buildSelectOrUnionExpression(const ASTPtr & select_or_union_query, + bool is_subquery, + const std::string & cte_name, + const ContextPtr & context) const { QueryTreeNodePtr query_node; if (select_or_union_query->as()) - query_node = buildSelectWithUnionExpression(select_or_union_query, is_subquery /*is_subquery*/, cte_name /*cte_name*/); + query_node = buildSelectWithUnionExpression(select_or_union_query, is_subquery /*is_subquery*/, cte_name /*cte_name*/, context); else if (select_or_union_query->as()) - query_node = buildSelectIntersectExceptQuery(select_or_union_query, is_subquery /*is_subquery*/, cte_name /*cte_name*/); + query_node = buildSelectIntersectExceptQuery(select_or_union_query, is_subquery /*is_subquery*/, cte_name /*cte_name*/, context); else if (select_or_union_query->as()) - query_node = buildSelectExpression(select_or_union_query, is_subquery /*is_subquery*/, cte_name /*cte_name*/); + query_node = buildSelectExpression(select_or_union_query, is_subquery /*is_subquery*/, cte_name /*cte_name*/, context); else throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "SELECT or UNION query {} is not supported", select_or_union_query->formatForErrorMessage()); return query_node; } -QueryTreeNodePtr QueryTreeBuilder::buildSelectWithUnionExpression(const ASTPtr & select_with_union_query, bool is_subquery, const std::string & cte_name) const +QueryTreeNodePtr QueryTreeBuilder::buildSelectWithUnionExpression(const ASTPtr & select_with_union_query, + bool is_subquery, + const std::string & cte_name, + const ContextPtr & context) const { auto & select_with_union_query_typed = select_with_union_query->as(); auto & select_lists = select_with_union_query_typed.list_of_selects->as(); if (select_lists.children.size() == 1) - return buildSelectOrUnionExpression(select_lists.children[0], is_subquery, cte_name); + return buildSelectOrUnionExpression(select_lists.children[0], is_subquery, cte_name, context); - auto union_node = std::make_shared(select_with_union_query_typed.union_mode); + auto union_node = std::make_shared(Context::createCopy(context), select_with_union_query_typed.union_mode); union_node->setIsSubquery(is_subquery); union_node->setIsCTE(!cte_name.empty()); union_node->setCTEName(cte_name); @@ -156,20 +171,23 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectWithUnionExpression(const ASTPtr & for (size_t i = 0; i < select_lists_children_size; ++i) { auto & select_list_node = select_lists.children[i]; - QueryTreeNodePtr query_node = buildSelectOrUnionExpression(select_list_node, false /*is_subquery*/, {} /*cte_name*/); + QueryTreeNodePtr query_node = buildSelectOrUnionExpression(select_list_node, false /*is_subquery*/, {} /*cte_name*/, context); union_node->getQueries().getNodes().push_back(std::move(query_node)); } return union_node; } -QueryTreeNodePtr QueryTreeBuilder::buildSelectIntersectExceptQuery(const ASTPtr & select_intersect_except_query, bool is_subquery, const std::string & cte_name) const +QueryTreeNodePtr QueryTreeBuilder::buildSelectIntersectExceptQuery(const ASTPtr & select_intersect_except_query, + bool is_subquery, + const std::string & cte_name, + const ContextPtr & context) const { auto & select_intersect_except_query_typed = select_intersect_except_query->as(); auto select_lists = select_intersect_except_query_typed.getListOfSelects(); if (select_lists.size() == 1) - return buildSelectExpression(select_lists[0], is_subquery, cte_name); + return buildSelectExpression(select_lists[0], is_subquery, cte_name, context); SelectUnionMode union_mode; if (select_intersect_except_query_typed.final_operator == ASTSelectIntersectExceptQuery::Operator::INTERSECT_ALL) @@ -183,7 +201,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectIntersectExceptQuery(const ASTPtr else throw Exception(ErrorCodes::LOGICAL_ERROR, "UNION type is not initialized"); - auto union_node = std::make_shared(union_mode); + auto union_node = std::make_shared(Context::createCopy(context), union_mode); union_node->setIsSubquery(is_subquery); union_node->setIsCTE(!cte_name.empty()); union_node->setCTEName(cte_name); @@ -194,17 +212,32 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectIntersectExceptQuery(const ASTPtr for (size_t i = 0; i < select_lists_size; ++i) { auto & select_list_node = select_lists[i]; - QueryTreeNodePtr query_node = buildSelectOrUnionExpression(select_list_node, false /*is_subquery*/, {} /*cte_name*/); + QueryTreeNodePtr query_node = buildSelectOrUnionExpression(select_list_node, false /*is_subquery*/, {} /*cte_name*/, context); union_node->getQueries().getNodes().push_back(std::move(query_node)); } return union_node; } -QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression(const ASTPtr & select_query, bool is_subquery, const std::string & cte_name) const +QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression(const ASTPtr & select_query, + bool is_subquery, + const std::string & cte_name, + const ContextPtr & context) const { const auto & select_query_typed = select_query->as(); - auto current_query_tree = std::make_shared(); + + auto updated_context = Context::createCopy(context); + auto select_settings = select_query_typed.settings(); + SettingsChanges settings_changes; + + if (select_settings) + { + auto & set_query = select_settings->as(); + updated_context->applySettingsChanges(set_query.changes); + settings_changes = set_query.changes; + } + + auto current_query_tree = std::make_shared(std::move(updated_context), std::move(settings_changes)); current_query_tree->setIsSubquery(is_subquery); current_query_tree->setIsCTE(!cte_name.empty()); @@ -218,30 +251,25 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression(const ASTPtr & select_q current_query_tree->setIsGroupByAll(select_query_typed.group_by_all); current_query_tree->setOriginalAST(select_query); - auto select_settings = select_query_typed.settings(); - if (select_settings) - { - auto & set_query = select_settings->as(); - current_query_tree->setSettingsChanges(set_query.changes); - } + auto current_context = current_query_tree->getContext(); - current_query_tree->getJoinTree() = buildJoinTree(select_query_typed.tables()); + current_query_tree->getJoinTree() = buildJoinTree(select_query_typed.tables(), current_context); auto select_with_list = select_query_typed.with(); if (select_with_list) - current_query_tree->getWithNode() = buildExpressionList(select_with_list); + current_query_tree->getWithNode() = buildExpressionList(select_with_list, current_context); auto select_expression_list = select_query_typed.select(); if (select_expression_list) - current_query_tree->getProjectionNode() = buildExpressionList(select_expression_list); + current_query_tree->getProjectionNode() = buildExpressionList(select_expression_list, current_context); auto prewhere_expression = select_query_typed.prewhere(); if (prewhere_expression) - current_query_tree->getPrewhere() = buildExpression(prewhere_expression); + current_query_tree->getPrewhere() = buildExpression(prewhere_expression, current_context); auto where_expression = select_query_typed.where(); if (where_expression) - current_query_tree->getWhere() = buildExpression(where_expression); + current_query_tree->getWhere() = buildExpression(where_expression, current_context); auto group_by_list = select_query_typed.groupBy(); if (group_by_list) @@ -254,56 +282,56 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression(const ASTPtr & select_q for (auto & grouping_sets_keys : group_by_children) { - auto grouping_sets_keys_list_node = buildExpressionList(grouping_sets_keys); + auto grouping_sets_keys_list_node = buildExpressionList(grouping_sets_keys, current_context); current_query_tree->getGroupBy().getNodes().emplace_back(std::move(grouping_sets_keys_list_node)); } } else { - current_query_tree->getGroupByNode() = buildExpressionList(group_by_list); + current_query_tree->getGroupByNode() = buildExpressionList(group_by_list, current_context); } } auto having_expression = select_query_typed.having(); if (having_expression) - current_query_tree->getHaving() = buildExpression(having_expression); + current_query_tree->getHaving() = buildExpression(having_expression, current_context); auto window_list = select_query_typed.window(); if (window_list) - current_query_tree->getWindowNode() = buildWindowList(window_list); + current_query_tree->getWindowNode() = buildWindowList(window_list, current_context); auto select_order_by_list = select_query_typed.orderBy(); if (select_order_by_list) - current_query_tree->getOrderByNode() = buildSortList(select_order_by_list); + current_query_tree->getOrderByNode() = buildSortList(select_order_by_list, current_context); auto interpolate_list = select_query_typed.interpolate(); if (interpolate_list) - current_query_tree->getInterpolate() = buildInterpolateList(interpolate_list); + current_query_tree->getInterpolate() = buildInterpolateList(interpolate_list, current_context); auto select_limit_by_limit = select_query_typed.limitByLength(); if (select_limit_by_limit) - current_query_tree->getLimitByLimit() = buildExpression(select_limit_by_limit); + current_query_tree->getLimitByLimit() = buildExpression(select_limit_by_limit, current_context); auto select_limit_by_offset = select_query_typed.limitOffset(); if (select_limit_by_offset) - current_query_tree->getLimitByOffset() = buildExpression(select_limit_by_offset); + current_query_tree->getLimitByOffset() = buildExpression(select_limit_by_offset, current_context); auto select_limit_by = select_query_typed.limitBy(); if (select_limit_by) - current_query_tree->getLimitByNode() = buildExpressionList(select_limit_by); + current_query_tree->getLimitByNode() = buildExpressionList(select_limit_by, current_context); auto select_limit = select_query_typed.limitLength(); if (select_limit) - current_query_tree->getLimit() = buildExpression(select_limit); + current_query_tree->getLimit() = buildExpression(select_limit, current_context); auto select_offset = select_query_typed.limitOffset(); if (select_offset) - current_query_tree->getOffset() = buildExpression(select_offset); + current_query_tree->getOffset() = buildExpression(select_offset, current_context); return current_query_tree; } -QueryTreeNodePtr QueryTreeBuilder::buildSortList(const ASTPtr & order_by_expression_list) const +QueryTreeNodePtr QueryTreeBuilder::buildSortList(const ASTPtr & order_by_expression_list, const ContextPtr & context) const { auto list_node = std::make_shared(); @@ -324,7 +352,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildSortList(const ASTPtr & order_by_express collator = std::make_shared(order_by_element.collation->as().value.get()); const auto & sort_expression_ast = order_by_element.children.at(0); - auto sort_expression = buildExpression(sort_expression_ast); + auto sort_expression = buildExpression(sort_expression_ast, context); auto sort_node = std::make_shared(std::move(sort_expression), sort_direction, nulls_sort_direction, @@ -332,11 +360,11 @@ QueryTreeNodePtr QueryTreeBuilder::buildSortList(const ASTPtr & order_by_express order_by_element.with_fill); if (order_by_element.fill_from) - sort_node->getFillFrom() = buildExpression(order_by_element.fill_from); + sort_node->getFillFrom() = buildExpression(order_by_element.fill_from, context); if (order_by_element.fill_to) - sort_node->getFillTo() = buildExpression(order_by_element.fill_to); + sort_node->getFillTo() = buildExpression(order_by_element.fill_to, context); if (order_by_element.fill_step) - sort_node->getFillStep() = buildExpression(order_by_element.fill_step); + sort_node->getFillStep() = buildExpression(order_by_element.fill_step, context); list_node->getNodes().push_back(std::move(sort_node)); } @@ -344,7 +372,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildSortList(const ASTPtr & order_by_express return list_node; } -QueryTreeNodePtr QueryTreeBuilder::buildInterpolateList(const ASTPtr & interpolate_expression_list) const +QueryTreeNodePtr QueryTreeBuilder::buildInterpolateList(const ASTPtr & interpolate_expression_list, const ContextPtr & context) const { auto list_node = std::make_shared(); @@ -355,7 +383,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildInterpolateList(const ASTPtr & interpola { const auto & interpolate_element = expression->as(); auto expression_to_interpolate = std::make_shared(Identifier(interpolate_element.column)); - auto interpolate_expression = buildExpression(interpolate_element.expr); + auto interpolate_expression = buildExpression(interpolate_element.expr, context); auto interpolate_node = std::make_shared(std::move(expression_to_interpolate), std::move(interpolate_expression)); list_node->getNodes().push_back(std::move(interpolate_node)); @@ -364,7 +392,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildInterpolateList(const ASTPtr & interpola return list_node; } -QueryTreeNodePtr QueryTreeBuilder::buildWindowList(const ASTPtr & window_definition_list) const +QueryTreeNodePtr QueryTreeBuilder::buildWindowList(const ASTPtr & window_definition_list, const ContextPtr & context) const { auto list_node = std::make_shared(); @@ -375,7 +403,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildWindowList(const ASTPtr & window_definit { const auto & window_list_element_typed = window_list_element->as(); - auto window_node = buildWindow(window_list_element_typed.definition); + auto window_node = buildWindow(window_list_element_typed.definition, context); window_node->setAlias(window_list_element_typed.name); list_node->getNodes().push_back(std::move(window_node)); @@ -384,7 +412,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildWindowList(const ASTPtr & window_definit return list_node; } -QueryTreeNodePtr QueryTreeBuilder::buildExpressionList(const ASTPtr & expression_list) const +QueryTreeNodePtr QueryTreeBuilder::buildExpressionList(const ASTPtr & expression_list, const ContextPtr & context) const { auto list_node = std::make_shared(); @@ -393,14 +421,14 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpressionList(const ASTPtr & expression for (auto & expression : expression_list_typed.children) { - auto expression_node = buildExpression(expression); + auto expression_node = buildExpression(expression, context); list_node->getNodes().push_back(std::move(expression_node)); } return list_node; } -QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression) const +QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression, const ContextPtr & context) const { QueryTreeNodePtr result; @@ -411,13 +439,13 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression) co } else if (const auto * asterisk = expression->as()) { - auto column_transformers = buildColumnTransformers(expression, 0 /*start_child_index*/); + auto column_transformers = buildColumnTransformers(expression, 0 /*start_child_index*/, context); result = std::make_shared(std::move(column_transformers)); } else if (const auto * qualified_asterisk = expression->as()) { auto & qualified_identifier = qualified_asterisk->children.at(0)->as(); - auto column_transformers = buildColumnTransformers(expression, 1 /*start_child_index*/); + auto column_transformers = buildColumnTransformers(expression, 1 /*start_child_index*/, context); result = std::make_shared(Identifier(qualified_identifier.name_parts), std::move(column_transformers)); } else if (const auto * ast_literal = expression->as()) @@ -466,7 +494,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression) co } const auto & lambda_expression = lambda_arguments_and_expression.at(1); - auto lambda_expression_node = buildExpression(lambda_expression); + auto lambda_expression_node = buildExpression(lambda_expression, context); result = std::make_shared(std::move(lambda_arguments), std::move(lambda_expression_node)); } @@ -478,20 +506,20 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression) co { const auto & function_parameters_list = function->parameters->as()->children; for (const auto & argument : function_parameters_list) - function_node->getParameters().getNodes().push_back(buildExpression(argument)); + function_node->getParameters().getNodes().push_back(buildExpression(argument, context)); } if (function->arguments) { const auto & function_arguments_list = function->arguments->as()->children; for (const auto & argument : function_arguments_list) - function_node->getArguments().getNodes().push_back(buildExpression(argument)); + function_node->getArguments().getNodes().push_back(buildExpression(argument, context)); } if (function->is_window_function) { if (function->window_definition) - function_node->getWindowNode() = buildWindow(function->window_definition); + function_node->getWindowNode() = buildWindow(function->window_definition, context); else function_node->getWindowNode() = std::make_shared(Identifier(function->window_name)); } @@ -502,20 +530,20 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression) co else if (const auto * subquery = expression->as()) { auto subquery_query = subquery->children[0]; - auto query_node = buildSelectWithUnionExpression(subquery_query, true /*is_subquery*/, {} /*cte_name*/); + auto query_node = buildSelectWithUnionExpression(subquery_query, true /*is_subquery*/, {} /*cte_name*/, context); result = std::move(query_node); } else if (const auto * with_element = expression->as()) { auto with_element_subquery = with_element->subquery->as().children.at(0); - auto query_node = buildSelectWithUnionExpression(with_element_subquery, true /*is_subquery*/, with_element->name /*cte_name*/); + auto query_node = buildSelectWithUnionExpression(with_element_subquery, true /*is_subquery*/, with_element->name /*cte_name*/, context); result = std::move(query_node); } else if (const auto * columns_regexp_matcher = expression->as()) { - auto column_transformers = buildColumnTransformers(expression, 0 /*start_child_index*/); + auto column_transformers = buildColumnTransformers(expression, 0 /*start_child_index*/, context); result = std::make_shared(columns_regexp_matcher->getMatcher(), std::move(column_transformers)); } else if (const auto * columns_list_matcher = expression->as()) @@ -529,13 +557,13 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression) co column_list_identifiers.emplace_back(Identifier{column_list_identifier.name_parts}); } - auto column_transformers = buildColumnTransformers(expression, 0 /*start_child_index*/); + auto column_transformers = buildColumnTransformers(expression, 0 /*start_child_index*/, context); result = std::make_shared(std::move(column_list_identifiers), std::move(column_transformers)); } else if (const auto * qualified_columns_regexp_matcher = expression->as()) { auto & qualified_identifier = qualified_columns_regexp_matcher->children.at(0)->as(); - auto column_transformers = buildColumnTransformers(expression, 1 /*start_child_index*/); + auto column_transformers = buildColumnTransformers(expression, 1 /*start_child_index*/, context); result = std::make_shared(Identifier(qualified_identifier.name_parts), qualified_columns_regexp_matcher->getMatcher(), std::move(column_transformers)); } else if (const auto * qualified_columns_list_matcher = expression->as()) @@ -551,7 +579,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression) co column_list_identifiers.emplace_back(Identifier{column_list_identifier.name_parts}); } - auto column_transformers = buildColumnTransformers(expression, 1 /*start_child_index*/); + auto column_transformers = buildColumnTransformers(expression, 1 /*start_child_index*/, context); result = std::make_shared(Identifier(qualified_identifier.name_parts), std::move(column_list_identifiers), std::move(column_transformers)); } else @@ -567,7 +595,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildExpression(const ASTPtr & expression) co return result; } -QueryTreeNodePtr QueryTreeBuilder::buildWindow(const ASTPtr & window_definition) const +QueryTreeNodePtr QueryTreeBuilder::buildWindow(const ASTPtr & window_definition, const ContextPtr & context) const { const auto & window_definition_typed = window_definition->as(); WindowFrame window_frame; @@ -586,23 +614,23 @@ QueryTreeNodePtr QueryTreeBuilder::buildWindow(const ASTPtr & window_definition) window_node->setParentWindowName(window_definition_typed.parent_window_name); if (window_definition_typed.partition_by) - window_node->getPartitionByNode() = buildExpressionList(window_definition_typed.partition_by); + window_node->getPartitionByNode() = buildExpressionList(window_definition_typed.partition_by, context); if (window_definition_typed.order_by) - window_node->getOrderByNode() = buildSortList(window_definition_typed.order_by); + window_node->getOrderByNode() = buildSortList(window_definition_typed.order_by, context); if (window_definition_typed.frame_begin_offset) - window_node->getFrameBeginOffsetNode() = buildExpression(window_definition_typed.frame_begin_offset); + window_node->getFrameBeginOffsetNode() = buildExpression(window_definition_typed.frame_begin_offset, context); if (window_definition_typed.frame_end_offset) - window_node->getFrameEndOffsetNode() = buildExpression(window_definition_typed.frame_end_offset); + window_node->getFrameEndOffsetNode() = buildExpression(window_definition_typed.frame_end_offset, context); window_node->setOriginalAST(window_definition); return window_node; } -QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTPtr & tables_in_select_query) const +QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTPtr & tables_in_select_query, const ContextPtr & context) const { if (!tables_in_select_query) { @@ -668,7 +696,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTPtr & tables_in_select auto & subquery_expression = table_expression.subquery->as(); const auto & select_with_union_query = subquery_expression.children[0]; - auto node = buildSelectWithUnionExpression(select_with_union_query, true /*is_subquery*/, {} /*cte_name*/); + auto node = buildSelectWithUnionExpression(select_with_union_query, true /*is_subquery*/, {} /*cte_name*/, context); node->setAlias(subquery_expression.tryGetAlias()); node->setOriginalAST(select_with_union_query); @@ -694,9 +722,9 @@ QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTPtr & tables_in_select for (const auto & argument : function_arguments_list) { if (argument->as() || argument->as() || argument->as()) - node->getArguments().getNodes().push_back(buildSelectOrUnionExpression(argument, false /*is_subquery*/, {} /*cte_name*/)); + node->getArguments().getNodes().push_back(buildSelectOrUnionExpression(argument, false /*is_subquery*/, {} /*cte_name*/, context)); else - node->getArguments().getNodes().push_back(buildExpression(argument)); + node->getArguments().getNodes().push_back(buildExpression(argument, context)); } } @@ -726,9 +754,9 @@ QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTPtr & tables_in_select QueryTreeNodePtr join_expression; if (table_join.using_expression_list) - join_expression = buildExpressionList(table_join.using_expression_list); + join_expression = buildExpressionList(table_join.using_expression_list, context); else if (table_join.on_expression) - join_expression = buildExpression(table_join.on_expression); + join_expression = buildExpression(table_join.on_expression, context); const auto & settings = context->getSettingsRef(); auto join_default_strictness = settings.join_default_strictness; @@ -785,7 +813,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTPtr & tables_in_select auto last_table_expression = std::move(table_expressions.back()); table_expressions.pop_back(); - auto array_join_expressions_list = buildExpressionList(array_join_expression.expression_list); + auto array_join_expressions_list = buildExpressionList(array_join_expression.expression_list, context); auto array_join_node = std::make_shared(std::move(last_table_expression), std::move(array_join_expressions_list), is_left_array_join); /** Original AST is not set because it will contain only array join part and does @@ -805,7 +833,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildJoinTree(const ASTPtr & tables_in_select } -ColumnTransformersNodes QueryTreeBuilder::buildColumnTransformers(const ASTPtr & matcher_expression, size_t start_child_index) const +ColumnTransformersNodes QueryTreeBuilder::buildColumnTransformers(const ASTPtr & matcher_expression, size_t start_child_index, const ContextPtr & context) const { ColumnTransformersNodes column_transformers; size_t children_size = matcher_expression->children.size(); @@ -818,14 +846,14 @@ ColumnTransformersNodes QueryTreeBuilder::buildColumnTransformers(const ASTPtr & { if (apply_transformer->lambda) { - auto lambda_query_tree_node = buildExpression(apply_transformer->lambda); + auto lambda_query_tree_node = buildExpression(apply_transformer->lambda, context); column_transformers.emplace_back(std::make_shared(std::move(lambda_query_tree_node))); } else { auto function_node = std::make_shared(apply_transformer->func_name); if (apply_transformer->parameters) - function_node->getParametersNode() = buildExpressionList(apply_transformer->parameters); + function_node->getParametersNode() = buildExpressionList(apply_transformer->parameters, context); column_transformers.emplace_back(std::make_shared(std::move(function_node))); } @@ -856,7 +884,7 @@ ColumnTransformersNodes QueryTreeBuilder::buildColumnTransformers(const ASTPtr & for (const auto & replace_transformer_child : replace_transformer->children) { auto & replacement = replace_transformer_child->as(); - replacements.emplace_back(ReplaceColumnTransformerNode::Replacement{replacement.name, buildExpression(replacement.expr)}); + replacements.emplace_back(ReplaceColumnTransformerNode::Replacement{replacement.name, buildExpression(replacement.expr, context)}); } column_transformers.emplace_back(std::make_shared(replacements, replace_transformer->is_strict)); diff --git a/src/Analyzer/QueryTreeBuilder.h b/src/Analyzer/QueryTreeBuilder.h index de0f6270230..acff62e07c9 100644 --- a/src/Analyzer/QueryTreeBuilder.h +++ b/src/Analyzer/QueryTreeBuilder.h @@ -13,6 +13,8 @@ namespace DB * AST that represent query ASTSelectWithUnionQuery, ASTSelectIntersectExceptQuery, ASTSelectQuery. * AST that represent a list of expressions ASTExpressionList. * AST that represent expression ASTIdentifier, ASTAsterisk, ASTLiteral, ASTFunction. + * + * For QUERY and UNION nodes contexts are created with respect to specified SETTINGS. */ QueryTreeNodePtr buildQueryTree(ASTPtr query, ContextPtr context); diff --git a/src/Analyzer/UnionNode.cpp b/src/Analyzer/UnionNode.cpp index 19f94e68a33..18733b32437 100644 --- a/src/Analyzer/UnionNode.cpp +++ b/src/Analyzer/UnionNode.cpp @@ -3,8 +3,6 @@ #include #include -#include - #include #include #include @@ -18,9 +16,12 @@ #include #include +#include #include +#include + #include #include @@ -33,8 +34,9 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -UnionNode::UnionNode(SelectUnionMode union_mode_) +UnionNode::UnionNode(ContextMutablePtr context_, SelectUnionMode union_mode_) : IQueryTreeNode(children_size) + , context(std::move(context_)) , union_mode(union_mode_) { if (union_mode == SelectUnionMode::UNION_DEFAULT || @@ -129,7 +131,7 @@ void UnionNode::updateTreeHashImpl(HashState & state) const QueryTreeNodePtr UnionNode::cloneImpl() const { - auto result_union_node = std::make_shared(union_mode); + auto result_union_node = std::make_shared(context, union_mode); result_union_node->is_subquery = is_subquery; result_union_node->is_cte = is_cte; diff --git a/src/Analyzer/UnionNode.h b/src/Analyzer/UnionNode.h index c58d1bc0e12..5e3861da814 100644 --- a/src/Analyzer/UnionNode.h +++ b/src/Analyzer/UnionNode.h @@ -3,12 +3,14 @@ #include #include +#include + #include #include #include #include -#include +#include namespace DB { @@ -37,8 +39,26 @@ using UnionNodePtr = std::shared_ptr; class UnionNode final : public IQueryTreeNode { public: - /// Construct union node with normalized union mode - explicit UnionNode(SelectUnionMode union_mode_); + /// Construct union node with context and normalized union mode + explicit UnionNode(ContextMutablePtr context_, SelectUnionMode union_mode_); + + /// Get context + ContextPtr getContext() const + { + return context; + } + + /// Get mutable context + const ContextMutablePtr & getMutableContext() const + { + return context; + } + + /// Get mutable context + ContextMutablePtr & getMutableContext() + { + return context; + } /// Returns true if union node is subquery, false otherwise bool isSubquery() const @@ -129,6 +149,7 @@ private: bool is_subquery = false; bool is_cte = false; std::string cte_name; + ContextMutablePtr context; SelectUnionMode union_mode; static constexpr size_t queries_child_index = 0; diff --git a/src/Common/CMakeLists.txt b/src/Common/CMakeLists.txt index 490628a2180..e527b3dec43 100644 --- a/src/Common/CMakeLists.txt +++ b/src/Common/CMakeLists.txt @@ -1,5 +1,9 @@ add_subdirectory(StringUtils) +if (ENABLE_BENCHMARKS) + add_subdirectory(benchmarks) +endif() + if (ENABLE_EXAMPLES) add_subdirectory(examples) endif() diff --git a/src/Common/Epoll.cpp b/src/Common/Epoll.cpp index 9b2589f0589..fa31734d432 100644 --- a/src/Common/Epoll.cpp +++ b/src/Common/Epoll.cpp @@ -3,7 +3,6 @@ #include "Epoll.h" #include #include -#include namespace DB { @@ -70,9 +69,6 @@ size_t Epoll::getManyReady(int max_events, epoll_event * events_out, bool blocki if (ready_size == -1 && errno != EINTR) throwFromErrno("Error in epoll_wait", DB::ErrorCodes::EPOLL_ERROR); - - if (errno == EINTR) - LOG_TEST(&Poco::Logger::get("Epoll"), "EINTR"); } while (ready_size <= 0 && (ready_size != 0 || blocking)); diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index e4e718e7ebc..25ba56fa046 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -531,6 +531,11 @@ void increment(Event event, Count amount) DB::CurrentThread::getProfileEvents().increment(event, amount); } +void incrementNoTrace(Event event, Count amount) +{ + DB::CurrentThread::getProfileEvents().incrementNoTrace(event, amount); +} + void Counters::increment(Event event, Count amount) { Counters * current = this; @@ -547,6 +552,16 @@ void Counters::increment(Event event, Count amount) DB::TraceSender::send(DB::TraceType::ProfileEvent, StackTrace(), {.event = event, .increment = amount}); } +void Counters::incrementNoTrace(Event event, Count amount) +{ + Counters * current = this; + do + { + current->counters[event].fetch_add(amount, std::memory_order_relaxed); + current = current->parent; + } while (current != nullptr); +} + CountersIncrement::CountersIncrement(Counters::Snapshot const & snapshot) { init(); diff --git a/src/Common/ProfileEvents.h b/src/Common/ProfileEvents.h index 256a17cc080..867b5b551c6 100644 --- a/src/Common/ProfileEvents.h +++ b/src/Common/ProfileEvents.h @@ -54,6 +54,7 @@ namespace ProfileEvents } void increment(Event event, Count amount = 1); + void incrementNoTrace(Event event, Count amount = 1); struct Snapshot { @@ -105,6 +106,10 @@ namespace ProfileEvents /// Increment a counter for event. Thread-safe. void increment(Event event, Count amount = 1); + /// The same as above but ignores value of setting 'trace_profile_events' + /// and never sends profile event to trace log. + void incrementNoTrace(Event event, Count amount = 1); + /// Get name of event by identifier. Returns statically allocated string. const char * getName(Event event); diff --git a/src/Common/QueryProfiler.cpp b/src/Common/QueryProfiler.cpp index 14a6a06088c..e0a59405a62 100644 --- a/src/Common/QueryProfiler.cpp +++ b/src/Common/QueryProfiler.cpp @@ -50,11 +50,11 @@ namespace /// But pass with some frequency to avoid drop of all traces. if (overrun_count > 0 && write_trace_iteration % (overrun_count + 1) == 0) { - ProfileEvents::increment(ProfileEvents::QueryProfilerSignalOverruns, overrun_count); + ProfileEvents::incrementNoTrace(ProfileEvents::QueryProfilerSignalOverruns, overrun_count); } else { - ProfileEvents::increment(ProfileEvents::QueryProfilerSignalOverruns, std::max(0, overrun_count) + 1); + ProfileEvents::incrementNoTrace(ProfileEvents::QueryProfilerSignalOverruns, std::max(0, overrun_count) + 1); return; } } @@ -67,7 +67,7 @@ namespace const StackTrace stack_trace(signal_context); TraceSender::send(trace_type, stack_trace, {}); - ProfileEvents::increment(ProfileEvents::QueryProfilerRuns); + ProfileEvents::incrementNoTrace(ProfileEvents::QueryProfilerRuns); errno = saved_errno; } diff --git a/src/Common/SipHash.h b/src/Common/SipHash.h index d931b871550..96b095724c2 100644 --- a/src/Common/SipHash.h +++ b/src/Common/SipHash.h @@ -164,8 +164,13 @@ public: void get128(char * out) { finalize(); +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + unalignedStore(out + 8, v0 ^ v1); + unalignedStore(out, v2 ^ v3); +#else unalignedStore(out, v0 ^ v1); unalignedStore(out + 8, v2 ^ v3); +#endif } template diff --git a/src/Common/TimerDescriptor.cpp b/src/Common/TimerDescriptor.cpp index 1301ebce0ba..a7c74dab8be 100644 --- a/src/Common/TimerDescriptor.cpp +++ b/src/Common/TimerDescriptor.cpp @@ -6,8 +6,6 @@ #include #include -#include - namespace DB { @@ -72,8 +70,6 @@ void TimerDescriptor::drain() const if (errno != EINTR) throwFromErrno("Cannot drain timer_fd", ErrorCodes::CANNOT_READ_FROM_SOCKET); - else - LOG_TEST(&Poco::Logger::get("TimerDescriptor"), "EINTR"); } } } diff --git a/src/Common/benchmarks/CMakeLists.txt b/src/Common/benchmarks/CMakeLists.txt new file mode 100644 index 00000000000..57ed837db8b --- /dev/null +++ b/src/Common/benchmarks/CMakeLists.txt @@ -0,0 +1,9 @@ +clickhouse_add_executable(integer_hash_tables_and_hashes integer_hash_tables_and_hashes.cpp) +target_link_libraries (integer_hash_tables_and_hashes PRIVATE + ch_contrib::gbenchmark_all + dbms + ch_contrib::abseil_swiss_tables + ch_contrib::sparsehash + ch_contrib::wyhash + ch_contrib::farmhash + ch_contrib::xxHash) diff --git a/src/Common/examples/integer_hash_tables_and_hashes.cpp b/src/Common/benchmarks/integer_hash_tables_and_hashes.cpp similarity index 94% rename from src/Common/examples/integer_hash_tables_and_hashes.cpp rename to src/Common/benchmarks/integer_hash_tables_and_hashes.cpp index 0e9390ab3ac..c245fc471cc 100644 --- a/src/Common/examples/integer_hash_tables_and_hashes.cpp +++ b/src/Common/benchmarks/integer_hash_tables_and_hashes.cpp @@ -1,5 +1,8 @@ -#include +#include + #include +#include +#include #include #include @@ -13,12 +16,23 @@ //#define DBMS_HASH_MAP_COUNT_COLLISIONS //#define DBMS_HASH_MAP_DEBUG_RESIZES -#include -#include +#include +#include #include +#include +#include #include #include +#include +#include + +#ifdef __clang__ +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wused-but-marked-unused" +#endif +#include + using Key = UInt64; using Value = UInt64; @@ -282,98 +296,91 @@ namespace Hashes return res; } }; + + struct FarmHash + { + size_t operator()(Key x) const { return NAMESPACE_FOR_HASH_FUNCTIONS::Hash64(reinterpret_cast(&x), sizeof(x)); } + }; + + struct WyHash + { + size_t operator()(Key x) const { return wyhash(reinterpret_cast(&x), sizeof(x), 0, _wyp); } + }; + + struct XXH3Hash + { + size_t operator()(Key x) const { return XXH_INLINE_XXH3_64bits(reinterpret_cast(&x), sizeof(x)); } + }; } template