diff --git a/.clang-tidy b/.clang-tidy index dc1cebe9430..d53047f0552 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -123,7 +123,21 @@ Checks: [ '-readability-uppercase-literal-suffix', '-readability-use-anyofallof', - '-zircon-*' + '-zircon-*', + + # These are new in clang-18, and we have to sort them out: + '-readability-avoid-nested-conditional-operator', + '-modernize-use-designated-initializers', + '-performance-enum-size', + '-readability-redundant-inline-specifier', + '-readability-redundant-member-init', + '-bugprone-crtp-constructor-accessibility', + '-bugprone-suspicious-stringview-data-usage', + '-bugprone-multi-level-implicit-pointer-conversion', + '-cert-err33-c', + + # This is a good check, but clang-tidy crashes, see https://github.com/llvm/llvm-project/issues/91872 + '-modernize-use-constraints', ] WarningsAsErrors: '*' diff --git a/.github/workflows/merge_queue.yml b/.github/workflows/merge_queue.yml new file mode 100644 index 00000000000..1b6cc320ec4 --- /dev/null +++ b/.github/workflows/merge_queue.yml @@ -0,0 +1,96 @@ +# yamllint disable rule:comments-indentation +name: MergeQueueCI + +env: + # Force the stdout and stderr streams to be unbuffered + PYTHONUNBUFFERED: 1 + +on: # yamllint disable-line rule:truthy + merge_group: + +jobs: + RunConfig: + runs-on: [self-hosted, style-checker-aarch64] + outputs: + data: ${{ steps.runconfig.outputs.CI_DATA }} + steps: + - name: DebugInfo + uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6 + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true # to ensure correct digests + fetch-depth: 0 # to get version + filter: tree:0 + - name: Python unit tests + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + echo "Testing the main ci directory" + python3 -m unittest discover -s . -p 'test_*.py' + for dir in *_lambda/; do + echo "Testing $dir" + python3 -m unittest discover -s "$dir" -p 'test_*.py' + done + - name: PrepareRunConfig + id: runconfig + run: | + python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --outfile ${{ runner.temp }}/ci_run_data.json + + echo "::group::CI configuration" + python3 -m json.tool ${{ runner.temp }}/ci_run_data.json + echo "::endgroup::" + + { + echo 'CI_DATA<> "$GITHUB_OUTPUT" + BuildDockers: + needs: [RunConfig] + if: ${{ !failure() && !cancelled() && toJson(fromJson(needs.RunConfig.outputs.data).docker_data.missing_multi) != '[]' }} + uses: ./.github/workflows/reusable_docker.yml + with: + data: ${{ needs.RunConfig.outputs.data }} + StyleCheck: + needs: [RunConfig, BuildDockers] + if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Style check')}} + uses: ./.github/workflows/reusable_test.yml + with: + test_name: Style check + runner_type: style-checker + run_command: | + python3 style_check.py + data: ${{ needs.RunConfig.outputs.data }} + secrets: + secret_envs: | + ROBOT_CLICKHOUSE_SSH_KEY<> "$GITHUB_OUTPUT" - name: Re-create GH statuses for skipped jobs if any - if: ${{ github.event_name != 'merge_group' }} run: | python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ runner.temp }}/ci_run_data.json --update-gh-statuses BuildDockers: @@ -83,7 +80,7 @@ jobs: ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}} RCSK FastTest: - needs: [RunConfig, BuildDockers] + needs: [RunConfig, BuildDockers, StyleCheck] if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Fast test') }} uses: ./.github/workflows/reusable_test.yml with: @@ -163,20 +160,16 @@ jobs: steps: - name: Check out repository code uses: ClickHouse/checkout@v1 - - name: Check sync status - if: ${{ github.event_name == 'merge_group' }} - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 sync_pr.py --status - name: Finish label run: | cd "$GITHUB_WORKSPACE/tests/ci" - python3 finish_check.py ${{ (contains(needs.*.result, 'failure') && github.event_name == 'merge_group') && '--pipeline-failure' || '' }} - - name: Auto merge if approved - if: ${{ github.event_name != 'merge_group' }} - run: | - cd "$GITHUB_WORKSPACE/tests/ci" - python3 merge_pr.py --check-approved + python3 finish_check.py + # FIXME: merge on approval does not work with MQ. Could be fixed by using defaul GH's automerge after some corrections in Mergeable Check status + # - name: Auto merge if approved + # if: ${{ github.event_name != 'merge_group' }} + # run: | + # cd "$GITHUB_WORKSPACE/tests/ci" + # python3 merge_pr.py --check-approved ############################################################################################# diff --git a/.gitmodules b/.gitmodules index fb72dc6e3ec..28696428e8c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,7 +6,7 @@ url = https://github.com/facebook/zstd [submodule "contrib/lz4"] path = contrib/lz4 - url = https://github.com/ClickHouse/lz4 + url = https://github.com/lz4/lz4 [submodule "contrib/librdkafka"] path = contrib/librdkafka url = https://github.com/ClickHouse/librdkafka diff --git a/CMakeLists.txt b/CMakeLists.txt index be804a14765..abbc48ab23a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -135,23 +135,21 @@ endif () include (cmake/check_flags.cmake) include (cmake/add_warning.cmake) -if (COMPILER_CLANG) - # generate ranges for fast "addr2line" search - if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE") - # NOTE: that clang has a bug because of it does not emit .debug_aranges - # with ThinLTO, so custom ld.lld wrapper is shipped in docker images. - set(COMPILER_FLAGS "${COMPILER_FLAGS} -gdwarf-aranges") - endif () - - # See https://blog.llvm.org/posts/2021-04-05-constructor-homing-for-debug-info/ - if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO") - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -fuse-ctor-homing") - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing") - endif() - - no_warning(enum-constexpr-conversion) # breaks Protobuf in clang-16 +# generate ranges for fast "addr2line" search +if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE") + # NOTE: that clang has a bug because of it does not emit .debug_aranges + # with ThinLTO, so custom ld.lld wrapper is shipped in docker images. + set(COMPILER_FLAGS "${COMPILER_FLAGS} -gdwarf-aranges") endif () +# See https://blog.llvm.org/posts/2021-04-05-constructor-homing-for-debug-info/ +if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -fuse-ctor-homing") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing") +endif() + +no_warning(enum-constexpr-conversion) # breaks Protobuf in clang-16 + option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" ON) option(ENABLE_EXAMPLES "Build all example programs in 'examples' subdirectories" OFF) option(ENABLE_BENCHMARKS "Build all benchmark programs in 'benchmarks' subdirectories" OFF) @@ -284,16 +282,12 @@ endif () option (ENABLE_BUILD_PROFILING "Enable profiling of build time" OFF) if (ENABLE_BUILD_PROFILING) - if (COMPILER_CLANG) - set (COMPILER_FLAGS "${COMPILER_FLAGS} -ftime-trace") + set (COMPILER_FLAGS "${COMPILER_FLAGS} -ftime-trace") - if (LINKER_NAME MATCHES "lld") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--time-trace") - set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Wl,--time-trace") - endif () - else () - message (${RECONFIGURE_MESSAGE_LEVEL} "Build profiling is only available with CLang") - endif () + if (LINKER_NAME MATCHES "lld") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--time-trace") + set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Wl,--time-trace") + endif () endif () set (CMAKE_CXX_STANDARD 23) @@ -304,22 +298,20 @@ set (CMAKE_C_STANDARD 11) set (CMAKE_C_EXTENSIONS ON) # required by most contribs written in C set (CMAKE_C_STANDARD_REQUIRED ON) -if (COMPILER_CLANG) - # Enable C++14 sized global deallocation functions. It should be enabled by setting -std=c++14 but I'm not sure. - # See https://reviews.llvm.org/D112921 - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsized-deallocation") +# Enable C++14 sized global deallocation functions. It should be enabled by setting -std=c++14 but I'm not sure. +# See https://reviews.llvm.org/D112921 +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsized-deallocation") - # falign-functions=32 prevents from random performance regressions with the code change. Thus, providing more stable - # benchmarks. - set(COMPILER_FLAGS "${COMPILER_FLAGS} -falign-functions=32") +# falign-functions=32 prevents from random performance regressions with the code change. Thus, providing more stable +# benchmarks. +set(COMPILER_FLAGS "${COMPILER_FLAGS} -falign-functions=32") - if (ARCH_AMD64) - # align branches within a 32-Byte boundary to avoid the potential performance loss when code layout change, - # which makes benchmark results more stable. - set(BRANCHES_WITHIN_32B_BOUNDARIES "-mbranches-within-32B-boundaries") - set(COMPILER_FLAGS "${COMPILER_FLAGS} ${BRANCHES_WITHIN_32B_BOUNDARIES}") - endif() -endif () +if (ARCH_AMD64) + # align branches within a 32-Byte boundary to avoid the potential performance loss when code layout change, + # which makes benchmark results more stable. + set(BRANCHES_WITHIN_32B_BOUNDARIES "-mbranches-within-32B-boundaries") + set(COMPILER_FLAGS "${COMPILER_FLAGS} ${BRANCHES_WITHIN_32B_BOUNDARIES}") +endif() # Disable floating-point expression contraction in order to get consistent floating point calculation results across platforms set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffp-contract=off") @@ -348,39 +340,34 @@ set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} ${COMPILER_FLAGS} $ set (CMAKE_ASM_FLAGS_RELWITHDEBINFO "${CMAKE_ASM_FLAGS_RELWITHDEBINFO} -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_ASM_FLAGS_ADD}") set (CMAKE_ASM_FLAGS_DEBUG "${CMAKE_ASM_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} ${CMAKE_ASM_FLAGS_ADD}") -if (COMPILER_CLANG) - if (OS_DARWIN) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-U,_inside_main") - endif() +if (OS_DARWIN) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-U,_inside_main") +endif() - # Display absolute paths in error messages. Otherwise KDevelop fails to navigate to correct file and opens a new file instead. - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-absolute-paths") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-absolute-paths") +# Display absolute paths in error messages. Otherwise KDevelop fails to navigate to correct file and opens a new file instead. +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-absolute-paths") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-absolute-paths") - if (NOT ENABLE_TESTS AND NOT SANITIZE AND NOT SANITIZE_COVERAGE AND OS_LINUX) - # https://clang.llvm.org/docs/ThinLTO.html - # Applies to clang and linux only. - # Disabled when building with tests or sanitizers. - option(ENABLE_THINLTO "Clang-specific link time optimization" ON) - endif() +if (NOT ENABLE_TESTS AND NOT SANITIZE AND NOT SANITIZE_COVERAGE AND OS_LINUX) + # https://clang.llvm.org/docs/ThinLTO.html + # Applies to clang and linux only. + # Disabled when building with tests or sanitizers. + option(ENABLE_THINLTO "Clang-specific link time optimization" ON) +endif() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fstrict-vtable-pointers") - - # We cannot afford to use LTO when compiling unit tests, and it's not enough - # to only supply -fno-lto at the final linking stage. So we disable it - # completely. - if (ENABLE_THINLTO AND NOT ENABLE_TESTS AND NOT SANITIZE) - # Link time optimization - set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -flto=thin -fwhole-program-vtables") - set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -flto=thin -fwhole-program-vtables") - set (CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -flto=thin -fwhole-program-vtables") - elseif (ENABLE_THINLTO) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot enable ThinLTO") - endif () +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fstrict-vtable-pointers") +# We cannot afford to use LTO when compiling unit tests, and it's not enough +# to only supply -fno-lto at the final linking stage. So we disable it +# completely. +if (ENABLE_THINLTO AND NOT ENABLE_TESTS AND NOT SANITIZE) + # Link time optimization + set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -flto=thin -fwhole-program-vtables") + set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -flto=thin -fwhole-program-vtables") + set (CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -flto=thin -fwhole-program-vtables") elseif (ENABLE_THINLTO) - message (${RECONFIGURE_MESSAGE_LEVEL} "ThinLTO is only available with Clang") + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot enable ThinLTO") endif () # Turns on all external libs like s3, kafka, ODBC, ... diff --git a/README.md b/README.md index d7202454332..73d989210b5 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,6 @@ Every month we get together with the community (users, contributors, customers, Keep an eye out for upcoming meetups and events around the world. Somewhere else you want us to be? Please feel free to reach out to tyler `` clickhouse `` com. You can also peruse [ClickHouse Events](https://clickhouse.com/company/news-events) for a list of all upcoming trainings, meetups, speaking engagements, etc. -* [ClickHouse Meetup in Bengaluru](https://www.meetup.com/clickhouse-bangalore-user-group/events/300405581/) - May 4 * [ClickHouse Happy Hour @ Tom's Watch Bar - Los Angeles](https://www.meetup.com/clickhouse-los-angeles-user-group/events/300740584/) - May 22 * [ClickHouse & Confluent Meetup in Dubai](https://www.meetup.com/clickhouse-dubai-meetup-group/events/299629189/) - May 28 * [ClickHouse Meetup in Stockholm](https://www.meetup.com/clickhouse-stockholm-user-group/events/299752651/) - Jun 3 @@ -49,6 +48,7 @@ Keep an eye out for upcoming meetups and events around the world. Somewhere else * [ClickHouse Meetup in Amsterdam](https://www.meetup.com/clickhouse-netherlands-user-group/events/300781068/) - Jun 27 * [ClickHouse Meetup in Paris](https://www.meetup.com/clickhouse-france-user-group/events/300783448/) - Jul 9 * [ClickHouse Meetup @ Ramp - New York City](https://www.meetup.com/clickhouse-new-york-user-group/events/300595845/) - Jul 9 +* [ClickHouse Meetup @ Klaviyo - Boston](https://www.meetup.com/clickhouse-boston-user-group/events/300907870) - Jul 11 ## Recent Recordings * **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments" diff --git a/base/base/DecomposedFloat.h b/base/base/DecomposedFloat.h index f152637b94e..4837782621e 100644 --- a/base/base/DecomposedFloat.h +++ b/base/base/DecomposedFloat.h @@ -51,11 +51,9 @@ struct DecomposedFloat /// Returns 0 for both +0. and -0. int sign() const { - return (exponent() == 0 && mantissa() == 0) - ? 0 - : (isNegative() - ? -1 - : 1); + if (exponent() == 0 && mantissa() == 0) + return 0; + return isNegative() ? -1 : 1; } uint16_t exponent() const diff --git a/base/base/EnumReflection.h b/base/base/EnumReflection.h index e3208f16a75..4a9de4d17a3 100644 --- a/base/base/EnumReflection.h +++ b/base/base/EnumReflection.h @@ -11,7 +11,7 @@ namespace detail template constexpr void static_for(F && f, std::index_sequence) { - (std::forward(f)(std::integral_constant(I)>()) , ...); + (f(std::integral_constant(I)>()) , ...); } } diff --git a/base/base/JSON.cpp b/base/base/JSON.cpp index 9da059c98b6..a68b6663e50 100644 --- a/base/base/JSON.cpp +++ b/base/base/JSON.cpp @@ -651,7 +651,9 @@ std::string_view JSON::getRawString() const Pos s = ptr_begin; if (*s != '"') throw JSONException(std::string("JSON: expected \", got ") + *s); - while (++s != ptr_end && *s != '"'); + ++s; + while (s != ptr_end && *s != '"') + ++s; if (s != ptr_end) return std::string_view(ptr_begin + 1, s - ptr_begin - 1); throw JSONException("JSON: incorrect syntax (expected end of string, found end of JSON)."); diff --git a/base/base/JSON.h b/base/base/JSON.h index bc053670a96..7b9acf11d9a 100644 --- a/base/base/JSON.h +++ b/base/base/JSON.h @@ -74,7 +74,7 @@ public: const char * data() const { return ptr_begin; } const char * dataEnd() const { return ptr_end; } - enum ElementType + enum ElementType : uint8_t { TYPE_OBJECT, TYPE_ARRAY, diff --git a/base/base/TypeList.h b/base/base/TypeList.h index 310f0c0c586..ebbe1b48b29 100644 --- a/base/base/TypeList.h +++ b/base/base/TypeList.h @@ -27,7 +27,7 @@ namespace TypeListUtils /// In some contexts it's more handy to use functions in constexpr Root changeRoot(TypeList) { return {}; } template - constexpr void forEach(TypeList, F && f) { (std::forward(f)(TypeList{}), ...); } + constexpr void forEach(TypeList, F && f) { (f(TypeList{}), ...); } } template diff --git a/base/base/constexpr_helpers.h b/base/base/constexpr_helpers.h index 2dad106a7a3..13bb5d85a56 100644 --- a/base/base/constexpr_helpers.h +++ b/base/base/constexpr_helpers.h @@ -21,7 +21,7 @@ bool func_wrapper(Func && func, Arg && arg) template constexpr bool static_for_impl(Func && f, std::integer_sequence) { - return (func_wrapper(std::forward(f), std::integral_constant{}) || ...); + return (func_wrapper(f, std::integral_constant{}) || ...); } template diff --git a/base/base/find_symbols.h b/base/base/find_symbols.h index fda94edaa88..30ee759ba33 100644 --- a/base/base/find_symbols.h +++ b/base/base/find_symbols.h @@ -147,7 +147,7 @@ constexpr uint16_t maybe_negate(uint16_t x) return ~x; } -enum class ReturnMode +enum class ReturnMode : uint8_t { End, Nullptr, diff --git a/base/base/getMemoryAmount.cpp b/base/base/getMemoryAmount.cpp index 3d01e301f45..f47cba9833d 100644 --- a/base/base/getMemoryAmount.cpp +++ b/base/base/getMemoryAmount.cpp @@ -77,8 +77,7 @@ uint64_t getMemoryAmountOrZero() { uint64_t limit_v1; if (limit_file_v1 >> limit_v1) - if (limit_v1 < memory_amount) - memory_amount = limit_v1; + memory_amount = std::min(memory_amount, limit_v1); } } diff --git a/base/base/hex.h b/base/base/hex.h index 931f220aa08..5e88ce76386 100644 --- a/base/base/hex.h +++ b/base/base/hex.h @@ -146,7 +146,7 @@ namespace impl TUInt res; if constexpr (sizeof(TUInt) == 1) { - res = static_cast(unhexDigit(data[0])) * 0x10 + static_cast(unhexDigit(data[1])); + res = unhexDigit(data[0]) * 0x10 + unhexDigit(data[1]); } else if constexpr (sizeof(TUInt) == 2) { @@ -176,17 +176,19 @@ namespace impl }; /// Helper template class to convert a value of any supported type to hexadecimal representation and back. - template + template struct HexConversion; template - struct HexConversion>> : public HexConversionUInt {}; + requires(std::is_integral_v) + struct HexConversion : public HexConversionUInt {}; template struct HexConversion> : public HexConversionUInt> {}; template /// Partial specialization here allows not to include in this header. - struct HexConversion>> + requires(std::is_same_v) + struct HexConversion { static const constexpr size_t num_hex_digits = 32; diff --git a/base/base/iostream_debug_helpers.h b/base/base/iostream_debug_helpers.h index 5c601251272..b23d3d9794d 100644 --- a/base/base/iostream_debug_helpers.h +++ b/base/base/iostream_debug_helpers.h @@ -20,24 +20,26 @@ Out & dumpValue(Out &, T &&); /// Catch-all case. template -std::enable_if_t & dumpImpl(Out & out, T &&) // NOLINT(cppcoreguidelines-missing-std-forward) +requires(priority == -1) +Out & dumpImpl(Out & out, T &&) // NOLINT(cppcoreguidelines-missing-std-forward) { return out << "{...}"; } /// An object, that could be output with operator <<. template -std::enable_if_t & dumpImpl(Out & out, T && x, std::decay_t() << std::declval())> * = nullptr) // NOLINT(cppcoreguidelines-missing-std-forward) +requires(priority == 0) +Out & dumpImpl(Out & out, T && x, std::decay_t() << std::declval())> * = nullptr) // NOLINT(cppcoreguidelines-missing-std-forward) { return out << x; } /// A pointer-like object. template -std::enable_if_t, std::decay_t())>> - , Out> & dumpImpl(Out & out, T && x, std::decay_t())> * = nullptr) // NOLINT(cppcoreguidelines-missing-std-forward) + && !std::is_same_v, std::decay_t())>>) +Out & dumpImpl(Out & out, T && x, std::decay_t())> * = nullptr) // NOLINT(cppcoreguidelines-missing-std-forward) { if (!x) return out << "nullptr"; @@ -46,7 +48,8 @@ std::enable_if_t -std::enable_if_t & dumpImpl(Out & out, T && x, std::decay_t()))> * = nullptr) // NOLINT(cppcoreguidelines-missing-std-forward) +requires(priority == 2) +Out & dumpImpl(Out & out, T && x, std::decay_t()))> * = nullptr) // NOLINT(cppcoreguidelines-missing-std-forward) { bool first = true; out << "{"; @@ -63,8 +66,8 @@ std::enable_if_t & dumpImpl(Out & out, T && x, std::decay_t< template -std::enable_if_t>, Out> & -dumpImpl(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward) +requires(priority == 3 && std::is_enum_v>) +Out & dumpImpl(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward) { return out << magic_enum::enum_name(x); } @@ -72,8 +75,8 @@ dumpImpl(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward) /// string and const char * - output not as container or pointer. template -std::enable_if_t, std::string> || std::is_same_v, const char *>), Out> & -dumpImpl(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward) +requires(priority == 3 && (std::is_same_v, std::string> || std::is_same_v, const char *>)) +Out & dumpImpl(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward) { return out << std::quoted(x); } @@ -81,8 +84,8 @@ dumpImpl(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward) /// UInt8 - output as number, not char. template -std::enable_if_t, unsigned char>, Out> & -dumpImpl(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward) +requires(priority == 3 && std::is_same_v, unsigned char>) +Out & dumpImpl(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-forward) { return out << int(x); } @@ -108,7 +111,8 @@ Out & dumpTupleImpl(Out & out, T && x) // NOLINT(cppcoreguidelines-missing-std-f } template -std::enable_if_t & dumpImpl(Out & out, T && x, std::decay_t(std::declval()))> * = nullptr) // NOLINT(cppcoreguidelines-missing-std-forward) +requires(priority == 4) +Out & dumpImpl(Out & out, T && x, std::decay_t(std::declval()))> * = nullptr) // NOLINT(cppcoreguidelines-missing-std-forward) { return dumpTupleImpl<0>(out, x); } diff --git a/base/base/itoa.cpp b/base/base/itoa.cpp index fd8fd8de025..9a2d02e3388 100644 --- a/base/base/itoa.cpp +++ b/base/base/itoa.cpp @@ -250,14 +250,16 @@ ALWAYS_INLINE inline char * uitoa, 1>(char * p, UnsignedOfSize //===----------------------------------------------------------===// // itoa: handle unsigned integral operands (selected by SFINAE) -template && std::is_integral_v> * = nullptr> +template +requires(!std::is_signed_v && std::is_integral_v) ALWAYS_INLINE inline char * itoa(U u, char * p) { return convert::uitoa(p, u); } // itoa: handle signed integral operands (selected by SFINAE) -template && std::is_integral_v> * = nullptr> +template +requires(std::is_signed_v && std::is_integral_v) ALWAYS_INLINE inline char * itoa(I i, char * p) { // Need "mask" to be filled with a copy of the sign bit. diff --git a/base/base/map.h b/base/base/map.h index 043d8363619..0de42ebfdf6 100644 --- a/base/base/map.h +++ b/base/base/map.h @@ -19,8 +19,8 @@ auto map(const Collection & collection, Mapper && mapper) using value_type = unqualified_t; return Collection( - boost::make_transform_iterator(std::begin(collection), std::forward(mapper)), - boost::make_transform_iterator(std::end(collection), std::forward(mapper))); + boost::make_transform_iterator(std::begin(collection), mapper), + boost::make_transform_iterator(std::end(collection), mapper)); } /** \brief Returns collection of specified container-type, @@ -33,8 +33,8 @@ auto map(const Collection & collection, Mapper && mapper) using value_type = unqualified_t; return ResultCollection( - boost::make_transform_iterator(std::begin(collection), std::forward(mapper)), - boost::make_transform_iterator(std::end(collection), std::forward(mapper))); + boost::make_transform_iterator(std::begin(collection), mapper), + boost::make_transform_iterator(std::end(collection), mapper)); } /** \brief Returns collection of specified type, @@ -45,8 +45,8 @@ template auto map(const Collection & collection, Mapper && mapper) { return ResultCollection( - boost::make_transform_iterator(std::begin(collection), std::forward(mapper)), - boost::make_transform_iterator(std::end(collection), std::forward(mapper))); + boost::make_transform_iterator(std::begin(collection), mapper), + boost::make_transform_iterator(std::end(collection), mapper)); } } diff --git a/base/base/range.h b/base/base/range.h index aacd7e433a4..c75359a44c3 100644 --- a/base/base/range.h +++ b/base/base/range.h @@ -23,12 +23,10 @@ namespace internal /// For loop adaptor which is used to iterate through a half-closed interval [begin, end). /// The parameters `begin` and `end` can have any integral or enum types. -template || std::is_enum_v) && - (std::is_integral_v || std::is_enum_v) && - (!std::is_enum_v || !std::is_enum_v || std::is_same_v), void>> +template +requires((std::is_integral_v || std::is_enum_v) && + (std::is_integral_v || std::is_enum_v) && + (!std::is_enum_v || !std::is_enum_v || std::is_same_v)) inline auto range(BeginType begin, EndType end) { if constexpr (std::is_integral_v && std::is_integral_v) @@ -50,8 +48,8 @@ inline auto range(BeginType begin, EndType end) /// For loop adaptor which is used to iterate through a half-closed interval [0, end). /// The parameter `end` can have any integral or enum type. /// The same as range(0, end). -template || std::is_enum_v, void>> +template +requires(std::is_integral_v || std::is_enum_v) inline auto range(Type end) { if constexpr (std::is_integral_v) diff --git a/base/base/sleep.cpp b/base/base/sleep.cpp index 9611f8cc40f..312a5a5db0b 100644 --- a/base/base/sleep.cpp +++ b/base/base/sleep.cpp @@ -2,6 +2,7 @@ #include #include +#include #if defined(OS_DARWIN) #include @@ -34,7 +35,8 @@ void sleepForNanoseconds(uint64_t nanoseconds) constexpr auto clock_type = CLOCK_MONOTONIC; struct timespec current_time; - clock_gettime(clock_type, ¤t_time); + if (0 != clock_gettime(clock_type, ¤t_time)) + throw std::system_error(std::error_code(errno, std::system_category())); constexpr uint64_t resolution = 1'000'000'000; struct timespec finish_time = current_time; diff --git a/base/base/wide_integer.h b/base/base/wide_integer.h index ffd30460c03..f3a4dc9e6d5 100644 --- a/base/base/wide_integer.h +++ b/base/base/wide_integer.h @@ -111,7 +111,8 @@ public: constexpr explicit operator bool() const noexcept; - template , T>> + template + requires(std::is_arithmetic_v) constexpr operator T() const noexcept; constexpr operator long double() const noexcept; @@ -208,12 +209,14 @@ constexpr integer operator<<(const integer & lhs, in template constexpr integer operator>>(const integer & lhs, int n) noexcept; -template >> +template +requires(!std::is_same_v) constexpr integer operator<<(const integer & lhs, Int n) noexcept { return lhs << int(n); } -template >> +template +requires(!std::is_same_v) constexpr integer operator>>(const integer & lhs, Int n) noexcept { return lhs >> int(n); @@ -262,4 +265,3 @@ struct hash>; // NOLINTEND(*) #include "wide_integer_impl.h" - diff --git a/base/base/wide_integer_impl.h b/base/base/wide_integer_impl.h index 0e98b6e5ee6..3787971a20e 100644 --- a/base/base/wide_integer_impl.h +++ b/base/base/wide_integer_impl.h @@ -1246,7 +1246,8 @@ constexpr integer::operator bool() const noexcept } template -template +template +requires(std::is_arithmetic_v) constexpr integer::operator T() const noexcept { static_assert(std::numeric_limits::is_integer); diff --git a/cmake/clang_tidy.cmake b/cmake/clang_tidy.cmake index 4323c20463a..4c9331f6283 100644 --- a/cmake/clang_tidy.cmake +++ b/cmake/clang_tidy.cmake @@ -5,14 +5,14 @@ if (ENABLE_CLANG_TIDY) find_program (CLANG_TIDY_CACHE_PATH NAMES "clang-tidy-cache") if (CLANG_TIDY_CACHE_PATH) - find_program (_CLANG_TIDY_PATH NAMES "clang-tidy-17" "clang-tidy-16" "clang-tidy") + find_program (_CLANG_TIDY_PATH NAMES "clang-tidy-18" "clang-tidy-17" "clang-tidy-16" "clang-tidy") # Why do we use ';' here? # It's a cmake black magic: https://cmake.org/cmake/help/latest/prop_tgt/LANG_CLANG_TIDY.html#prop_tgt:%3CLANG%3E_CLANG_TIDY # The CLANG_TIDY_PATH is passed to CMAKE_CXX_CLANG_TIDY, which follows CXX_CLANG_TIDY syntax. set (CLANG_TIDY_PATH "${CLANG_TIDY_CACHE_PATH};${_CLANG_TIDY_PATH}" CACHE STRING "A combined command to run clang-tidy with caching wrapper") else () - find_program (CLANG_TIDY_PATH NAMES "clang-tidy-17" "clang-tidy-16" "clang-tidy") + find_program (CLANG_TIDY_PATH NAMES "clang-tidy-18" "clang-tidy-17" "clang-tidy-16" "clang-tidy") endif () if (CLANG_TIDY_PATH) diff --git a/cmake/linux/default_libs.cmake b/cmake/linux/default_libs.cmake index e5ca8e296fc..4a06243243e 100644 --- a/cmake/linux/default_libs.cmake +++ b/cmake/linux/default_libs.cmake @@ -5,17 +5,15 @@ set (DEFAULT_LIBS "-nodefaultlibs") # We need builtins from Clang's RT even without libcxx - for ubsan+int128. # See https://bugs.llvm.org/show_bug.cgi?id=16404 -if (COMPILER_CLANG) - execute_process (COMMAND ${CMAKE_CXX_COMPILER} --target=${CMAKE_CXX_COMPILER_TARGET} --print-libgcc-file-name --rtlib=compiler-rt OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE) +execute_process (COMMAND ${CMAKE_CXX_COMPILER} --target=${CMAKE_CXX_COMPILER_TARGET} --print-libgcc-file-name --rtlib=compiler-rt OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE) - # Apparently, in clang-19, the UBSan support library for C++ was moved out into ubsan_standalone_cxx.a, so we have to include both. - if (SANITIZE STREQUAL undefined) - string(REPLACE "builtins.a" "ubsan_standalone_cxx.a" EXTRA_BUILTINS_LIBRARY "${BUILTINS_LIBRARY}") - endif () +# Apparently, in clang-19, the UBSan support library for C++ was moved out into ubsan_standalone_cxx.a, so we have to include both. +if (SANITIZE STREQUAL undefined) + string(REPLACE "builtins.a" "ubsan_standalone_cxx.a" EXTRA_BUILTINS_LIBRARY "${BUILTINS_LIBRARY}") +endif () - if (NOT EXISTS "${BUILTINS_LIBRARY}") - set (BUILTINS_LIBRARY "-lgcc") - endif () +if (NOT EXISTS "${BUILTINS_LIBRARY}") + set (BUILTINS_LIBRARY "-lgcc") endif () if (OS_ANDROID) diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake index a3523203912..08716c1196b 100644 --- a/cmake/sanitize.cmake +++ b/cmake/sanitize.cmake @@ -26,9 +26,7 @@ if (SANITIZE) elseif (SANITIZE STREQUAL "thread") set (TSAN_FLAGS "-fsanitize=thread") - if (COMPILER_CLANG) - set (TSAN_FLAGS "${TSAN_FLAGS} -fsanitize-ignorelist=${PROJECT_SOURCE_DIR}/tests/tsan_ignorelist.txt") - endif() + set (TSAN_FLAGS "${TSAN_FLAGS} -fsanitize-ignorelist=${PROJECT_SOURCE_DIR}/tests/tsan_ignorelist.txt") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${TSAN_FLAGS}") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${TSAN_FLAGS}") @@ -44,9 +42,7 @@ if (SANITIZE) # that's why we often receive reports about UIO. The simplest way to avoid this is just set this flag here. set(UBSAN_FLAGS "${UBSAN_FLAGS} -fno-sanitize=unsigned-integer-overflow") endif() - if (COMPILER_CLANG) - set (UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize-ignorelist=${PROJECT_SOURCE_DIR}/tests/ubsan_ignorelist.txt") - endif() + set (UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize-ignorelist=${PROJECT_SOURCE_DIR}/tests/ubsan_ignorelist.txt") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${UBSAN_FLAGS}") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${UBSAN_FLAGS}") diff --git a/cmake/tools.cmake b/cmake/tools.cmake index 1ba3007b0f3..7aa5d4c51ce 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -1,10 +1,6 @@ # Compiler -if (CMAKE_CXX_COMPILER_ID MATCHES "AppleClang") - set (COMPILER_CLANG 1) # Safe to treat AppleClang as a regular Clang, in general. -elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") - set (COMPILER_CLANG 1) -else () +if (NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang") message (FATAL_ERROR "Compiler ${CMAKE_CXX_COMPILER_ID} is not supported") endif () @@ -13,34 +9,30 @@ execute_process(COMMAND ${CMAKE_CXX_COMPILER} --version OUTPUT_VARIABLE COMPILER message (STATUS "Using compiler:\n${COMPILER_SELF_IDENTIFICATION}") # Require minimum compiler versions -set (CLANG_MINIMUM_VERSION 16) +set (CLANG_MINIMUM_VERSION 17) set (XCODE_MINIMUM_VERSION 12.0) set (APPLE_CLANG_MINIMUM_VERSION 12.0.0) -if (COMPILER_CLANG) - if (CMAKE_CXX_COMPILER_ID MATCHES "AppleClang") - # (Experimental!) Specify "-DALLOW_APPLECLANG=ON" when running CMake configuration step, if you want to experiment with using it. - if (NOT ALLOW_APPLECLANG AND NOT DEFINED ENV{ALLOW_APPLECLANG}) - message (FATAL_ERROR "Compilation with AppleClang is unsupported. Please use vanilla Clang, e.g. from Homebrew.") - endif () +if (CMAKE_CXX_COMPILER_ID MATCHES "AppleClang") + # (Experimental!) Specify "-DALLOW_APPLECLANG=ON" when running CMake configuration step, if you want to experiment with using it. + if (NOT ALLOW_APPLECLANG AND NOT DEFINED ENV{ALLOW_APPLECLANG}) + message (FATAL_ERROR "Compilation with AppleClang is unsupported. Please use vanilla Clang, e.g. from Homebrew.") + endif () - # For a mapping between XCode / AppleClang / vanilla Clang versions, see https://en.wikipedia.org/wiki/Xcode - if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${APPLE_CLANG_MINIMUM_VERSION}) - message (FATAL_ERROR "Compilation with AppleClang version ${CMAKE_CXX_COMPILER_VERSION} is unsupported, the minimum required version is ${APPLE_CLANG_MINIMUM_VERSION} (Xcode ${XCODE_MINIMUM_VERSION}).") - endif () - else () - if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION}) - message (FATAL_ERROR "Compilation with Clang version ${CMAKE_CXX_COMPILER_VERSION} is unsupported, the minimum required version is ${CLANG_MINIMUM_VERSION}.") - endif () + # For a mapping between XCode / AppleClang / vanilla Clang versions, see https://en.wikipedia.org/wiki/Xcode + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${APPLE_CLANG_MINIMUM_VERSION}) + message (FATAL_ERROR "Compilation with AppleClang version ${CMAKE_CXX_COMPILER_VERSION} is unsupported, the minimum required version is ${APPLE_CLANG_MINIMUM_VERSION} (Xcode ${XCODE_MINIMUM_VERSION}).") + endif () +else () + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION}) + message (FATAL_ERROR "Compilation with Clang version ${CMAKE_CXX_COMPILER_VERSION} is unsupported, the minimum required version is ${CLANG_MINIMUM_VERSION}.") endif () endif () -# Linker - string (REGEX MATCHALL "[0-9]+" COMPILER_VERSION_LIST ${CMAKE_CXX_COMPILER_VERSION}) list (GET COMPILER_VERSION_LIST 0 COMPILER_VERSION_MAJOR) -# Example values: `lld-10` +# Linker option (LINKER_NAME "Linker name or full path") if (LINKER_NAME MATCHES "gold") @@ -48,19 +40,15 @@ if (LINKER_NAME MATCHES "gold") endif () if (NOT LINKER_NAME) - if (COMPILER_CLANG) - if (OS_LINUX AND NOT ARCH_S390X) - find_program (LLD_PATH NAMES "ld.lld-${COMPILER_VERSION_MAJOR}" "ld.lld") - elseif (OS_DARWIN) - find_program (LLD_PATH NAMES "ld") - endif () + if (OS_LINUX AND NOT ARCH_S390X) + find_program (LLD_PATH NAMES "ld.lld-${COMPILER_VERSION_MAJOR}" "ld.lld") + elseif (OS_DARWIN) + find_program (LLD_PATH NAMES "ld") endif () if (LLD_PATH) if (OS_LINUX OR OS_DARWIN) - if (COMPILER_CLANG) - # Clang driver simply allows full linker path. - set (LINKER_NAME ${LLD_PATH}) - endif () + # Clang driver simply allows full linker path. + set (LINKER_NAME ${LLD_PATH}) endif () endif() endif() @@ -82,47 +70,28 @@ else () endif () # Archiver - -if (COMPILER_CLANG) - find_program (LLVM_AR_PATH NAMES "llvm-ar-${COMPILER_VERSION_MAJOR}" "llvm-ar") -endif () - +find_program (LLVM_AR_PATH NAMES "llvm-ar-${COMPILER_VERSION_MAJOR}" "llvm-ar") if (LLVM_AR_PATH) set (CMAKE_AR "${LLVM_AR_PATH}") endif () - message(STATUS "Using archiver: ${CMAKE_AR}") # Ranlib - -if (COMPILER_CLANG) - find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib-${COMPILER_VERSION_MAJOR}" "llvm-ranlib") -endif () - +find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib-${COMPILER_VERSION_MAJOR}" "llvm-ranlib") if (LLVM_RANLIB_PATH) set (CMAKE_RANLIB "${LLVM_RANLIB_PATH}") endif () - message(STATUS "Using ranlib: ${CMAKE_RANLIB}") # Install Name Tool - -if (COMPILER_CLANG) - find_program (LLVM_INSTALL_NAME_TOOL_PATH NAMES "llvm-install-name-tool-${COMPILER_VERSION_MAJOR}" "llvm-install-name-tool") -endif () - +find_program (LLVM_INSTALL_NAME_TOOL_PATH NAMES "llvm-install-name-tool-${COMPILER_VERSION_MAJOR}" "llvm-install-name-tool") if (LLVM_INSTALL_NAME_TOOL_PATH) set (CMAKE_INSTALL_NAME_TOOL "${LLVM_INSTALL_NAME_TOOL_PATH}") endif () - message(STATUS "Using install-name-tool: ${CMAKE_INSTALL_NAME_TOOL}") # Objcopy - -if (COMPILER_CLANG) - find_program (OBJCOPY_PATH NAMES "llvm-objcopy-${COMPILER_VERSION_MAJOR}" "llvm-objcopy" "objcopy") -endif () - +find_program (OBJCOPY_PATH NAMES "llvm-objcopy-${COMPILER_VERSION_MAJOR}" "llvm-objcopy" "objcopy") if (OBJCOPY_PATH) message (STATUS "Using objcopy: ${OBJCOPY_PATH}") else () @@ -130,11 +99,7 @@ else () endif () # Strip - -if (COMPILER_CLANG) - find_program (STRIP_PATH NAMES "llvm-strip-${COMPILER_VERSION_MAJOR}" "llvm-strip" "strip") -endif () - +find_program (STRIP_PATH NAMES "llvm-strip-${COMPILER_VERSION_MAJOR}" "llvm-strip" "strip") if (STRIP_PATH) message (STATUS "Using strip: ${STRIP_PATH}") else () diff --git a/cmake/warnings.cmake b/cmake/warnings.cmake index 455e4f09939..807d92d9077 100644 --- a/cmake/warnings.cmake +++ b/cmake/warnings.cmake @@ -15,37 +15,35 @@ if ((NOT CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") AND (NOT SANITIZE) AND (NOT CMAKE add_warning(frame-larger-than=65536) endif () -if (COMPILER_CLANG) - # Add some warnings that are not available even with -Wall -Wextra -Wpedantic. - # We want to get everything out of the compiler for code quality. - add_warning(everything) - add_warning(pedantic) - no_warning(zero-length-array) - no_warning(c++98-compat-pedantic) - no_warning(c++98-compat) - no_warning(c++20-compat) # Use constinit in C++20 without warnings - no_warning(sign-conversion) - no_warning(implicit-int-conversion) - no_warning(implicit-int-float-conversion) - no_warning(ctad-maybe-unsupported) # clang 9+, linux-only - no_warning(disabled-macro-expansion) - no_warning(documentation-unknown-command) - no_warning(double-promotion) - no_warning(exit-time-destructors) - no_warning(float-equal) - no_warning(global-constructors) - no_warning(missing-prototypes) - no_warning(missing-variable-declarations) - no_warning(padded) - no_warning(switch-enum) - no_warning(undefined-func-template) - no_warning(unused-template) - no_warning(vla) - no_warning(weak-template-vtables) - no_warning(weak-vtables) - no_warning(thread-safety-negative) # experimental flag, too many false positives - no_warning(enum-constexpr-conversion) # breaks magic-enum library in clang-16 - no_warning(unsafe-buffer-usage) # too aggressive - no_warning(switch-default) # conflicts with "defaults in a switch covering all enum values" - # TODO Enable conversion, sign-conversion, double-promotion warnings. -endif () +# Add some warnings that are not available even with -Wall -Wextra -Wpedantic. +# We want to get everything out of the compiler for code quality. +add_warning(everything) +add_warning(pedantic) +no_warning(zero-length-array) +no_warning(c++98-compat-pedantic) +no_warning(c++98-compat) +no_warning(c++20-compat) # Use constinit in C++20 without warnings +no_warning(sign-conversion) +no_warning(implicit-int-conversion) +no_warning(implicit-int-float-conversion) +no_warning(ctad-maybe-unsupported) # clang 9+, linux-only +no_warning(disabled-macro-expansion) +no_warning(documentation-unknown-command) +no_warning(double-promotion) +no_warning(exit-time-destructors) +no_warning(float-equal) +no_warning(global-constructors) +no_warning(missing-prototypes) +no_warning(missing-variable-declarations) +no_warning(padded) +no_warning(switch-enum) +no_warning(undefined-func-template) +no_warning(unused-template) +no_warning(vla) +no_warning(weak-template-vtables) +no_warning(weak-vtables) +no_warning(thread-safety-negative) # experimental flag, too many false positives +no_warning(enum-constexpr-conversion) # breaks magic-enum library in clang-16 +no_warning(unsafe-buffer-usage) # too aggressive +no_warning(switch-default) # conflicts with "defaults in a switch covering all enum values" +# TODO Enable conversion, sign-conversion, double-promotion warnings. diff --git a/contrib/abseil-cpp-cmake/CMakeLists.txt b/contrib/abseil-cpp-cmake/CMakeLists.txt index e6c3268c57a..7372195bb0d 100644 --- a/contrib/abseil-cpp-cmake/CMakeLists.txt +++ b/contrib/abseil-cpp-cmake/CMakeLists.txt @@ -52,7 +52,7 @@ function(absl_cc_library) ) target_include_directories(${_NAME} - PUBLIC "${ABSL_COMMON_INCLUDE_DIRS}") + SYSTEM PUBLIC "${ABSL_COMMON_INCLUDE_DIRS}") target_compile_options(${_NAME} PRIVATE ${ABSL_CC_LIB_COPTS}) target_compile_definitions(${_NAME} PUBLIC ${ABSL_CC_LIB_DEFINES}) @@ -61,7 +61,7 @@ function(absl_cc_library) # Generating header-only library add_library(${_NAME} INTERFACE) target_include_directories(${_NAME} - INTERFACE "${ABSL_COMMON_INCLUDE_DIRS}") + SYSTEM INTERFACE "${ABSL_COMMON_INCLUDE_DIRS}") target_link_libraries(${_NAME} INTERFACE diff --git a/contrib/capnproto-cmake/CMakeLists.txt b/contrib/capnproto-cmake/CMakeLists.txt index e76268592ee..c07e9e6925b 100644 --- a/contrib/capnproto-cmake/CMakeLists.txt +++ b/contrib/capnproto-cmake/CMakeLists.txt @@ -81,9 +81,7 @@ set (CAPNPC_SRCS add_library(_capnpc ${CAPNPC_SRCS}) target_link_libraries(_capnpc PUBLIC _capnp) -if (COMPILER_CLANG) - set (CAPNP_PRIVATE_CXX_FLAGS -fno-char8_t) -endif () +set (CAPNP_PRIVATE_CXX_FLAGS -fno-char8_t) target_compile_options(_kj PRIVATE ${CAPNP_PRIVATE_CXX_FLAGS}) target_compile_options(_capnp PRIVATE ${CAPNP_PRIVATE_CXX_FLAGS}) diff --git a/contrib/lz4 b/contrib/lz4 index ce45a9dbdb0..145f3804ca5 160000 --- a/contrib/lz4 +++ b/contrib/lz4 @@ -1 +1 @@ -Subproject commit ce45a9dbdb059511a3e9576b19db3e7f1a4f172e +Subproject commit 145f3804ca5ef5482cda0f2a4f6a2d04ba57f965 diff --git a/contrib/openssl-cmake/CMakeLists.txt b/contrib/openssl-cmake/CMakeLists.txt index 021c88bcb04..72846143b9e 100644 --- a/contrib/openssl-cmake/CMakeLists.txt +++ b/contrib/openssl-cmake/CMakeLists.txt @@ -91,12 +91,10 @@ set(LIB_SOVERSION ${VERSION_MAJOR}) enable_language(ASM) -if(COMPILER_CLANG) - add_definitions(-Wno-unused-command-line-argument) - # Note that s390x build uses mold linker - if(NOT ARCH_S390X) - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld") # only relevant for -DENABLE_OPENSSL_DYNAMIC=1 - endif() +add_definitions(-Wno-unused-command-line-argument) +# Note that s390x build uses mold linker +if(NOT ARCH_S390X) + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld") # only relevant for -DENABLE_OPENSSL_DYNAMIC=1 endif() if(ARCH_AMD64) diff --git a/contrib/sentry-native-cmake/CMakeLists.txt b/contrib/sentry-native-cmake/CMakeLists.txt index 6364e75db28..6e4c8c36081 100644 --- a/contrib/sentry-native-cmake/CMakeLists.txt +++ b/contrib/sentry-native-cmake/CMakeLists.txt @@ -1,4 +1,4 @@ -if (NOT OS_FREEBSD AND NOT (OS_DARWIN AND COMPILER_CLANG)) +if (NOT OS_FREEBSD AND NOT OS_DARWIN) option (ENABLE_SENTRY "Enable Sentry" ${ENABLE_LIBRARIES}) else() option (ENABLE_SENTRY "Enable Sentry" OFF) diff --git a/contrib/yaml-cpp b/contrib/yaml-cpp index 0c86adac6d1..f91e9383412 160000 --- a/contrib/yaml-cpp +++ b/contrib/yaml-cpp @@ -1 +1 @@ -Subproject commit 0c86adac6d117ee2b4afcedb8ade19036ca0327d +Subproject commit f91e938341273b5f9d341380ab17bcc3de5daa06 diff --git a/docker/packager/README.md b/docker/packager/README.md index 3604e8585a4..12947aed62f 100644 --- a/docker/packager/README.md +++ b/docker/packager/README.md @@ -3,10 +3,10 @@ compilers and build settings. Correctly configured Docker daemon is single depen Usage: -Build deb package with `clang-17` in `debug` mode: +Build deb package with `clang-18` in `debug` mode: ``` $ mkdir deb/test_output -$ ./packager --output-dir deb/test_output/ --package-type deb --compiler=clang-17 --debug-build +$ ./packager --output-dir deb/test_output/ --package-type deb --compiler=clang-18 --debug-build $ ls -l deb/test_output -rw-r--r-- 1 root root 3730 clickhouse-client_22.2.2+debug_all.deb -rw-r--r-- 1 root root 84221888 clickhouse-common-static_22.2.2+debug_amd64.deb @@ -17,11 +17,11 @@ $ ls -l deb/test_output ``` -Build ClickHouse binary with `clang-17` and `address` sanitizer in `relwithdebuginfo` +Build ClickHouse binary with `clang-18` and `address` sanitizer in `relwithdebuginfo` mode: ``` $ mkdir $HOME/some_clickhouse -$ ./packager --output-dir=$HOME/some_clickhouse --package-type binary --compiler=clang-17 --sanitizer=address +$ ./packager --output-dir=$HOME/some_clickhouse --package-type binary --compiler=clang-18 --sanitizer=address $ ls -l $HOME/some_clickhouse -rwxr-xr-x 1 root root 787061952 clickhouse lrwxrwxrwx 1 root root 10 clickhouse-benchmark -> clickhouse diff --git a/docker/packager/packager b/docker/packager/packager index 23fc26bc1a4..f11cd30078b 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -403,19 +403,19 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--compiler", choices=( - "clang-17", - "clang-17-darwin", - "clang-17-darwin-aarch64", - "clang-17-aarch64", - "clang-17-aarch64-v80compat", - "clang-17-ppc64le", - "clang-17-riscv64", - "clang-17-s390x", - "clang-17-amd64-compat", - "clang-17-amd64-musl", - "clang-17-freebsd", + "clang-18", + "clang-18-darwin", + "clang-18-darwin-aarch64", + "clang-18-aarch64", + "clang-18-aarch64-v80compat", + "clang-18-ppc64le", + "clang-18-riscv64", + "clang-18-s390x", + "clang-18-amd64-compat", + "clang-18-amd64-musl", + "clang-18-freebsd", ), - default="clang-17", + default="clang-18", help="a compiler to use", ) parser.add_argument( diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 7292163023d..d82be0e63f6 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -1,11 +1,14 @@ FROM ubuntu:20.04 # see https://github.com/moby/moby/issues/4032#issuecomment-192327844 +# It could be removed after we move on a version 23:04+ ARG DEBIAN_FRONTEND=noninteractive # ARG for quick switch to a given ubuntu mirror ARG apt_archive="http://archive.ubuntu.com" +# We shouldn't use `apt upgrade` to not change the upstream image. It's updated biweekly + # user/group precreated explicitly with fixed uid/gid on purpose. # It is especially important for rootless containers: in that case entrypoint # can't do chown and owners of mounted volumes should be configured externally. @@ -16,13 +19,11 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list && groupadd -r clickhouse --gid=101 \ && useradd -r -g clickhouse --uid=101 --home-dir=/var/lib/clickhouse --shell=/bin/bash clickhouse \ && apt-get update \ - && apt-get upgrade -yq \ && apt-get install --yes --no-install-recommends \ ca-certificates \ locales \ tzdata \ wget \ - && apt-get clean \ && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* ARG REPO_CHANNEL="stable" @@ -30,6 +31,9 @@ ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https ARG VERSION="24.4.1.2088" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" +#docker-official-library:off +# The part between `docker-official-library` tags is related to our builds + # set non-empty deb_location_url url to create a docker image # from debs created by CI build, for example: # docker build . --network host --build-arg version="21.4.1.6282" --build-arg deb_location_url="https://..." -t ... @@ -80,19 +84,22 @@ RUN if [ -n "${single_binary_location_url}" ]; then \ && rm -rf /tmp/* ; \ fi +# The rest is the same in the official docker and in our build system +#docker-official-library:on + # A fallback to installation from ClickHouse repository RUN if ! clickhouse local -q "SELECT ''" > /dev/null 2>&1; then \ apt-get update \ && apt-get install --yes --no-install-recommends \ apt-transport-https \ - ca-certificates \ dirmngr \ gnupg2 \ && mkdir -p /etc/apt/sources.list.d \ && GNUPGHOME=$(mktemp -d) \ - && GNUPGHOME="$GNUPGHOME" gpg --no-default-keyring \ + && GNUPGHOME="$GNUPGHOME" gpg --batch --no-default-keyring \ --keyring /usr/share/keyrings/clickhouse-keyring.gpg \ - --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 8919F6BD2B48D754 \ + --keyserver hkp://keyserver.ubuntu.com:80 \ + --recv-keys 3a9ea1193a97b548be1457d48919f6bd2b48d754 \ && rm -rf "$GNUPGHOME" \ && chmod +r /usr/share/keyrings/clickhouse-keyring.gpg \ && echo "${REPOSITORY}" > /etc/apt/sources.list.d/clickhouse.list \ @@ -127,7 +134,6 @@ RUN mkdir /docker-entrypoint-initdb.d COPY docker_related_config.xml /etc/clickhouse-server/config.d/ COPY entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh EXPOSE 9000 8123 9009 VOLUME /var/lib/clickhouse diff --git a/docker/server/README.md b/docker/server/README.md index d6cf2dfdf09..65239126790 100644 --- a/docker/server/README.md +++ b/docker/server/README.md @@ -4,33 +4,34 @@ ClickHouse is an open-source column-oriented DBMS (columnar database management system) for online analytical processing (OLAP) that allows users to generate analytical reports using SQL queries in real-time. -ClickHouse works 100-1000x faster than traditional database management systems, and processes hundreds of millions to over a billion rows and tens of gigabytes of data per server per second. With a widespread user base around the globe, the technology has received praise for its reliability, ease of use, and fault tolerance. +ClickHouse works 100-1000x faster than traditional database management systems, and processes hundreds of millions to over a billion rows and tens of gigabytes of data per server per second. With a widespread user base around the globe, the technology has received praise for its reliability, ease of use, and fault tolerance. For more information and documentation see https://clickhouse.com/. ## Versions -- The `latest` tag points to the latest release of the latest stable branch. -- Branch tags like `22.2` point to the latest release of the corresponding branch. -- Full version tags like `22.2.3.5` point to the corresponding release. -- The tag `head` is built from the latest commit to the default branch. -- Each tag has optional `-alpine` suffix to reflect that it's built on top of `alpine`. +- The `latest` tag points to the latest release of the latest stable branch. +- Branch tags like `22.2` point to the latest release of the corresponding branch. +- Full version tags like `22.2.3.5` point to the corresponding release. +- The tag `head` is built from the latest commit to the default branch. +- Each tag has optional `-alpine` suffix to reflect that it's built on top of `alpine`. ### Compatibility -- The amd64 image requires support for [SSE3 instructions](https://en.wikipedia.org/wiki/SSE3). Virtually all x86 CPUs after 2005 support SSE3. -- The arm64 image requires support for the [ARMv8.2-A architecture](https://en.wikipedia.org/wiki/AArch64#ARMv8.2-A) and additionally the Load-Acquire RCpc register. The register is optional in version ARMv8.2-A and mandatory in [ARMv8.3-A](https://en.wikipedia.org/wiki/AArch64#ARMv8.3-A). Supported in Graviton >=2, Azure and GCP instances. Examples for unsupported devices are Raspberry Pi 4 (ARMv8.0-A) and Jetson AGX Xavier/Orin (ARMv8.2-A). +- The amd64 image requires support for [SSE3 instructions](https://en.wikipedia.org/wiki/SSE3). Virtually all x86 CPUs after 2005 support SSE3. +- The arm64 image requires support for the [ARMv8.2-A architecture](https://en.wikipedia.org/wiki/AArch64#ARMv8.2-A) and additionally the Load-Acquire RCpc register. The register is optional in version ARMv8.2-A and mandatory in [ARMv8.3-A](https://en.wikipedia.org/wiki/AArch64#ARMv8.3-A). Supported in Graviton >=2, Azure and GCP instances. Examples for unsupported devices are Raspberry Pi 4 (ARMv8.0-A) and Jetson AGX Xavier/Orin (ARMv8.2-A). ## How to use this image ### start server instance + ```bash docker run -d --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server ``` By default, ClickHouse will be accessible only via the Docker network. See the [networking section below](#networking). -By default, starting above server instance will be run as the `default` user without password. +By default, starting above server instance will be run as the `default` user without password. ### connect to it from a native client @@ -66,9 +67,7 @@ docker run -d -p 18123:8123 -p19000:9000 --name some-clickhouse-server --ulimit echo 'SELECT version()' | curl 'http://localhost:18123/' --data-binary @- ``` -``` -22.6.3.35 -``` +`22.6.3.35` or by allowing the container to use [host ports directly](https://docs.docker.com/network/host/) using `--network=host` (also allows achieving better network performance): @@ -77,16 +76,14 @@ docker run -d --network=host --name some-clickhouse-server --ulimit nofile=26214 echo 'SELECT version()' | curl 'http://localhost:8123/' --data-binary @- ``` -``` -22.6.3.35 -``` +`22.6.3.35` ### Volumes Typically you may want to mount the following folders inside your container to achieve persistency: -* `/var/lib/clickhouse/` - main folder where ClickHouse stores the data -* `/var/log/clickhouse-server/` - logs +- `/var/lib/clickhouse/` - main folder where ClickHouse stores the data +- `/var/log/clickhouse-server/` - logs ```bash docker run -d \ @@ -97,9 +94,9 @@ docker run -d \ You may also want to mount: -* `/etc/clickhouse-server/config.d/*.xml` - files with server configuration adjustments -* `/etc/clickhouse-server/users.d/*.xml` - files with user settings adjustments -* `/docker-entrypoint-initdb.d/` - folder with database initialization scripts (see below). +- `/etc/clickhouse-server/config.d/*.xml` - files with server configuration adjustments +- `/etc/clickhouse-server/users.d/*.xml` - files with user settings adjustments +- `/docker-entrypoint-initdb.d/` - folder with database initialization scripts (see below). ### Linux capabilities @@ -150,7 +147,7 @@ docker run --rm -e CLICKHOUSE_DB=my_database -e CLICKHOUSE_USER=username -e CLIC ## How to extend this image -To perform additional initialization in an image derived from this one, add one or more `*.sql`, `*.sql.gz`, or `*.sh` scripts under `/docker-entrypoint-initdb.d`. After the entrypoint calls `initdb`, it will run any `*.sql` files, run any executable `*.sh` scripts, and source any non-executable `*.sh` scripts found in that directory to do further initialization before starting the service. +To perform additional initialization in an image derived from this one, add one or more `*.sql`, `*.sql.gz`, or `*.sh` scripts under `/docker-entrypoint-initdb.d`. After the entrypoint calls `initdb`, it will run any `*.sql` files, run any executable `*.sh` scripts, and source any non-executable `*.sh` scripts found in that directory to do further initialization before starting the service. Also, you can provide environment variables `CLICKHOUSE_USER` & `CLICKHOUSE_PASSWORD` that will be used for clickhouse-client during initialization. For example, to add an additional user and database, add the following to `/docker-entrypoint-initdb.d/init-db.sh`: diff --git a/docker/test/base/setup_export_logs.sh b/docker/test/base/setup_export_logs.sh index 8858e12c50e..0c869a95db2 100755 --- a/docker/test/base/setup_export_logs.sh +++ b/docker/test/base/setup_export_logs.sh @@ -127,9 +127,6 @@ function setup_logs_replication echo 'Create all configured system logs' clickhouse-client --query "SYSTEM FLUSH LOGS" - # It's doesn't make sense to try creating tables if SYNC fails - echo "SYSTEM SYNC DATABASE REPLICA default" | clickhouse-client "${CONNECTION_ARGS[@]}" || return 0 - debug_or_sanitizer_build=$(clickhouse-client -q "WITH ((SELECT value FROM system.build_options WHERE name='BUILD_TYPE') AS build, (SELECT value FROM system.build_options WHERE name='CXX_FLAGS') as flags) SELECT build='Debug' OR flags LIKE '%fsanitize%'") echo "Build is debug or sanitizer: $debug_or_sanitizer_build" @@ -143,7 +140,7 @@ function setup_logs_replication time DateTime COMMENT 'The time of test run', test_name String COMMENT 'The name of the test', coverage Array(UInt64) COMMENT 'An array of addresses of the code (a subset of addresses instrumented for coverage) that were encountered during the test run' - ) ENGINE = Null COMMENT 'Contains information about per-test coverage from the CI, but used only for exporting to the CI cluster' + ) ENGINE = MergeTree ORDER BY test_name COMMENT 'Contains information about per-test coverage from the CI, but used only for exporting to the CI cluster' " # For each system log table: diff --git a/docker/test/fuzzer/query-fuzzer-tweaks-users.xml b/docker/test/fuzzer/query-fuzzer-tweaks-users.xml index c31d2fd7f39..ad261be1abe 100644 --- a/docker/test/fuzzer/query-fuzzer-tweaks-users.xml +++ b/docker/test/fuzzer/query-fuzzer-tweaks-users.xml @@ -31,6 +31,11 @@ + + + + + diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 76661a5b51c..d1ccdea304b 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -17,7 +17,7 @@ stage=${stage:-} script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" echo "$script_dir" repo_dir=ch -BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-17_debug_none_unsplitted_disable_False_binary"} +BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-18_debug_none_unsplitted_disable_False_binary"} BINARY_URL_TO_DOWNLOAD=${BINARY_URL_TO_DOWNLOAD:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/$BINARY_TO_DOWNLOAD/clickhouse"} function git_clone_with_retry diff --git a/docker/test/integration/runner/Dockerfile b/docker/test/integration/runner/Dockerfile index 8297a7100d1..23d8a37d822 100644 --- a/docker/test/integration/runner/Dockerfile +++ b/docker/test/integration/runner/Dockerfile @@ -101,7 +101,8 @@ RUN python3 -m pip install --no-cache-dir \ retry==0.9.2 \ bs4==0.0.2 \ lxml==5.1.0 \ - urllib3==2.0.7 + urllib3==2.0.7 \ + jwcrypto==1.5.6 # bs4, lxml are for cloud tests, do not delete # Hudi supports only spark 3.3.*, not 3.4 diff --git a/docker/test/keeper-jepsen/run.sh b/docker/test/keeper-jepsen/run.sh index 576a0f0ef8e..444f3cd0de7 100644 --- a/docker/test/keeper-jepsen/run.sh +++ b/docker/test/keeper-jepsen/run.sh @@ -2,7 +2,7 @@ set -euo pipefail -CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-17_relwithdebuginfo_none_unsplitted_disable_False_binary/clickhouse"} +CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-18_relwithdebuginfo_none_unsplitted_disable_False_binary/clickhouse"} CLICKHOUSE_REPO_PATH=${CLICKHOUSE_REPO_PATH:=""} diff --git a/docker/test/server-jepsen/run.sh b/docker/test/server-jepsen/run.sh index 09198ca1968..0d3372b43be 100644 --- a/docker/test/server-jepsen/run.sh +++ b/docker/test/server-jepsen/run.sh @@ -2,7 +2,7 @@ set -euo pipefail -CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-17_relwithdebuginfo_none_unsplitted_disable_False_binary/clickhouse"} +CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-18_relwithdebuginfo_none_unsplitted_disable_False_binary/clickhouse"} CLICKHOUSE_REPO_PATH=${CLICKHOUSE_REPO_PATH:=""} diff --git a/docker/test/sqltest/run.sh b/docker/test/sqltest/run.sh index 1d939805c7b..7edc1341d7d 100755 --- a/docker/test/sqltest/run.sh +++ b/docker/test/sqltest/run.sh @@ -6,7 +6,7 @@ set -e set -u set -o pipefail -BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-17_debug_none_unsplitted_disable_False_binary"} +BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-18_debug_none_unsplitted_disable_False_binary"} BINARY_URL_TO_DOWNLOAD=${BINARY_URL_TO_DOWNLOAD:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/$BINARY_TO_DOWNLOAD/clickhouse"} function wget_with_retry diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index f02a628d13e..c3d80a7334b 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -89,8 +89,8 @@ ENV MINIO_ROOT_USER="clickhouse" ENV MINIO_ROOT_PASSWORD="clickhouse" ENV EXPORT_S3_STORAGE_POLICIES=1 -RUN npm install -g azurite \ - && npm install -g tslib +RUN npm install -g azurite@3.30.0 \ + && npm install -g tslib && npm install -g node COPY run.sh / COPY setup_minio.sh / diff --git a/docker/test/stateless/stress_tests.lib b/docker/test/stateless/stress_tests.lib index a0865f638a1..6aaddbfe590 100644 --- a/docker/test/stateless/stress_tests.lib +++ b/docker/test/stateless/stress_tests.lib @@ -120,13 +120,41 @@ EOL local max_users_mem max_users_mem=$((total_mem*30/100)) # 30% + # Similar to docker/test/fuzzer/query-fuzzer-tweaks-users.xml echo "Setting max_memory_usage_for_user=$max_users_mem and max_memory_usage for queries to 10G" - cat > /etc/clickhouse-server/users.d/max_memory_usage_for_user.xml < /etc/clickhouse-server/users.d/stress_test_tweaks-users.xml < + 60 10G ${max_users_mem} + + 200 + + + + 60 + + + + 10G + + + + 200 + + + + + + + + + + + + diff --git a/docker/test/util/Dockerfile b/docker/test/util/Dockerfile index 5446adf3793..bbd473b3b7a 100644 --- a/docker/test/util/Dockerfile +++ b/docker/test/util/Dockerfile @@ -5,7 +5,7 @@ FROM ubuntu:22.04 ARG apt_archive="http://archive.ubuntu.com" RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list -ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=17 +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=18 RUN apt-get update \ && apt-get install \ diff --git a/docs/en/development/build-cross-osx.md b/docs/en/development/build-cross-osx.md index eddf24448c1..66c6e2c6912 100644 --- a/docs/en/development/build-cross-osx.md +++ b/docs/en/development/build-cross-osx.md @@ -13,14 +13,14 @@ The cross-build for macOS is based on the [Build instructions](../development/bu The following sections provide a walk-through for building ClickHouse for `x86_64` macOS. If you’re targeting ARM architecture, simply substitute all occurrences of `x86_64` with `aarch64`. For example, replace `x86_64-apple-darwin` with `aarch64-apple-darwin` throughout the steps. -## Install Clang-17 +## Install clang-18 Follow the instructions from https://apt.llvm.org/ for your Ubuntu or Debian setup. For example the commands for Bionic are like: ``` bash sudo echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-17 main" >> /etc/apt/sources.list -sudo apt-get install clang-17 +sudo apt-get install clang-18 ``` ## Install Cross-Compilation Toolset {#install-cross-compilation-toolset} @@ -59,7 +59,7 @@ curl -L 'https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11 cd ClickHouse mkdir build-darwin cd build-darwin -CC=clang-17 CXX=clang++-17 cmake -DCMAKE_AR:FILEPATH=${CCTOOLS}/bin/x86_64-apple-darwin-ar -DCMAKE_INSTALL_NAME_TOOL=${CCTOOLS}/bin/x86_64-apple-darwin-install_name_tool -DCMAKE_RANLIB:FILEPATH=${CCTOOLS}/bin/x86_64-apple-darwin-ranlib -DLINKER_NAME=${CCTOOLS}/bin/x86_64-apple-darwin-ld -DCMAKE_TOOLCHAIN_FILE=cmake/darwin/toolchain-x86_64.cmake .. +CC=clang-18 CXX=clang++-18 cmake -DCMAKE_AR:FILEPATH=${CCTOOLS}/bin/x86_64-apple-darwin-ar -DCMAKE_INSTALL_NAME_TOOL=${CCTOOLS}/bin/x86_64-apple-darwin-install_name_tool -DCMAKE_RANLIB:FILEPATH=${CCTOOLS}/bin/x86_64-apple-darwin-ranlib -DLINKER_NAME=${CCTOOLS}/bin/x86_64-apple-darwin-ld -DCMAKE_TOOLCHAIN_FILE=cmake/darwin/toolchain-x86_64.cmake .. ninja ``` diff --git a/docs/en/development/build-cross-riscv.md b/docs/en/development/build-cross-riscv.md index 9ee5346f258..759d97823e2 100644 --- a/docs/en/development/build-cross-riscv.md +++ b/docs/en/development/build-cross-riscv.md @@ -23,7 +23,7 @@ sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ``` bash cd ClickHouse mkdir build-riscv64 -CC=clang-17 CXX=clang++-17 cmake . -Bbuild-riscv64 -G Ninja -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-riscv64.cmake -DGLIBC_COMPATIBILITY=OFF -DENABLE_LDAP=OFF -DOPENSSL_NO_ASM=ON -DENABLE_JEMALLOC=ON -DENABLE_PARQUET=OFF -DENABLE_GRPC=OFF -DENABLE_HDFS=OFF -DENABLE_MYSQL=OFF +CC=clang-18 CXX=clang++-18 cmake . -Bbuild-riscv64 -G Ninja -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-riscv64.cmake -DGLIBC_COMPATIBILITY=OFF -DENABLE_LDAP=OFF -DOPENSSL_NO_ASM=ON -DENABLE_JEMALLOC=ON -DENABLE_PARQUET=OFF -DENABLE_GRPC=OFF -DENABLE_HDFS=OFF -DENABLE_MYSQL=OFF ninja -C build-riscv64 ``` diff --git a/docs/en/development/build-cross-s390x.md b/docs/en/development/build-cross-s390x.md index 3c1667350e1..4c111a76d82 100644 --- a/docs/en/development/build-cross-s390x.md +++ b/docs/en/development/build-cross-s390x.md @@ -5,22 +5,13 @@ title: How to Build, Run and Debug ClickHouse on Linux for s390x (zLinux) sidebar_label: Build on Linux for s390x (zLinux) --- -As of writing (2023/3/10) building for s390x considered to be experimental. Not all features can be enabled, has broken features and is currently under active development. +At the time of writing (2024 May), support for the s390x platform is considered experimental, i.e. some features are disabled or broken on s390x. +## Building ClickHouse for s390x -## Building - -s390x has two OpenSSL-related build options. -- By default, the s390x build will dynamically link to OpenSSL libraries. It will build OpenSSL shared objects, so it's not necessary to install OpenSSL beforehand. (This option is recommended in all cases.) -- Another option is to build OpenSSL in-tree. In this case two build flags need to be supplied to cmake -```bash --DENABLE_OPENSSL_DYNAMIC=0 -``` - -:::note -s390x builds are temporarily disabled in CI. -::: - +s390x has two OpenSSL-related build options: +- By default, OpenSSL is build on s390x as a shared library. This is different from all other platforms, where OpenSSL is build as static library. +- To build OpenSSL as a static library regardless, pass `-DENABLE_OPENSSL_DYNAMIC=0` to CMake. These instructions assume that the host machine is x86_64 and has all the tooling required to build natively based on the [build instructions](../development/build.md). It also assumes that the host is Ubuntu 22.04 but the following instructions should also work on Ubuntu 20.04. @@ -31,11 +22,16 @@ apt-get install binutils-s390x-linux-gnu libc6-dev-s390x-cross gcc-s390x-linux-g ``` If you wish to cross compile rust code install the rust cross compile target for s390x: + ```bash rustup target add s390x-unknown-linux-gnu ``` +The s390x build uses the mold linker, download it from https://github.com/rui314/mold/releases/download/v2.0.0/mold-2.0.0-x86_64-linux.tar.gz +and place it into your `$PATH`. + To build for s390x: + ```bash cmake -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-s390x.cmake .. ninja diff --git a/docs/en/development/build.md b/docs/en/development/build.md index 6aad31ae3b5..227a4d62484 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -109,7 +109,7 @@ The build requires the following components: - Git (used to checkout the sources, not needed for the build) - CMake 3.20 or newer -- Compiler: clang-17 or newer +- Compiler: clang-18 or newer - Linker: lld-17 or newer - Ninja - Yasm diff --git a/docs/en/development/continuous-integration.md b/docs/en/development/continuous-integration.md index 46a30f56f11..91253ca5e44 100644 --- a/docs/en/development/continuous-integration.md +++ b/docs/en/development/continuous-integration.md @@ -153,7 +153,7 @@ Builds ClickHouse in various configurations for use in further steps. You have t ### Report Details -- **Compiler**: `clang-17`, optionally with the name of a target platform +- **Compiler**: `clang-18`, optionally with the name of a target platform - **Build type**: `Debug` or `RelWithDebInfo` (cmake). - **Sanitizer**: `none` (without sanitizers), `address` (ASan), `memory` (MSan), `undefined` (UBSan), or `thread` (TSan). - **Status**: `success` or `fail` @@ -177,7 +177,7 @@ Performs static analysis and code style checks using `clang-tidy`. The report is There is a convenience `packager` script that runs the clang-tidy build in docker ```sh mkdir build_tidy -./docker/packager/packager --output-dir=./build_tidy --package-type=binary --compiler=clang-17 --debug-build --clang-tidy +./docker/packager/packager --output-dir=./build_tidy --package-type=binary --compiler=clang-18 --debug-build --clang-tidy ``` diff --git a/docs/en/development/developer-instruction.md b/docs/en/development/developer-instruction.md index 6623c46fa9f..772d1f97590 100644 --- a/docs/en/development/developer-instruction.md +++ b/docs/en/development/developer-instruction.md @@ -121,7 +121,7 @@ While inside the `build` directory, configure your build by running CMake. Befor export CC=clang CXX=clang++ cmake .. -If you installed clang using the automatic installation script above, also specify the version of clang installed in the first command, e.g. `export CC=clang-17 CXX=clang++-17`. The clang version will be in the script output. +If you installed clang using the automatic installation script above, also specify the version of clang installed in the first command, e.g. `export CC=clang-18 CXX=clang++-18`. The clang version will be in the script output. The `CC` variable specifies the compiler for C (short for C Compiler), and `CXX` variable instructs which C++ compiler is to be used for building. diff --git a/docs/en/engines/database-engines/materialized-mysql.md b/docs/en/engines/database-engines/materialized-mysql.md index f32698f84f6..2b4d5fe04aa 100644 --- a/docs/en/engines/database-engines/materialized-mysql.md +++ b/docs/en/engines/database-engines/materialized-mysql.md @@ -51,6 +51,9 @@ ENGINE = MaterializedMySQL('host:port', ['database' | database], 'user', 'passwo ### allows_query_when_mysql_lost `allows_query_when_mysql_lost` — Allows to query a materialized table when MySQL is lost. Default: `0` (`false`). +### allow_startup_database_without_connection_to_mysql +`allow_startup_database_without_connection_to_mysql` — Allow to create and attach database without available connection to MySQL. Default: `0` (`false`). + ### materialized_mysql_tables_list `materialized_mysql_tables_list` — a comma-separated list of mysql database tables, which will be replicated by MaterializedMySQL database engine. Default value: empty list — means whole tables will be replicated. diff --git a/docs/en/engines/table-engines/integrations/rabbitmq.md b/docs/en/engines/table-engines/integrations/rabbitmq.md index a4d0cf78066..a8315a5ad9e 100644 --- a/docs/en/engines/table-engines/integrations/rabbitmq.md +++ b/docs/en/engines/table-engines/integrations/rabbitmq.md @@ -73,6 +73,7 @@ Optional parameters: - `rabbitmq_queue_consume` - Use user-defined queues and do not make any RabbitMQ setup: declaring exchanges, queues, bindings. Default: `false`. - `rabbitmq_username` - RabbitMQ username. - `rabbitmq_password` - RabbitMQ password. +- `reject_unhandled_messages` - Reject messages (send RabbitMQ negative acknowledgement) in case of errors. This setting is automatically enabled if there is a `x-dead-letter-exchange` defined in `rabbitmq_queue_settings_list`. - `rabbitmq_commit_on_select` - Commit messages when select query is made. Default: `false`. - `rabbitmq_max_rows_per_message` — The maximum number of rows written in one RabbitMQ message for row-based formats. Default : `1`. - `rabbitmq_empty_queue_backoff_start` — A start backoff point to reschedule read if the rabbitmq queue is empty. diff --git a/docs/en/engines/table-engines/mergetree-family/annindexes.md b/docs/en/engines/table-engines/mergetree-family/annindexes.md index 78a27d3ff86..5a81313f62e 100644 --- a/docs/en/engines/table-engines/mergetree-family/annindexes.md +++ b/docs/en/engines/table-engines/mergetree-family/annindexes.md @@ -22,9 +22,8 @@ ORDER BY Distance(vectors, Point) LIMIT N ``` -`vectors` contains N-dimensional values of type [Array](../../../sql-reference/data-types/array.md) or -[Tuple](../../../sql-reference/data-types/tuple.md), for example embeddings. Function `Distance` computes the distance between two vectors. -Often, the Euclidean (L2) distance is chosen as distance function but [other +`vectors` contains N-dimensional values of type [Array(Float32)](../../../sql-reference/data-types/array.md), for example embeddings. +Function `Distance` computes the distance between two vectors. Often, the Euclidean (L2) distance is chosen as distance function but [other distance functions](/docs/en/sql-reference/functions/distance-functions.md) are also possible. `Point` is the reference point, e.g. `(0.17, 0.33, ...)`, and `N` limits the number of search results. @@ -47,7 +46,7 @@ of the search space (using clustering, search trees, etc.) which allows to compu # Creating and Using ANN Indexes {#creating_using_ann_indexes} -Syntax to create an ANN index over an [Array](../../../sql-reference/data-types/array.md) column: +Syntax to create an ANN index over an [Array(Float32)](../../../sql-reference/data-types/array.md) column: ```sql CREATE TABLE table_with_ann_index @@ -60,19 +59,6 @@ ENGINE = MergeTree ORDER BY id; ``` -Syntax to create an ANN index over a [Tuple](../../../sql-reference/data-types/tuple.md) column: - -```sql -CREATE TABLE table_with_ann_index -( - `id` Int64, - `vectors` Tuple(Float32[, Float32[, ...]]), - INDEX [ann_index_name] vectors TYPE [ann_index_type]([ann_index_parameters]) [GRANULARITY [N]] -) -ENGINE = MergeTree -ORDER BY id; -``` - ANN indexes are built during column insertion and merge. As a result, `INSERT` and `OPTIMIZE` statements will be slower than for ordinary tables. ANNIndexes are ideally used only with immutable or rarely changed data, respectively when are far more read requests than write requests. @@ -164,7 +150,7 @@ linear surfaces (lines in 2D, planes in 3D etc.). -Syntax to create an Annoy index over an [Array](../../../sql-reference/data-types/array.md) column: +Syntax to create an Annoy index over an [Array(Float32)](../../../sql-reference/data-types/array.md) column: ```sql CREATE TABLE table_with_annoy_index @@ -177,19 +163,6 @@ ENGINE = MergeTree ORDER BY id; ``` -Syntax to create an ANN index over a [Tuple](../../../sql-reference/data-types/tuple.md) column: - -```sql -CREATE TABLE table_with_annoy_index -( - id Int64, - vectors Tuple(Float32[, Float32[, ...]]), - INDEX [ann_index_name] vectors TYPE annoy([Distance[, NumTrees]]) [GRANULARITY N] -) -ENGINE = MergeTree -ORDER BY id; -``` - Annoy currently supports two distance functions: - `L2Distance`, also called Euclidean distance, is the length of a line segment between two points in Euclidean space ([Wikipedia](https://en.wikipedia.org/wiki/Euclidean_distance)). @@ -203,10 +176,9 @@ Parameter `NumTrees` is the number of trees which the algorithm creates (default more accurate search results but slower index creation / query times (approximately linearly) as well as larger index sizes. :::note -Indexes over columns of type `Array` will generally work faster than indexes on `Tuple` columns. All arrays must have same length. To avoid -errors, you can use a [CONSTRAINT](/docs/en/sql-reference/statements/create/table.md#constraints), for example, `CONSTRAINT -constraint_name_1 CHECK length(vectors) = 256`. Also, empty `Arrays` and unspecified `Array` values in INSERT statements (i.e. default -values) are not supported. +All arrays must have same length. To avoid errors, you can use a +[CONSTRAINT](/docs/en/sql-reference/statements/create/table.md#constraints), for example, `CONSTRAINT constraint_name_1 CHECK +length(vectors) = 256`. Also, empty `Arrays` and unspecified `Array` values in INSERT statements (i.e. default values) are not supported. ::: The creation of Annoy indexes (whenever a new part is build, e.g. at the end of a merge) is a relatively slow process. You can increase @@ -264,19 +236,6 @@ ENGINE = MergeTree ORDER BY id; ``` -Syntax to create an ANN index over a [Tuple](../../../sql-reference/data-types/tuple.md) column: - -```sql -CREATE TABLE table_with_usearch_index -( - id Int64, - vectors Tuple(Float32[, Float32[, ...]]), - INDEX [ann_index_name] vectors TYPE usearch([Distance[, ScalarKind]]) [GRANULARITY N] -) -ENGINE = MergeTree -ORDER BY id; -``` - USearch currently supports two distance functions: - `L2Distance`, also called Euclidean distance, is the length of a line segment between two points in Euclidean space ([Wikipedia](https://en.wikipedia.org/wiki/Euclidean_distance)). diff --git a/docs/en/engines/table-engines/mergetree-family/invertedindexes.md b/docs/en/engines/table-engines/mergetree-family/invertedindexes.md index 9374f6a3ac1..f58a06464b2 100644 --- a/docs/en/engines/table-engines/mergetree-family/invertedindexes.md +++ b/docs/en/engines/table-engines/mergetree-family/invertedindexes.md @@ -2,7 +2,7 @@ slug: /en/engines/table-engines/mergetree-family/invertedindexes sidebar_label: Full-text Indexes description: Quickly find search terms in text. -keywords: [full-text search, text search, inverted, index, indices] +keywords: [full-text search, text search, index, indices] --- # Full-text Search using Full-text Indexes [experimental] @@ -53,6 +53,10 @@ ENGINE = MergeTree ORDER BY key ``` +:::note +In earlier versions of ClickHouse, the corresponding index type name was `inverted`. +::: + where `N` specifies the tokenizer: - `full_text(0)` (or shorter: `full_text()`) set the tokenizer to "tokens", i.e. split strings along spaces, diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 886c29e755e..7862eef69f8 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -494,7 +494,7 @@ Syntax: `tokenbf_v1(size_of_bloom_filter_in_bytes, number_of_hash_functions, ran #### Special-purpose - Experimental indexes to support approximate nearest neighbor (ANN) search. See [here](annindexes.md) for details. -- An experimental inverted index to support full-text search. See [here](invertedindexes.md) for details. +- An experimental full-text index to support full-text search. See [here](invertedindexes.md) for details. ### Functions Support {#functions-support} @@ -502,31 +502,31 @@ Conditions in the `WHERE` clause contains calls of the functions that operate wi Indexes of type `set` can be utilized by all functions. The other index types are supported as follows: -| Function (operator) / Index | primary key | minmax | ngrambf_v1 | tokenbf_v1 | bloom_filter | inverted | -|------------------------------------------------------------------------------------------------------------|-------------|--------|------------|------------|--------------|----------| -| [equals (=, ==)](/docs/en/sql-reference/functions/comparison-functions.md/#equals) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | -| [notEquals(!=, <>)](/docs/en/sql-reference/functions/comparison-functions.md/#notequals) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | -| [like](/docs/en/sql-reference/functions/string-search-functions.md/#like) | ✔ | ✔ | ✔ | ✔ | ✗ | ✔ | -| [notLike](/docs/en/sql-reference/functions/string-search-functions.md/#notlike) | ✔ | ✔ | ✔ | ✔ | ✗ | ✔ | -| [match](/docs/en/sql-reference/functions/string-search-functions.md/#match) | ✗ | ✗ | ✔ | ✔ | ✗ | ✔ | -| [startsWith](/docs/en/sql-reference/functions/string-functions.md/#startswith) | ✔ | ✔ | ✔ | ✔ | ✗ | ✔ | -| [endsWith](/docs/en/sql-reference/functions/string-functions.md/#endswith) | ✗ | ✗ | ✔ | ✔ | ✗ | ✔ | -| [multiSearchAny](/docs/en/sql-reference/functions/string-search-functions.md/#multisearchany) | ✗ | ✗ | ✔ | ✗ | ✗ | ✔ | -| [in](/docs/en/sql-reference/functions/in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | -| [notIn](/docs/en/sql-reference/functions/in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | -| [less (<)](/docs/en/sql-reference/functions/comparison-functions.md/#less) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ | -| [greater (>)](/docs/en/sql-reference/functions/comparison-functions.md/#greater) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ | -| [lessOrEquals (<=)](/docs/en/sql-reference/functions/comparison-functions.md/#lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ | -| [greaterOrEquals (>=)](/docs/en/sql-reference/functions/comparison-functions.md/#greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ | -| [empty](/docs/en/sql-reference/functions/array-functions/#empty) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ | -| [notEmpty](/docs/en/sql-reference/functions/array-functions/#notempty) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ | -| [has](/docs/en/sql-reference/functions/array-functions/#has) | ✗ | ✗ | ✔ | ✔ | ✔ | ✔ | -| [hasAny](/docs/en/sql-reference/functions/array-functions/#hasany) | ✗ | ✗ | ✔ | ✔ | ✔ | ✗ | -| [hasAll](/docs/en/sql-reference/functions/array-functions/#hasall) | ✗ | ✗ | ✗ | ✗ | ✔ | ✗ | -| hasToken | ✗ | ✗ | ✗ | ✔ | ✗ | ✔ | -| hasTokenOrNull | ✗ | ✗ | ✗ | ✔ | ✗ | ✔ | -| hasTokenCaseInsensitive (*) | ✗ | ✗ | ✗ | ✔ | ✗ | ✗ | -| hasTokenCaseInsensitiveOrNull (*) | ✗ | ✗ | ✗ | ✔ | ✗ | ✗ | +| Function (operator) / Index | primary key | minmax | ngrambf_v1 | tokenbf_v1 | bloom_filter | full_text | +|------------------------------------------------------------------------------------------------------------|-------------|--------|------------|------------|--------------|-----------| +| [equals (=, ==)](/docs/en/sql-reference/functions/comparison-functions.md/#equals) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | +| [notEquals(!=, <>)](/docs/en/sql-reference/functions/comparison-functions.md/#notequals) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | +| [like](/docs/en/sql-reference/functions/string-search-functions.md/#like) | ✔ | ✔ | ✔ | ✔ | ✗ | ✔ | +| [notLike](/docs/en/sql-reference/functions/string-search-functions.md/#notlike) | ✔ | ✔ | ✔ | ✔ | ✗ | ✔ | +| [match](/docs/en/sql-reference/functions/string-search-functions.md/#match) | ✗ | ✗ | ✔ | ✔ | ✗ | ✔ | +| [startsWith](/docs/en/sql-reference/functions/string-functions.md/#startswith) | ✔ | ✔ | ✔ | ✔ | ✗ | ✔ | +| [endsWith](/docs/en/sql-reference/functions/string-functions.md/#endswith) | ✗ | ✗ | ✔ | ✔ | ✗ | ✔ | +| [multiSearchAny](/docs/en/sql-reference/functions/string-search-functions.md/#multisearchany) | ✗ | ✗ | ✔ | ✗ | ✗ | ✔ | +| [in](/docs/en/sql-reference/functions/in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | +| [notIn](/docs/en/sql-reference/functions/in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ | +| [less (<)](/docs/en/sql-reference/functions/comparison-functions.md/#less) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ | +| [greater (>)](/docs/en/sql-reference/functions/comparison-functions.md/#greater) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ | +| [lessOrEquals (<=)](/docs/en/sql-reference/functions/comparison-functions.md/#lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ | +| [greaterOrEquals (>=)](/docs/en/sql-reference/functions/comparison-functions.md/#greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ | +| [empty](/docs/en/sql-reference/functions/array-functions/#empty) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ | +| [notEmpty](/docs/en/sql-reference/functions/array-functions/#notempty) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ | +| [has](/docs/en/sql-reference/functions/array-functions/#has) | ✗ | ✗ | ✔ | ✔ | ✔ | ✔ | +| [hasAny](/docs/en/sql-reference/functions/array-functions/#hasany) | ✗ | ✗ | ✔ | ✔ | ✔ | ✗ | +| [hasAll](/docs/en/sql-reference/functions/array-functions/#hasall) | ✗ | ✗ | ✗ | ✗ | ✔ | ✗ | +| hasToken | ✗ | ✗ | ✗ | ✔ | ✗ | ✔ | +| hasTokenOrNull | ✗ | ✗ | ✗ | ✔ | ✗ | ✔ | +| hasTokenCaseInsensitive (*) | ✗ | ✗ | ✗ | ✔ | ✗ | ✗ | +| hasTokenCaseInsensitiveOrNull (*) | ✗ | ✗ | ✗ | ✔ | ✗ | ✗ | Functions with a constant argument that is less than ngram size can’t be used by `ngrambf_v1` for query optimization. diff --git a/docs/en/engines/table-engines/mergetree-family/replication.md b/docs/en/engines/table-engines/mergetree-family/replication.md index 822e2bc385f..65a5f58b166 100644 --- a/docs/en/engines/table-engines/mergetree-family/replication.md +++ b/docs/en/engines/table-engines/mergetree-family/replication.md @@ -10,7 +10,7 @@ sidebar_label: Data Replication In ClickHouse Cloud replication is managed for you. Please create your tables without adding arguments. For example, in the text below you would replace: ```sql -ENGINE = ReplicatedReplacingMergeTree( +ENGINE = ReplicatedMergeTree( '/clickhouse/tables/{shard}/table_name', '{replica}', ver @@ -20,7 +20,7 @@ ENGINE = ReplicatedReplacingMergeTree( with: ```sql -ENGINE = ReplicatedReplacingMergeTree +ENGINE = ReplicatedMergeTree ``` ::: @@ -140,11 +140,11 @@ The system monitors data synchronicity on replicas and is able to recover after :::note In ClickHouse Cloud replication is managed for you. Please create your tables without adding arguments. For example, in the text below you would replace: ``` -ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/table_name', '{replica}', ver) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/table_name', '{replica}', ver) ``` with: ``` -ENGINE = ReplicatedReplacingMergeTree +ENGINE = ReplicatedMergeTree ``` ::: @@ -177,7 +177,7 @@ CREATE TABLE table_name CounterID UInt32, UserID UInt32, ver UInt16 -) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/table_name', '{replica}', ver) +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/table_name', '{replica}', ver) PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID); diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index e2bb9f01037..979750d4776 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -91,6 +91,7 @@ The supported formats are: | [MySQLDump](#mysqldump) | ✔ | ✗ | | [DWARF](#dwarf) | ✔ | ✗ | | [Markdown](#markdown) | ✗ | ✔ | +| [Form](#form) | ✔ | ✗ | You can control some format processing parameters with the ClickHouse settings. For more information read the [Settings](/docs/en/operations/settings/settings-formats.md) section. @@ -207,7 +208,7 @@ SELECT * FROM nestedt FORMAT TSV Differs from `TabSeparated` format in that the rows are written without escaping. When parsing with this format, tabs or linefeeds are not allowed in each field. -This format is also available under the name `TSVRaw`. +This format is also available under the names `TSVRaw`, `Raw`. ## TabSeparatedWithNames {#tabseparatedwithnames} @@ -242,14 +243,14 @@ This format is also available under the name `TSVWithNamesAndTypes`. Differs from `TabSeparatedWithNames` format in that the rows are written without escaping. When parsing with this format, tabs or linefeeds are not allowed in each field. -This format is also available under the name `TSVRawWithNames`. +This format is also available under the names `TSVRawWithNames`, `RawWithNames`. ## TabSeparatedRawWithNamesAndTypes {#tabseparatedrawwithnamesandtypes} Differs from `TabSeparatedWithNamesAndTypes` format in that the rows are written without escaping. When parsing with this format, tabs or linefeeds are not allowed in each field. -This format is also available under the name `TSVRawWithNamesAndNames`. +This format is also available under the names `TSVRawWithNamesAndNames`, `RawWithNamesAndNames`. ## Template {#format-template} @@ -2844,3 +2845,31 @@ FORMAT Markdown ``` Markdown table will be generated automatically and can be used on markdown-enabled platforms, like Github. This format is used only for output. + +## Form {#form} + +The Form format can be used to read or write a single record in the application/x-www-form-urlencoded format in which data is formatted `key1=value1&key2=value2` + +Examples: + +Given a file `data.tmp` placed in the `user_files` path with some URL encoded data: + +```text +t_page=116&c.e=ls7xfkpm&c.tti.m=raf&rt.start=navigation&rt.bmr=390%2C11%2C10 +``` + +```sql +SELECT * FROM file(data.tmp, Form) FORMAT vertical; +``` + +Result: + +```text +Row 1: +────── +t_page: 116 +c.e: ls7xfkpm +c.tti.m: raf +rt.start: navigation +rt.bmr: 390,11,10 +``` diff --git a/docs/en/interfaces/third-party/gui.md b/docs/en/interfaces/third-party/gui.md index be2d028e87f..bb326dc5242 100644 --- a/docs/en/interfaces/third-party/gui.md +++ b/docs/en/interfaces/third-party/gui.md @@ -210,6 +210,18 @@ Features: - Pre-built metrics dashboards. - Multiple users/projects via YAML config. +### clickhouse-monitoring {#clickhouse-monitoring} + +[clickhouse-monitoring](https://github.com/duyet/clickhouse-monitoring) is a simple Next.js dashboard that relies on `system.*` tables to help monitor and provide an overview of your ClickHouse cluster. + +Features: + +- Query monitor: current queries, query history, query resources (memory, parts read, file_open, ...), most expensive queries, most used tables or columns, etc. +- Cluster monitor: total memory/CPU usage, distributed queue, global settings, mergetree settings, metrics, etc. +- Tables and parts information: size, row count, compression, part size, etc., at the column level detail. +- Useful tools: Zookeeper data exploration, query EXPLAIN, kill queries, etc. +- Visualization metric charts: queries and resource usage, number of merges/mutation, merge performance, query performance, etc. + ## Commercial {#commercial} ### DataGrip {#datagrip} diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 4f158a4fd6e..91b544c6a82 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1925,7 +1925,9 @@ Default value: `16`. ### wait_for_async_insert {#wait-for-async-insert} -Enables or disables waiting for processing of asynchronous insertion. If enabled, server will return `OK` only after the data is inserted. Otherwise, it will return `OK` even if the data wasn't inserted. +Enables or disables waiting for processing of asynchronous insertion. If enabled, server will return `OK` only after the data is inserted. Otherwise, it will return `OK` as soon it has received the data, but it might still fail to parse or insert it later (You can check in system.asynchronous_insert_log) + +If you want to use asynchronous inserts, we need to also enable [`async_insert`](#async-insert). Possible values: diff --git a/docs/en/operations/storing-data.md b/docs/en/operations/storing-data.md index 389c917d427..7005783dd60 100644 --- a/docs/en/operations/storing-data.md +++ b/docs/en/operations/storing-data.md @@ -371,6 +371,8 @@ is equal to ``` +Starting from `24.5` it is possible configure any object storage disk (`s3`, `azure`, `local`) using `plain_rewritable` metadata type. + ### Using Azure Blob Storage {#azure-blob-storage} `MergeTree` family table engines can store data to [Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/) using a disk with type `azure_blob_storage`. diff --git a/docs/en/operations/system-tables/part_log.md b/docs/en/operations/system-tables/part_log.md index af582646653..c0caea1ce5e 100644 --- a/docs/en/operations/system-tables/part_log.md +++ b/docs/en/operations/system-tables/part_log.md @@ -14,7 +14,7 @@ The `system.part_log` table contains the following columns: - `event_type` ([Enum8](../../sql-reference/data-types/enum.md)) — Type of the event that occurred with the data part. Can have one of the following values: - `NewPart` — Inserting of a new data part. - `MergeParts` — Merging of data parts. - - `DownloadParts` — Downloading a data part. + - `DownloadPart` — Downloading a data part. - `RemovePart` — Removing or detaching a data part using [DETACH PARTITION](../../sql-reference/statements/alter/partition.md#alter_detach-partition). - `MutatePart` — Mutating of a data part. - `MovePart` — Moving the data part from the one disk to another one. diff --git a/docs/en/sql-reference/aggregate-functions/reference/any.md b/docs/en/sql-reference/aggregate-functions/reference/any.md index 4631060f33f..f1b5a6683e5 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/any.md +++ b/docs/en/sql-reference/aggregate-functions/reference/any.md @@ -7,15 +7,50 @@ sidebar_position: 6 Selects the first encountered value of a column. -By default, it ignores NULL values and returns the first NOT NULL value found in the column. As [`first_value`](../../../sql-reference/aggregate-functions/reference/first_value.md) if supports `RESPECT NULLS`, in which case it will select the first value passed, independently on whether it's NULL or not. +**Syntax** +```sql +any(column) +``` + +Aliases: `any_value`, [`first_value`](../reference/first_value.md). + +**Parameters** +- `column`: The column name. + +**Returned value** + +By default, it ignores NULL values and returns the first NOT NULL value found in the column. Like [`first_value`](../../../sql-reference/aggregate-functions/reference/first_value.md) it supports `RESPECT NULLS`, in which case it will select the first value passed, independently on whether it's NULL or not. + +:::note The return type of the function is the same as the input, except for LowCardinality which is discarded. This means that given no rows as input it will return the default value of that type (0 for integers, or Null for a Nullable() column). You might use the `-OrNull` [combinator](../../../sql-reference/aggregate-functions/combinators.md) ) to modify this behaviour. +::: +:::warning The query can be executed in any order and even in a different order each time, so the result of this function is indeterminate. -To get a determinate result, you can use the ‘min’ or ‘max’ function instead of ‘any’. +To get a determinate result, you can use the [`min`](../reference/min.md) or [`max`](../reference/max.md) function instead of `any`. +::: -In some cases, you can rely on the order of execution. This applies to cases when SELECT comes from a subquery that uses ORDER BY. +**Implementation details** + +In some cases, you can rely on the order of execution. This applies to cases when `SELECT` comes from a subquery that uses `ORDER BY`. When a `SELECT` query has the `GROUP BY` clause or at least one aggregate function, ClickHouse (in contrast to MySQL) requires that all expressions in the `SELECT`, `HAVING`, and `ORDER BY` clauses be calculated from keys or from aggregate functions. In other words, each column selected from the table must be used either in keys or inside aggregate functions. To get behavior like in MySQL, you can put the other columns in the `any` aggregate function. -- Alias: `any_value`, `first_value`. +**Example** + +Query: + +```sql +CREATE TABLE any_nulls (city Nullable(String)) ENGINE=Log; + +INSERT INTO any_nulls (city) VALUES (NULL), ('Amsterdam'), ('New York'), ('Tokyo'), ('Valencia'), (NULL); + +SELECT any(city) FROM any_nulls; +``` + +```response +┌─any(city)─┐ +│ Amsterdam │ +└───────────┘ +``` diff --git a/docs/en/sql-reference/aggregate-functions/reference/any_respect_nulls.md b/docs/en/sql-reference/aggregate-functions/reference/any_respect_nulls.md new file mode 100644 index 00000000000..99104a9b8c7 --- /dev/null +++ b/docs/en/sql-reference/aggregate-functions/reference/any_respect_nulls.md @@ -0,0 +1,44 @@ +--- +slug: /en/sql-reference/aggregate-functions/reference/any_respect_nulls +sidebar_position: 103 +--- + +# any_respect_nulls + +Selects the first encountered value of a column, irregardless of whether it is a `NULL` value or not. + +Alias: `any_value_respect_nulls`, `first_value_repect_nulls`. + +**Syntax** + +```sql +any_respect_nulls(column) +``` + +**Parameters** +- `column`: The column name. + +**Returned value** + +- The last value encountered, irregardless of whether it is a `NULL` value or not. + +**Example** + +Query: + +```sql +CREATE TABLE any_nulls (city Nullable(String)) ENGINE=Log; + +INSERT INTO any_nulls (city) VALUES (NULL), ('Amsterdam'), ('New York'), ('Tokyo'), ('Valencia'), (NULL); + +SELECT any(city), any_respect_nulls(city) FROM any_nulls; +``` + +```response +┌─any(city)─┬─any_respect_nulls(city)─┐ +│ Amsterdam │ ᴺᵁᴸᴸ │ +└───────────┴─────────────────────────┘ +``` + +**See Also** +- [any](../reference/any.md) diff --git a/docs/en/sql-reference/aggregate-functions/reference/anylast.md b/docs/en/sql-reference/aggregate-functions/reference/anylast.md index 351c9fd8e2f..8fcee2cf8e6 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/anylast.md +++ b/docs/en/sql-reference/aggregate-functions/reference/anylast.md @@ -5,5 +5,35 @@ sidebar_position: 104 # anyLast -Selects the last value encountered. -The result is just as indeterminate as for the [any](../../../sql-reference/aggregate-functions/reference/any.md) function. +Selects the last value encountered. The result is just as indeterminate as for the [any](../../../sql-reference/aggregate-functions/reference/any.md) function. + +**Syntax** + +```sql +anyLast(column) +``` + +**Parameters** +- `column`: The column name. + +**Returned value** + +- The last value encountered. + +**Example** + +Query: + +```sql +CREATE TABLE any_last_nulls (city Nullable(String)) ENGINE=Log; + +INSERT INTO any_last_nulls (city) VALUES ('Amsterdam'),(NULL),('New York'),('Tokyo'),('Valencia'),(NULL); + +SELECT anyLast(city) FROM any_last_nulls; +``` + +```response +┌─anyLast(city)─┐ +│ Valencia │ +└───────────────┘ +``` \ No newline at end of file diff --git a/docs/en/sql-reference/aggregate-functions/reference/anylast_respect_nulls.md b/docs/en/sql-reference/aggregate-functions/reference/anylast_respect_nulls.md new file mode 100644 index 00000000000..b6d0806f35d --- /dev/null +++ b/docs/en/sql-reference/aggregate-functions/reference/anylast_respect_nulls.md @@ -0,0 +1,39 @@ +--- +slug: /en/sql-reference/aggregate-functions/reference/anylast_respect_nulls +sidebar_position: 104 +--- + +# anyLast_respect_nulls + +Selects the last value encountered, irregardless of whether it is `NULL` or not. + +**Syntax** + +```sql +anyLast_respect_nulls(column) +``` + +**Parameters** +- `column`: The column name. + +**Returned value** + +- The last value encountered, irregardless of whether it is `NULL` or not. + +**Example** + +Query: + +```sql +CREATE TABLE any_last_nulls (city Nullable(String)) ENGINE=Log; + +INSERT INTO any_last_nulls (city) VALUES ('Amsterdam'),(NULL),('New York'),('Tokyo'),('Valencia'),(NULL); + +SELECT anyLast(city), anyLast_respect_nulls(city) FROM any_last_nulls; +``` + +```response +┌─anyLast(city)─┬─anyLast_respect_nulls(city)─┐ +│ Valencia │ ᴺᵁᴸᴸ │ +└───────────────┴─────────────────────────────┘ +``` \ No newline at end of file diff --git a/docs/en/sql-reference/aggregate-functions/reference/index.md b/docs/en/sql-reference/aggregate-functions/reference/index.md index cfcca1bb436..e9a7fe4fc2b 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/index.md +++ b/docs/en/sql-reference/aggregate-functions/reference/index.md @@ -37,8 +37,10 @@ Standard aggregate functions: ClickHouse-specific aggregate functions: +- [any](/docs/en/sql-reference/aggregate-functions/reference/any_respect_nulls.md) - [anyHeavy](/docs/en/sql-reference/aggregate-functions/reference/anyheavy.md) - [anyLast](/docs/en/sql-reference/aggregate-functions/reference/anylast.md) +- [anyLast](/docs/en/sql-reference/aggregate-functions/reference/anylast_respect_nulls.md) - [boundingRatio](/docs/en/sql-reference/aggregate-functions/reference/boundrat.md) - [first_value](/docs/en/sql-reference/aggregate-functions/reference/first_value.md) - [last_value](/docs/en/sql-reference/aggregate-functions/reference/last_value.md) @@ -97,6 +99,7 @@ ClickHouse-specific aggregate functions: - [quantileBFloat16Weighted](/docs/en/sql-reference/aggregate-functions/reference/quantilebfloat16.md#quantilebfloat16weighted) - [quantileDD](/docs/en/sql-reference/aggregate-functions/reference/quantileddsketch.md#quantileddsketch) - [simpleLinearRegression](/docs/en/sql-reference/aggregate-functions/reference/simplelinearregression.md) +- [singleValueOrNull](/docs/en/sql-reference/aggregate-functions/reference/singlevalueornull.md) - [stochasticLinearRegression](/docs/en/sql-reference/aggregate-functions/reference/stochasticlinearregression.md) - [stochasticLogisticRegression](/docs/en/sql-reference/aggregate-functions/reference/stochasticlogisticregression.md) - [categoricalInformationValue](/docs/en/sql-reference/aggregate-functions/reference/categoricalinformationvalue.md) diff --git a/docs/en/sql-reference/aggregate-functions/reference/singlevalueornull.md b/docs/en/sql-reference/aggregate-functions/reference/singlevalueornull.md new file mode 100644 index 00000000000..e39af77059a --- /dev/null +++ b/docs/en/sql-reference/aggregate-functions/reference/singlevalueornull.md @@ -0,0 +1,57 @@ +--- +slug: /en/sql-reference/aggregate-functions/reference/singlevalueornull +sidebar_position: 220 +--- + +# singleValueOrNull + +The aggregate function `singleValueOrNull` is used to implement subquery operators, such as `x = ALL (SELECT ...)`. It checks if there is only one unique non-NULL value in the data. +If there is only one unique value, it returns it. If there are zero or at least two distinct values, it returns NULL. + +**Syntax** + +``` sql +singleValueOrNull(x) +``` + +**Parameters** + +- `x` — Column of any [data type](../../data-types/index.md). + +**Returned values** + +- The unique value, if there is only one unique non-NULL value in `x`. +- `NULL`, if there are zero or at least two distinct values. + +**Examples** + +Query: + +``` sql +CREATE TABLE test (x UInt8 NULL) ENGINE=Log; +INSERT INTO test (x) VALUES (NULL), (NULL), (5), (NULL), (NULL); +SELECT singleValueOrNull(x) FROM test; +``` + +Result: + +```response +┌─singleValueOrNull(x)─┐ +│ 5 │ +└──────────────────────┘ +``` + +Query: + +```sql +INSERT INTO test (x) VALUES (10); +SELECT singleValueOrNull(x) FROM test; +``` + +Result: + +```response +┌─singleValueOrNull(x)─┐ +│ ᴺᵁᴸᴸ │ +└──────────────────────┘ +``` diff --git a/docs/en/sql-reference/data-types/map.md b/docs/en/sql-reference/data-types/map.md index 2c734969afc..18c7816f811 100644 --- a/docs/en/sql-reference/data-types/map.md +++ b/docs/en/sql-reference/data-types/map.md @@ -7,10 +7,11 @@ sidebar_label: Map(K, V) # Map(K, V) `Map(K, V)` data type stores `key:value` pairs. +The Map datatype is implemented as `Array(Tuple(key T1, value T2))`, which means that the order of keys in each map does not change, i.e., this data type maintains insertion order. **Parameters** -- `key` — The key part of the pair. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md), [LowCardinality](../../sql-reference/data-types/lowcardinality.md), [FixedString](../../sql-reference/data-types/fixedstring.md), [UUID](../../sql-reference/data-types/uuid.md), [Date](../../sql-reference/data-types/date.md), [DateTime](../../sql-reference/data-types/datetime.md), [Date32](../../sql-reference/data-types/date32.md), [Enum](../../sql-reference/data-types/enum.md). +- `key` — The key part of the pair. Arbitrary type, except [Nullable](../../sql-reference/data-types/nullable.md) and [LowCardinality](../../sql-reference/data-types/lowcardinality.md) nested with [Nullable](../../sql-reference/data-types/nullable.md) types. - `value` — The value part of the pair. Arbitrary type, including [Map](../../sql-reference/data-types/map.md) and [Array](../../sql-reference/data-types/array.md). To get the value from an `a Map('key', 'value')` column, use `a['key']` syntax. This lookup works now with a linear complexity. diff --git a/docs/en/sql-reference/functions/conditional-functions.md b/docs/en/sql-reference/functions/conditional-functions.md index eb4e98961f1..564186fd8db 100644 --- a/docs/en/sql-reference/functions/conditional-functions.md +++ b/docs/en/sql-reference/functions/conditional-functions.md @@ -234,3 +234,34 @@ SELECT least(toDateTime32(now() + toIntervalDay(1)), toDateTime64(now(), 3)) :::note The type returned is a DateTime64 as the DataTime32 must be promoted to 64 bit for the comparison. ::: + +## clamp + +Constrain the return value between A and B. + +**Syntax** + +``` sql +clamp(value, min, max) +``` + +**Arguments** + +- `value` – Input value. +- `min` – Limit the lower bound. +- `max` – Limit the upper bound. + +**Returned values** + +If the value is less than the minimum value, return the minimum value; if it is greater than the maximum value, return the maximum value; otherwise, return the current value. + +Examples: + +```sql +SELECT clamp(1, 2, 3) result, toTypeName(result) type; +``` +```response +┌─result─┬─type────┐ +│ 2 │ Float64 │ +└────────┴─────────┘ +``` \ No newline at end of file diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index ed98f986139..843f22e5a6f 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -1417,31 +1417,31 @@ toStartOfFifteenMinutes(toDateTime('2023-04-21 10:23:00')): 2023-04-21 10:15:00 This function generalizes other `toStartOf*()` functions with `toStartOfInterval(date_or_date_with_time, INTERVAL x unit [, time_zone])` syntax. For example, -- `toStartOfInterval(t, INTERVAL 1 year)` returns the same as `toStartOfYear(t)`, -- `toStartOfInterval(t, INTERVAL 1 month)` returns the same as `toStartOfMonth(t)`, -- `toStartOfInterval(t, INTERVAL 1 day)` returns the same as `toStartOfDay(t)`, -- `toStartOfInterval(t, INTERVAL 15 minute)` returns the same as `toStartOfFifteenMinutes(t)`. +- `toStartOfInterval(t, INTERVAL 1 YEAR)` returns the same as `toStartOfYear(t)`, +- `toStartOfInterval(t, INTERVAL 1 MONTH)` returns the same as `toStartOfMonth(t)`, +- `toStartOfInterval(t, INTERVAL 1 DAY)` returns the same as `toStartOfDay(t)`, +- `toStartOfInterval(t, INTERVAL 15 MINUTE)` returns the same as `toStartOfFifteenMinutes(t)`. The calculation is performed relative to specific points in time: | Interval | Start | |-------------|------------------------| -| year | year 0 | -| quarter | 1900 Q1 | -| month | 1900 January | -| week | 1970, 1st week (01-05) | -| day | 1970-01-01 | -| hour | (*) | -| minute | 1970-01-01 00:00:00 | -| second | 1970-01-01 00:00:00 | -| millisecond | 1970-01-01 00:00:00 | -| microsecond | 1970-01-01 00:00:00 | -| nanosecond | 1970-01-01 00:00:00 | +| YEAR | year 0 | +| QUARTER | 1900 Q1 | +| MONTH | 1900 January | +| WEEK | 1970, 1st week (01-05) | +| DAY | 1970-01-01 | +| HOUR | (*) | +| MINUTE | 1970-01-01 00:00:00 | +| SECOND | 1970-01-01 00:00:00 | +| MILLISECOND | 1970-01-01 00:00:00 | +| MICROSECOND | 1970-01-01 00:00:00 | +| NANOSECOND | 1970-01-01 00:00:00 | (*) hour intervals are special: the calculation is always performed relative to 00:00:00 (midnight) of the current day. As a result, only hour values between 1 and 23 are useful. -If unit `week` was specified, `toStartOfInterval` assumes that weeks start on Monday. Note that this behavior is different from that of function `toStartOfWeek` in which weeks start by default on Sunday. +If unit `WEEK` was specified, `toStartOfInterval` assumes that weeks start on Monday. Note that this behavior is different from that of function `toStartOfWeek` in which weeks start by default on Sunday. **See Also** @@ -2558,13 +2558,27 @@ Like function `YYYYMMDDhhmmssToDate()` but produces a [DateTime64](../../sql-ref Accepts an additional, optional `precision` parameter after the `timezone` parameter. -## addYears, addQuarters, addMonths, addWeeks, addDays, addHours, addMinutes, addSeconds, addMilliseconds, addMicroseconds, addNanoseconds +## addYears -These functions add units of the interval specified by the function name to a date, a date with time or a string-encoded date / date with time. A date or date with time is returned. +Adds a specified number of years to a date, a date with time or a string-encoded date / date with time. -Example: +**Syntax** -``` sql +```sql +addYears(date, num) +``` + +**Parameters** + +- `date`: Date / date with time to add specified number of years to. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md). +- `num`: Number of years to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). + +**Returned value** +- Returns `date` plus `num` years. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Example** + +```sql WITH toDate('2024-01-01') AS date, toDateTime('2024-01-01 00:00:00') AS date_time, @@ -2575,19 +2589,477 @@ SELECT addYears(date_time_string, 1) AS add_years_with_date_time_string ``` -``` text +```response ┌─add_years_with_date─┬─add_years_with_date_time─┬─add_years_with_date_time_string─┐ │ 2025-01-01 │ 2025-01-01 00:00:00 │ 2025-01-01 00:00:00.000 │ └─────────────────────┴──────────────────────────┴─────────────────────────────────┘ ``` -## subtractYears, subtractQuarters, subtractMonths, subtractWeeks, subtractDays, subtractHours, subtractMinutes, subtractSeconds, subtractMilliseconds, subtractMicroseconds, subtractNanoseconds +## addQuarters -These functions subtract units of the interval specified by the function name from a date, a date with time or a string-encoded date / date with time. A date or date with time is returned. +Adds a specified number of quarters to a date, a date with time or a string-encoded date / date with time. -Example: +**Syntax** -``` sql +```sql +addQuarters(date, num) +``` + +**Parameters** + +- `date`: Date / date with time to add specified number of quarters to. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md). +- `num`: Number of quarters to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). + +**Returned value** +- Returns `date` plus `num` quarters. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Example** + +```sql +WITH + toDate('2024-01-01') AS date, + toDateTime('2024-01-01 00:00:00') AS date_time, + '2024-01-01 00:00:00' AS date_time_string +SELECT + addQuarters(date, 1) AS add_quarters_with_date, + addQuarters(date_time, 1) AS add_quarters_with_date_time, + addQuarters(date_time_string, 1) AS add_quarters_with_date_time_string +``` + +```response +┌─add_quarters_with_date─┬─add_quarters_with_date_time─┬─add_quarters_with_date_time_string─┐ +│ 2024-04-01 │ 2024-04-01 00:00:00 │ 2024-04-01 00:00:00.000 │ +└────────────────────────┴─────────────────────────────┴────────────────────────────────────┘ +``` + +## addMonths + +Adds a specified number of months to a date, a date with time or a string-encoded date / date with time. + +**Syntax** + +```sql +addMonths(date, num) +``` + +**Parameters** + +- `date`: Date / date with time to add specified number of months to. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md). +- `num`: Number of months to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). + +**Returned value** +- Returns `date` plus `num` months. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Example** + +```sql +WITH + toDate('2024-01-01') AS date, + toDateTime('2024-01-01 00:00:00') AS date_time, + '2024-01-01 00:00:00' AS date_time_string +SELECT + addMonths(date, 6) AS add_months_with_date, + addMonths(date_time, 6) AS add_months_with_date_time, + addMonths(date_time_string, 6) AS add_months_with_date_time_string +``` + +```response +┌─add_months_with_date─┬─add_months_with_date_time─┬─add_months_with_date_time_string─┐ +│ 2024-07-01 │ 2024-07-01 00:00:00 │ 2024-07-01 00:00:00.000 │ +└──────────────────────┴───────────────────────────┴──────────────────────────────────┘ +``` + +## addWeeks + +Adds a specified number of weeks to a date, a date with time or a string-encoded date / date with time. + +**Syntax** + +```sql +addWeeks(date, num) +``` + +**Parameters** + +- `date`: Date / date with time to add specified number of weeks to. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md). +- `num`: Number of weeks to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). + +**Returned value** +- Returns `date` plus `num` weeks. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Example** + +```sql +WITH + toDate('2024-01-01') AS date, + toDateTime('2024-01-01 00:00:00') AS date_time, + '2024-01-01 00:00:00' AS date_time_string +SELECT + addWeeks(date, 5) AS add_weeks_with_date, + addWeeks(date_time, 5) AS add_weeks_with_date_time, + addWeeks(date_time_string, 5) AS add_weeks_with_date_time_string +``` + +```response +┌─add_weeks_with_date─┬─add_weeks_with_date_time─┬─add_weeks_with_date_time_string─┐ +│ 2024-02-05 │ 2024-02-05 00:00:00 │ 2024-02-05 00:00:00.000 │ +└─────────────────────┴──────────────────────────┴─────────────────────────────────┘ +``` + +## addDays + +Adds a specified number of days to a date, a date with time or a string-encoded date / date with time. + +**Syntax** + +```sql +addDays(date, num) +``` + +**Parameters** + +- `date`: Date / date with time to add specified number of days to. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md). +- `num`: Number of days to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). + +**Returned value** +- Returns `date` plus `num` days. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Example** + +```sql +WITH + toDate('2024-01-01') AS date, + toDateTime('2024-01-01 00:00:00') AS date_time, + '2024-01-01 00:00:00' AS date_time_string +SELECT + addDays(date, 5) AS add_days_with_date, + addDays(date_time, 5) AS add_days_with_date_time, + addDays(date_time_string, 5) AS add_days_with_date_time_string +``` + +```response +┌─add_days_with_date─┬─add_days_with_date_time─┬─add_days_with_date_time_string─┐ +│ 2024-01-06 │ 2024-01-06 00:00:00 │ 2024-01-06 00:00:00.000 │ +└────────────────────┴─────────────────────────┴────────────────────────────────┘ +``` + +## addHours + +Adds a specified number of days to a date, a date with time or a string-encoded date / date with time. + +**Syntax** + +```sql +addHours(date, num) +``` + +**Parameters** + +- `date`: Date / date with time to add specified number of hours to. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md). +- `num`: Number of hours to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). + +**Returned value** +- Returns `date` plus `num` hours. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Example** + +```sql +WITH + toDate('2024-01-01') AS date, + toDateTime('2024-01-01 00:00:00') AS date_time, + '2024-01-01 00:00:00' AS date_time_string +SELECT + addHours(date, 12) AS add_hours_with_date, + addHours(date_time, 12) AS add_hours_with_date_time, + addHours(date_time_string, 12) AS add_hours_with_date_time_string +``` + +```response +┌─add_hours_with_date─┬─add_hours_with_date_time─┬─add_hours_with_date_time_string─┐ +│ 2024-01-01 12:00:00 │ 2024-01-01 12:00:00 │ 2024-01-01 12:00:00.000 │ +└─────────────────────┴──────────────────────────┴─────────────────────────────────┘ +``` + +## addMinutes + +Adds a specified number of minutes to a date, a date with time or a string-encoded date / date with time. + +**Syntax** + +```sql +addMinutes(date, num) +``` + +**Parameters** + +- `date`: Date / date with time to add specified number of minutes to. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md). +- `num`: Number of minutes to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). + +**Returned value** +- Returns `date` plus `num` minutes. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Example** + +```sql +WITH + toDate('2024-01-01') AS date, + toDateTime('2024-01-01 00:00:00') AS date_time, + '2024-01-01 00:00:00' AS date_time_string +SELECT + addMinutes(date, 20) AS add_minutes_with_date, + addMinutes(date_time, 20) AS add_minutes_with_date_time, + addMinutes(date_time_string, 20) AS add_minutes_with_date_time_string +``` + +```response +┌─add_minutes_with_date─┬─add_minutes_with_date_time─┬─add_minutes_with_date_time_string─┐ +│ 2024-01-01 00:20:00 │ 2024-01-01 00:20:00 │ 2024-01-01 00:20:00.000 │ +└───────────────────────┴────────────────────────────┴───────────────────────────────────┘ +``` + +## addSeconds + +Adds a specified number of seconds to a date, a date with time or a string-encoded date / date with time. + +**Syntax** + +```sql +addSeconds(date, num) +``` + +**Parameters** + +- `date`: Date / date with time to add specified number of seconds to. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md). +- `num`: Number of seconds to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). + +**Returned value** +- Returns `date` plus `num` seconds. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Example** + +```sql +WITH + toDate('2024-01-01') AS date, + toDateTime('2024-01-01 00:00:00') AS date_time, + '2024-01-01 00:00:00' AS date_time_string +SELECT + addSeconds(date, 30) AS add_seconds_with_date, + addSeconds(date_time, 30) AS add_seconds_with_date_time, + addSeconds(date_time_string, 30) AS add_seconds_with_date_time_string +``` + +```response +┌─add_seconds_with_date─┬─add_seconds_with_date_time─┬─add_seconds_with_date_time_string─┐ +│ 2024-01-01 00:00:30 │ 2024-01-01 00:00:30 │ 2024-01-01 00:00:30.000 │ +└───────────────────────┴────────────────────────────┴───────────────────────────────────┘ +``` + +## addMilliseconds + +Adds a specified number of milliseconds to a date with time or a string-encoded date with time. + +**Syntax** + +```sql +addMilliseconds(date_time, num) +``` + +**Parameters** + +- `date_time`: Date with time to add specified number of milliseconds to. [DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md). +- `num`: Number of milliseconds to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). + +**Returned value** +- Returns `date_time` plus `num` milliseconds. [DateTime64](../data-types/datetime64.md). + +**Example** + +```sql +WITH + toDateTime('2024-01-01 00:00:00') AS date_time, + '2024-01-01 00:00:00' AS date_time_string +SELECT + addMilliseconds(date_time, 1000) AS add_milliseconds_with_date_time, + addMilliseconds(date_time_string, 1000) AS add_milliseconds_with_date_time_string +``` + +```response +┌─add_milliseconds_with_date_time─┬─add_milliseconds_with_date_time_string─┐ +│ 2024-01-01 00:00:01.000 │ 2024-01-01 00:00:01.000 │ +└─────────────────────────────────┴────────────────────────────────────────┘ +``` + +## addMicroseconds + +Adds a specified number of microseconds to a date with time or a string-encoded date with time. + +**Syntax** + +```sql +addMicroseconds(date_time, num) +``` + +**Parameters** + +- `date_time`: Date with time to add specified number of microseconds to. [DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md). +- `num`: Number of microseconds to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). + +**Returned value** +- Returns `date_time` plus `num` microseconds. [DateTime64](../data-types/datetime64.md). + +**Example** + +```sql +WITH + toDateTime('2024-01-01 00:00:00') AS date_time, + '2024-01-01 00:00:00' AS date_time_string +SELECT + addMicroseconds(date_time, 1000000) AS add_microseconds_with_date_time, + addMicroseconds(date_time_string, 1000000) AS add_microseconds_with_date_time_string +``` + +```response +┌─add_microseconds_with_date_time─┬─add_microseconds_with_date_time_string─┐ +│ 2024-01-01 00:00:01.000000 │ 2024-01-01 00:00:01.000000 │ +└─────────────────────────────────┴────────────────────────────────────────┘ +``` + +## addNanoseconds + +Adds a specified number of microseconds to a date with time or a string-encoded date with time. + +**Syntax** + +```sql +addNanoseconds(date_time, num) +``` + +**Parameters** + +- `date_time`: Date with time to add specified number of nanoseconds to. [DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md). +- `num`: Number of nanoseconds to add. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). + +**Returned value** +- Returns `date_time` plus `num` nanoseconds. [DateTime64](../data-types/datetime64.md). + +**Example** + +```sql +WITH + toDateTime('2024-01-01 00:00:00') AS date_time, + '2024-01-01 00:00:00' AS date_time_string +SELECT + addNanoseconds(date_time, 1000) AS add_nanoseconds_with_date_time, + addNanoseconds(date_time_string, 1000) AS add_nanoseconds_with_date_time_string +``` + +```response +┌─add_nanoseconds_with_date_time─┬─add_nanoseconds_with_date_time_string─┐ +│ 2024-01-01 00:00:00.000001000 │ 2024-01-01 00:00:00.000001000 │ +└────────────────────────────────┴───────────────────────────────────────┘ +``` + +## addInterval + +Adds an interval to another interval or tuple of intervals. + +**Syntax** + +```sql +addInterval(interval_1, interval_2) +``` + +**Parameters** + +- `interval_1`: First interval or tuple of intervals. [interval](../data-types/special-data-types/interval.md), [tuple](../data-types/tuple.md)([interval](../data-types/special-data-types/interval.md)). +- `interval_2`: Second interval to be added. [interval](../data-types/special-data-types/interval.md). + +**Returned value** +- Returns a tuple of intervals. [tuple](../data-types/tuple.md)([interval](../data-types/special-data-types/interval.md)). + +:::note +Intervals of the same type will be combined into a single interval. For instance if `toIntervalDay(1)` and `toIntervalDay(2)` are passed then the result will be `(3)` rather than `(1,1)`. +::: + +**Example** + +Query: + +```sql +SELECT addInterval(INTERVAL 1 DAY, INTERVAL 1 MONTH); +SELECT addInterval((INTERVAL 1 DAY, INTERVAL 1 YEAR), INTERVAL 1 MONTH); +SELECT addInterval(INTERVAL 2 DAY, INTERVAL 1 DAY); +``` + +Result: + +```response +┌─addInterval(toIntervalDay(1), toIntervalMonth(1))─┐ +│ (1,1) │ +└───────────────────────────────────────────────────┘ +┌─addInterval((toIntervalDay(1), toIntervalYear(1)), toIntervalMonth(1))─┐ +│ (1,1,1) │ +└────────────────────────────────────────────────────────────────────────┘ +┌─addInterval(toIntervalDay(2), toIntervalDay(1))─┐ +│ (3) │ +└─────────────────────────────────────────────────┘ +``` + +## addTupleOfIntervals + +Consecutively adds a tuple of intervals to a Date or a DateTime. + +**Syntax** + +```sql +addTupleOfIntervals(interval_1, interval_2) +``` + +**Parameters** + +- `date`: First interval or interval of tuples. [date](../data-types/date.md)/[date32](../data-types/date32.md)/[datetime](../data-types/datetime.md)/[datetime64](../data-types/datetime64.md). +- `intervals`: Tuple of intervals to add to `date`. [tuple](../data-types/tuple.md)([interval](../data-types/special-data-types/interval.md)). + +**Returned value** +- Returns `date` with added `intervals`. [date](../data-types/date.md)/[date32](../data-types/date32.md)/[datetime](../data-types/datetime.md)/[datetime64](../data-types/datetime64.md). + +**Example** + +Query: + +```sql +WITH toDate('2018-01-01') AS date +SELECT addTupleOfIntervals(date, (INTERVAL 1 DAY, INTERVAL 1 MONTH, INTERVAL 1 YEAR)) +``` + +Result: + +```response +┌─addTupleOfIntervals(date, (toIntervalDay(1), toIntervalMonth(1), toIntervalYear(1)))─┐ +│ 2019-02-02 │ +└──────────────────────────────────────────────────────────────────────────────────────┘ +``` +## subtractYears + +Subtracts a specified number of years from a date, a date with time or a string-encoded date / date with time. + +**Syntax** + +```sql +subtractYears(date, num) +``` + +**Parameters** + +- `date`: Date / date with time to subtract specified number of years from. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md). +- `num`: Number of years to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). + +**Returned value** +- Returns `date` minus `num` years. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Example** + +```sql WITH toDate('2024-01-01') AS date, toDateTime('2024-01-01 00:00:00') AS date_time, @@ -2598,12 +3070,456 @@ SELECT subtractYears(date_time_string, 1) AS subtract_years_with_date_time_string ``` -``` text +```response ┌─subtract_years_with_date─┬─subtract_years_with_date_time─┬─subtract_years_with_date_time_string─┐ │ 2023-01-01 │ 2023-01-01 00:00:00 │ 2023-01-01 00:00:00.000 │ └──────────────────────────┴───────────────────────────────┴──────────────────────────────────────┘ ``` +## subtractQuarters + +Subtracts a specified number of quarters from a date, a date with time or a string-encoded date / date with time. + +**Syntax** + +```sql +subtractQuarters(date, num) +``` + +**Parameters** + +- `date`: Date / date with time to subtract specified number of quarters from. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md). +- `num`: Number of quarters to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). + +**Returned value** +- Returns `date` minus `num` quarters. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Example** + +```sql +WITH + toDate('2024-01-01') AS date, + toDateTime('2024-01-01 00:00:00') AS date_time, + '2024-01-01 00:00:00' AS date_time_string +SELECT + subtractQuarters(date, 1) AS subtract_quarters_with_date, + subtractQuarters(date_time, 1) AS subtract_quarters_with_date_time, + subtractQuarters(date_time_string, 1) AS subtract_quarters_with_date_time_string +``` + +```response +┌─subtract_quarters_with_date─┬─subtract_quarters_with_date_time─┬─subtract_quarters_with_date_time_string─┐ +│ 2023-10-01 │ 2023-10-01 00:00:00 │ 2023-10-01 00:00:00.000 │ +└─────────────────────────────┴──────────────────────────────────┴─────────────────────────────────────────┘ +``` + +## subtractMonths + +Subtracts a specified number of months from a date, a date with time or a string-encoded date / date with time. + +**Syntax** + +```sql +subtractMonths(date, num) +``` + +**Parameters** + +- `date`: Date / date with time to subtract specified number of months from. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md). +- `num`: Number of months to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). + +**Returned value** +- Returns `date` minus `num` months. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Example** + +```sql +WITH + toDate('2024-01-01') AS date, + toDateTime('2024-01-01 00:00:00') AS date_time, + '2024-01-01 00:00:00' AS date_time_string +SELECT + subtractMonths(date, 1) AS subtract_months_with_date, + subtractMonths(date_time, 1) AS subtract_months_with_date_time, + subtractMonths(date_time_string, 1) AS subtract_months_with_date_time_string +``` + +```response +┌─subtract_months_with_date─┬─subtract_months_with_date_time─┬─subtract_months_with_date_time_string─┐ +│ 2023-12-01 │ 2023-12-01 00:00:00 │ 2023-12-01 00:00:00.000 │ +└───────────────────────────┴────────────────────────────────┴───────────────────────────────────────┘ +``` + +## subtractWeeks + +Subtracts a specified number of weeks from a date, a date with time or a string-encoded date / date with time. + +**Syntax** + +```sql +subtractWeeks(date, num) +``` + +**Parameters** + +- `date`: Date / date with time to subtract specified number of weeks from. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md). +- `num`: Number of weeks to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). + +**Returned value** +- Returns `date` minus `num` weeks. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Example** + +```sql +WITH + toDate('2024-01-01') AS date, + toDateTime('2024-01-01 00:00:00') AS date_time, + '2024-01-01 00:00:00' AS date_time_string +SELECT + subtractWeeks(date, 1) AS subtract_weeks_with_date, + subtractWeeks(date_time, 1) AS subtract_weeks_with_date_time, + subtractWeeks(date_time_string, 1) AS subtract_weeks_with_date_time_string +``` + +```response + ┌─subtract_weeks_with_date─┬─subtract_weeks_with_date_time─┬─subtract_weeks_with_date_time_string─┐ + │ 2023-12-25 │ 2023-12-25 00:00:00 │ 2023-12-25 00:00:00.000 │ + └──────────────────────────┴───────────────────────────────┴──────────────────────────────────────┘ +``` + +## subtractDays + +Subtracts a specified number of days from a date, a date with time or a string-encoded date / date with time. + +**Syntax** + +```sql +subtractDays(date, num) +``` + +**Parameters** + +- `date`: Date / date with time to subtract specified number of days from. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md). +- `num`: Number of days to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). + +**Returned value** +- Returns `date` minus `num` days. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Example** + +```sql +WITH + toDate('2024-01-01') AS date, + toDateTime('2024-01-01 00:00:00') AS date_time, + '2024-01-01 00:00:00' AS date_time_string +SELECT + subtractDays(date, 31) AS subtract_days_with_date, + subtractDays(date_time, 31) AS subtract_days_with_date_time, + subtractDays(date_time_string, 31) AS subtract_days_with_date_time_string +``` + +```response +┌─subtract_days_with_date─┬─subtract_days_with_date_time─┬─subtract_days_with_date_time_string─┐ +│ 2023-12-01 │ 2023-12-01 00:00:00 │ 2023-12-01 00:00:00.000 │ +└─────────────────────────┴──────────────────────────────┴─────────────────────────────────────┘ +``` + +## subtractHours + +Subtracts a specified number of hours from a date, a date with time or a string-encoded date / date with time. + +**Syntax** + +```sql +subtractHours(date, num) +``` + +**Parameters** + +- `date`: Date / date with time to subtract specified number of hours from. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[Datetime](../data-types/datetime.md)/[Datetime64](../data-types/datetime64.md), [String](../data-types/string.md). +- `num`: Number of hours to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). + +**Returned value** +- Returns `date` minus `num` hours. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[Datetime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Example** + +```sql +WITH + toDate('2024-01-01') AS date, + toDateTime('2024-01-01 00:00:00') AS date_time, + '2024-01-01 00:00:00' AS date_time_string +SELECT + subtractHours(date, 12) AS subtract_hours_with_date, + subtractHours(date_time, 12) AS subtract_hours_with_date_time, + subtractHours(date_time_string, 12) AS subtract_hours_with_date_time_string +``` + +```response +┌─subtract_hours_with_date─┬─subtract_hours_with_date_time─┬─subtract_hours_with_date_time_string─┐ +│ 2023-12-31 12:00:00 │ 2023-12-31 12:00:00 │ 2023-12-31 12:00:00.000 │ +└──────────────────────────┴───────────────────────────────┴──────────────────────────────────────┘ +``` + +## subtractMinutes + +Subtracts a specified number of minutes from a date, a date with time or a string-encoded date / date with time. + +**Syntax** + +```sql +subtractMinutes(date, num) +``` + +**Parameters** + +- `date`: Date / date with time to subtract specified number of minutes from. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md). +- `num`: Number of minutes to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). + +**Returned value** +- Returns `date` minus `num` minutes. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Example** + +```sql +WITH + toDate('2024-01-01') AS date, + toDateTime('2024-01-01 00:00:00') AS date_time, + '2024-01-01 00:00:00' AS date_time_string +SELECT + subtractMinutes(date, 30) AS subtract_minutes_with_date, + subtractMinutes(date_time, 30) AS subtract_minutes_with_date_time, + subtractMinutes(date_time_string, 30) AS subtract_minutes_with_date_time_string +``` + +```response +┌─subtract_minutes_with_date─┬─subtract_minutes_with_date_time─┬─subtract_minutes_with_date_time_string─┐ +│ 2023-12-31 23:30:00 │ 2023-12-31 23:30:00 │ 2023-12-31 23:30:00.000 │ +└────────────────────────────┴─────────────────────────────────┴────────────────────────────────────────┘ +``` + +## subtractSeconds + +Subtracts a specified number of seconds from a date, a date with time or a string-encoded date / date with time. + +**Syntax** + +```sql +subtractSeconds(date, num) +``` + +**Parameters** + +- `date`: Date / date with time to subtract specified number of seconds from. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md). +- `num`: Number of seconds to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). + +**Returned value** +- Returns `date` minus `num` seconds. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Example** + +```sql +WITH + toDate('2024-01-01') AS date, + toDateTime('2024-01-01 00:00:00') AS date_time, + '2024-01-01 00:00:00' AS date_time_string +SELECT + subtractSeconds(date, 60) AS subtract_seconds_with_date, + subtractSeconds(date_time, 60) AS subtract_seconds_with_date_time, + subtractSeconds(date_time_string, 60) AS subtract_seconds_with_date_time_string +``` + +```response +┌─subtract_seconds_with_date─┬─subtract_seconds_with_date_time─┬─subtract_seconds_with_date_time_string─┐ +│ 2023-12-31 23:59:00 │ 2023-12-31 23:59:00 │ 2023-12-31 23:59:00.000 │ +└────────────────────────────┴─────────────────────────────────┴────────────────────────────────────────┘ +``` + +## subtractMilliseconds + +Subtracts a specified number of milliseconds from a date with time or a string-encoded date with time. + +**Syntax** + +```sql +subtractMilliseconds(date_time, num) +``` + +**Parameters** + +- `date_time`: Date with time to subtract specified number of milliseconds from. [DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md). +- `num`: Number of milliseconds to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). + +**Returned value** +- Returns `date_time` minus `num` milliseconds. [DateTime64](../data-types/datetime64.md). + +**Example** + +```sql +WITH + toDateTime('2024-01-01 00:00:00') AS date_time, + '2024-01-01 00:00:00' AS date_time_string +SELECT + subtractMilliseconds(date_time, 1000) AS subtract_milliseconds_with_date_time, + subtractMilliseconds(date_time_string, 1000) AS subtract_milliseconds_with_date_time_string +``` + +```response +┌─subtract_milliseconds_with_date_time─┬─subtract_milliseconds_with_date_time_string─┐ +│ 2023-12-31 23:59:59.000 │ 2023-12-31 23:59:59.000 │ +└──────────────────────────────────────┴─────────────────────────────────────────────┘ +``` + +## subtractMicroseconds + +Subtracts a specified number of microseconds from a date with time or a string-encoded date with time. + +**Syntax** + +```sql +subtractMicroseconds(date_time, num) +``` + +**Parameters** + +- `date_time`: Date with time to subtract specified number of microseconds from. [DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md). +- `num`: Number of microseconds to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). + +**Returned value** +- Returns `date_time` minus `num` microseconds. [DateTime64](../data-types/datetime64.md). + +**Example** + +```sql +WITH + toDateTime('2024-01-01 00:00:00') AS date_time, + '2024-01-01 00:00:00' AS date_time_string +SELECT + subtractMicroseconds(date_time, 1000000) AS subtract_microseconds_with_date_time, + subtractMicroseconds(date_time_string, 1000000) AS subtract_microseconds_with_date_time_string +``` + +```response +┌─subtract_microseconds_with_date_time─┬─subtract_microseconds_with_date_time_string─┐ +│ 2023-12-31 23:59:59.000000 │ 2023-12-31 23:59:59.000000 │ +└──────────────────────────────────────┴─────────────────────────────────────────────┘ +``` + +## subtractNanoseconds + +Subtracts a specified number of nanoseconds from a date with time or a string-encoded date with time. + +**Syntax** + +```sql +subtractNanoseconds(date_time, num) +``` + +**Parameters** + +- `date_time`: Date with time to subtract specified number of nanoseconds from. [DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md), [String](../data-types/string.md). +- `num`: Number of nanoseconds to subtract. [(U)Int*](../data-types/int-uint.md), [Float*](../data-types/float.md). + +**Returned value** +- Returns `date_time` minus `num` nanoseconds. [DateTime64](../data-types/datetime64.md). + +**Example** + +```sql +WITH + toDateTime('2024-01-01 00:00:00') AS date_time, + '2024-01-01 00:00:00' AS date_time_string +SELECT + subtractNanoseconds(date_time, 1000) AS subtract_nanoseconds_with_date_time, + subtractNanoseconds(date_time_string, 1000) AS subtract_nanoseconds_with_date_time_string +``` + +```response +┌─subtract_nanoseconds_with_date_time─┬─subtract_nanoseconds_with_date_time_string─┐ +│ 2023-12-31 23:59:59.999999000 │ 2023-12-31 23:59:59.999999000 │ +└─────────────────────────────────────┴────────────────────────────────────────────┘ +``` + +## subtractInterval + +Adds a negated interval to another interval or tuple of intervals. + +**Syntax** + +```sql +subtractInterval(interval_1, interval_2) +``` + +**Parameters** + +- `interval_1`: First interval or interval of tuples. [interval](../data-types/special-data-types/interval.md), [tuple](../data-types/tuple.md)([interval](../data-types/special-data-types/interval.md)). +- `interval_2`: Second interval to be negated. [interval](../data-types/special-data-types/interval.md). + +**Returned value** +- Returns a tuple of intervals. [tuple](../data-types/tuple.md)([interval](../data-types/special-data-types/interval.md)). + +:::note +Intervals of the same type will be combined into a single interval. For instance if `toIntervalDay(2)` and `toIntervalDay(1)` are passed then the result will be `(1)` rather than `(2,1)` +::: + +**Example** + +Query: + +```sql +SELECT subtractInterval(INTERVAL 1 DAY, INTERVAL 1 MONTH); +SELECT subtractInterval((INTERVAL 1 DAY, INTERVAL 1 YEAR), INTERVAL 1 MONTH); +SELECT subtractInterval(INTERVAL 2 DAY, INTERVAL 1 DAY); +``` + +Result: + +```response +┌─subtractInterval(toIntervalDay(1), toIntervalMonth(1))─┐ +│ (1,-1) │ +└────────────────────────────────────────────────────────┘ +┌─subtractInterval((toIntervalDay(1), toIntervalYear(1)), toIntervalMonth(1))─┐ +│ (1,1,-1) │ +└─────────────────────────────────────────────────────────────────────────────┘ +┌─subtractInterval(toIntervalDay(2), toIntervalDay(1))─┐ +│ (1) │ +└──────────────────────────────────────────────────────┘ +``` + +## subtractTupleOfIntervals + +Consecutively subtracts a tuple of intervals from a Date or a DateTime. + +**Syntax** + +```sql +subtractTupleOfIntervals(interval_1, interval_2) +``` + +**Parameters** + +- `date`: First interval or interval of tuples. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). +- `intervals`: Tuple of intervals to subtract from `date`. [tuple](../data-types/tuple.md)([interval](../data-types/special-data-types/interval.md)). + +**Returned value** +- Returns `date` with subtracted `intervals`. [Date](../data-types/date.md)/[Date32](../data-types/date32.md)/[DateTime](../data-types/datetime.md)/[DateTime64](../data-types/datetime64.md). + +**Example** + +Query: + +```sql +WITH toDate('2018-01-01') AS date SELECT subtractTupleOfIntervals(date, (INTERVAL 1 DAY, INTERVAL 1 YEAR)) +``` + +Result: + +```response +┌─subtractTupleOfIntervals(date, (toIntervalDay(1), toIntervalYear(1)))─┐ +│ 2016-12-31 │ +└───────────────────────────────────────────────────────────────────────┘ +``` + ## timeSlots(StartTime, Duration,\[, Size\]) For a time interval starting at ‘StartTime’ and continuing for ‘Duration’ seconds, it returns an array of moments in time, consisting of points from this interval rounded down to the ‘Size’ in seconds. ‘Size’ is an optional parameter set to 1800 (30 minutes) by default. diff --git a/docs/en/sql-reference/functions/functions-for-nulls.md b/docs/en/sql-reference/functions/functions-for-nulls.md index 4dfbf4262ed..90520145b9d 100644 --- a/docs/en/sql-reference/functions/functions-for-nulls.md +++ b/docs/en/sql-reference/functions/functions-for-nulls.md @@ -12,6 +12,8 @@ Returns whether the argument is [NULL](../../sql-reference/syntax.md#null). See also operator [`IS NULL`](../operators/index.md#is_null). +**Syntax** + ``` sql isNull(x) ``` @@ -52,6 +54,45 @@ Result: └───┘ ``` +## isNullable + +Returns `1` if a column is [Nullable](../data-types/nullable.md) (i.e allows `NULL` values), `0` otherwise. + +**Syntax** + +``` sql +isNullable(x) +``` + +**Arguments** + +- `x` — column. + +**Returned value** + +- `1` if `x` allows `NULL` values. [UInt8](../data-types/int-uint.md). +- `0` if `x` does not allow `NULL` values. [UInt8](../data-types/int-uint.md). + +**Example** + +Query: + +``` sql +CREATE TABLE tab (ordinary_col UInt32, nullable_col Nullable(UInt32)) ENGINE = Log; +INSERT INTO tab (ordinary_col, nullable_col) VALUES (1,1), (2, 2), (3,3); +SELECT isNullable(ordinary_col), isNullable(nullable_col) FROM tab; +``` + +Result: + +``` text + ┌───isNullable(ordinary_col)──┬───isNullable(nullable_col)──┐ +1. │ 0 │ 1 │ +2. │ 0 │ 1 │ +3. │ 0 │ 1 │ + └─────────────────────────────┴─────────────────────────────┘ +``` + ## isNotNull Returns whether the argument is not [NULL](../../sql-reference/syntax.md#null-literal). @@ -96,6 +137,36 @@ Result: └───┘ ``` +## isNotDistinctFrom + +Performs null-safe comparison. Used to compare JOIN keys which contain NULL values in the JOIN ON section. +This function will consider two `NULL` values as identical and will return `true`, which is distinct from the usual +equals behavior where comparing two `NULL` values would return `NULL`. + +:::note +This function is an internal function used by the implementation of JOIN ON. Please do not use it manually in queries. +::: + +**Syntax** + +``` sql +isNotDistinctFrom(x, y) +``` + +**Arguments** + +- `x` — first JOIN key. +- `y` — second JOIN key. + +**Returned value** + +- `true` when `x` and `y` are both `NULL`. +- `false` otherwise. + +**Example** + +For a complete example see: [NULL values in JOIN keys](../../sql-reference/statements/select/join#null-values-in-join-keys). + ## isZeroOrNull Returns whether the argument is 0 (zero) or [NULL](../../sql-reference/syntax.md#null-literal). diff --git a/docs/en/sql-reference/functions/math-functions.md b/docs/en/sql-reference/functions/math-functions.md index 5ebc6191010..945166056af 100644 --- a/docs/en/sql-reference/functions/math-functions.md +++ b/docs/en/sql-reference/functions/math-functions.md @@ -792,6 +792,39 @@ Result: │ -1 │ └──────────┘ ``` +## sigmoid + +Returns the [sigmoid function](https://en.wikipedia.org/wiki/Sigmoid_function). + +**Syntax** + +```sql +sigmoid(x) +``` + +**Parameters** + +- `x` — input value. Values from the interval: `-∞ < x < +∞`. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md). + +**Returned value** + +- Corresponding value along the sigmoid curve between 0 and 1. [Float64](../../sql-reference/data-types/float.md). + +**Example** + +Query: + +``` sql +SELECT round(sigmoid(x), 5) FROM (SELECT arrayJoin([-1, 0, 1]) AS x); +``` + +Result: + +```result +0.26894 +0.5 +0.73106 +``` ## degrees diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index e6703b573cb..ba23870a584 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -1058,7 +1058,7 @@ convertCharset(s, from, to) ## base58Encode -Encodes a String using [Base58](https://tools.ietf.org/id/draft-msporny-base58-01.html) in the "Bitcoin" alphabet. +Encodes a String using [Base58](https://datatracker.ietf.org/doc/html/draft-msporny-base58) in the "Bitcoin" alphabet. **Syntax** @@ -1092,7 +1092,7 @@ Result: ## base58Decode -Accepts a String and decodes it using [Base58](https://tools.ietf.org/id/draft-msporny-base58-01.html) encoding scheme using "Bitcoin" alphabet. +Accepts a String and decodes it using [Base58](https://datatracker.ietf.org/doc/html/draft-msporny-base58) encoding scheme using "Bitcoin" alphabet. **Syntax** diff --git a/docs/en/sql-reference/functions/tuple-map-functions.md b/docs/en/sql-reference/functions/tuple-map-functions.md index 8283de95994..377283bc006 100644 --- a/docs/en/sql-reference/functions/tuple-map-functions.md +++ b/docs/en/sql-reference/functions/tuple-map-functions.md @@ -16,7 +16,7 @@ map(key1, value1[, key2, value2, ...]) **Arguments** -- `key` — The key part of the pair. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md), [LowCardinality](../../sql-reference/data-types/lowcardinality.md), [FixedString](../../sql-reference/data-types/fixedstring.md), [UUID](../../sql-reference/data-types/uuid.md), [Date](../../sql-reference/data-types/date.md), [DateTime](../../sql-reference/data-types/datetime.md), [Date32](../../sql-reference/data-types/date32.md), [Enum](../../sql-reference/data-types/enum.md). +- `key` — The key part of the pair. Arbitrary type, except [Nullable](../../sql-reference/data-types/nullable.md) and [LowCardinality](../../sql-reference/data-types/lowcardinality.md) nested with [Nullable](../../sql-reference/data-types/nullable.md). - `value` — The value part of the pair. Arbitrary type, including [Map](../../sql-reference/data-types/map.md) and [Array](../../sql-reference/data-types/array.md). **Returned value** diff --git a/docs/en/sql-reference/statements/select/join.md b/docs/en/sql-reference/statements/select/join.md index 4ef407a4d13..34c6016235a 100644 --- a/docs/en/sql-reference/statements/select/join.md +++ b/docs/en/sql-reference/statements/select/join.md @@ -151,6 +151,14 @@ Result: Query with `INNER` type of a join and conditions with `OR` and `AND`: +:::note + +By default, non-equal conditions are supported as long as they use columns from the same table. +For example, `t1.a = t2.key AND t1.b > 0 AND t2.b > t2.c`, because `t1.b > 0` uses columns only from `t1` and `t2.b > t2.c` uses columns only from `t2`. +However, you can try experimental support for conditions like `t1.a = t2.key AND t1.b > t2.key`, check out section below for more details. + +::: + ``` sql SELECT a, b, val FROM t1 INNER JOIN t2 ON t1.a = t2.key OR t1.b = t2.key AND t2.val > 3; ``` @@ -165,7 +173,7 @@ Result: └───┴────┴─────┘ ``` -## [experimental] Join with inequality conditions +## [experimental] Join with inequality conditions for columns from different tables :::note This feature is experimental. To use it, set `allow_experimental_join_condition` to 1 in your configuration files or by using the `SET` command: diff --git a/docs/en/sql-reference/statements/select/with.md b/docs/en/sql-reference/statements/select/with.md index a59ef463419..aa0e0c61c4e 100644 --- a/docs/en/sql-reference/statements/select/with.md +++ b/docs/en/sql-reference/statements/select/with.md @@ -5,21 +5,21 @@ sidebar_label: WITH # WITH Clause -ClickHouse supports Common Table Expressions ([CTE](https://en.wikipedia.org/wiki/Hierarchical_and_recursive_queries_in_SQL)) and substitutes the code defined in the `WITH` clause in all places of use for the rest of `SELECT` query. Named subqueries can be included to the current and child query context in places where table objects are allowed. Recursion is prevented by hiding the current level CTEs from the WITH expression. +ClickHouse supports Common Table Expressions ([CTE](https://en.wikipedia.org/wiki/Hierarchical_and_recursive_queries_in_SQL)) and substitutes the code defined in the `WITH` clause in all places of use for the rest of `SELECT` query. Named subqueries can be included to the current and child query context in places where table objects are allowed. Recursion is prevented by hiding the current level CTEs from the WITH expression. Please note that CTEs do not guarantee the same results in all places they are called because the query will be re-executed for each use case. An example of such behavior is below ``` sql -with cte_numbers as +with cte_numbers as ( - select - num - from generateRandom('num UInt64', NULL) + select + num + from generateRandom('num UInt64', NULL) limit 1000000 ) select - count() + count() from cte_numbers where num in (select num from cte_numbers) ``` @@ -87,3 +87,226 @@ LIMIT 10; WITH test1 AS (SELECT i + 1, j + 1 FROM test1) SELECT * FROM test1; ``` + +## Recursive Queries + +The optional RECURSIVE modifier allows for a WITH query to refer to its own output. Example: + +**Example:** Sum integers from 1 through 100 + +```sql +WITH RECURSIVE test_table AS ( + SELECT 1 AS number +UNION ALL + SELECT number + 1 FROM test_table WHERE number < 100 +) +SELECT sum(number) FROM test_table; +``` + +``` text +┌─sum(number)─┐ +│ 5050 │ +└─────────────┘ +``` + +The general form of a recursive `WITH` query is always a non-recursive term, then `UNION ALL`, then a recursive term, where only the recursive term can contain a reference to the query's own output. Recursive CTE query is executed as follows: + +1. Evaluate the non-recursive term. Place result of non-recursive term query in a temporary working table. +2. As long as the working table is not empty, repeat these steps: + 1. Evaluate the recursive term, substituting the current contents of the working table for the recursive self-reference. Place result of recursive term query in a temporary intermediate table. + 2. Replace the contents of the working table with the contents of the intermediate table, then empty the intermediate table. + +Recursive queries are typically used to work with hierarchical or tree-structured data. For example, we can write a query that performs tree traversal: + +**Example:** Tree traversal + +First let's create tree table: + +```sql +DROP TABLE IF EXISTS tree; +CREATE TABLE tree +( + id UInt64, + parent_id Nullable(UInt64), + data String +) ENGINE = MergeTree ORDER BY id; + +INSERT INTO tree VALUES (0, NULL, 'ROOT'), (1, 0, 'Child_1'), (2, 0, 'Child_2'), (3, 1, 'Child_1_1'); +``` + +We can traverse those tree with such query: + +**Example:** Tree traversal +```sql +WITH RECURSIVE search_tree AS ( + SELECT id, parent_id, data + FROM tree t + WHERE t.id = 0 +UNION ALL + SELECT t.id, t.parent_id, t.data + FROM tree t, search_tree st + WHERE t.parent_id = st.id +) +SELECT * FROM search_tree; +``` + +```text +┌─id─┬─parent_id─┬─data──────┐ +│ 0 │ ᴺᵁᴸᴸ │ ROOT │ +│ 1 │ 0 │ Child_1 │ +│ 2 │ 0 │ Child_2 │ +│ 3 │ 1 │ Child_1_1 │ +└────┴───────────┴───────────┘ +``` + +### Search order + +To create a depth-first order, we compute for each result row an array of rows that we have already visited: + +**Example:** Tree traversal depth-first order +```sql +WITH RECURSIVE search_tree AS ( + SELECT id, parent_id, data, [t.id] AS path + FROM tree t + WHERE t.id = 0 +UNION ALL + SELECT t.id, t.parent_id, t.data, arrayConcat(path, [t.id]) + FROM tree t, search_tree st + WHERE t.parent_id = st.id +) +SELECT * FROM search_tree ORDER BY path; +``` + +```text +┌─id─┬─parent_id─┬─data──────┬─path────┐ +│ 0 │ ᴺᵁᴸᴸ │ ROOT │ [0] │ +│ 1 │ 0 │ Child_1 │ [0,1] │ +│ 3 │ 1 │ Child_1_1 │ [0,1,3] │ +│ 2 │ 0 │ Child_2 │ [0,2] │ +└────┴───────────┴───────────┴─────────┘ +``` + +To create a breadth-first order, standard approach is to add column that tracks the depth of the search: + +**Example:** Tree traversal breadth-first order +```sql +WITH RECURSIVE search_tree AS ( + SELECT id, parent_id, data, [t.id] AS path, toUInt64(0) AS depth + FROM tree t + WHERE t.id = 0 +UNION ALL + SELECT t.id, t.parent_id, t.data, arrayConcat(path, [t.id]), depth + 1 + FROM tree t, search_tree st + WHERE t.parent_id = st.id +) +SELECT * FROM search_tree ORDER BY depth; +``` + +```text +┌─id─┬─link─┬─data──────┬─path────┬─depth─┐ +│ 0 │ ᴺᵁᴸᴸ │ ROOT │ [0] │ 0 │ +│ 1 │ 0 │ Child_1 │ [0,1] │ 1 │ +│ 2 │ 0 │ Child_2 │ [0,2] │ 1 │ +│ 3 │ 1 │ Child_1_1 │ [0,1,3] │ 2 │ +└────┴──────┴───────────┴─────────┴───────┘ +``` + +### Cycle detection + +First let's create graph table: + +```sql +DROP TABLE IF EXISTS graph; +CREATE TABLE graph +( + from UInt64, + to UInt64, + label String +) ENGINE = MergeTree ORDER BY (from, to); + +INSERT INTO graph VALUES (1, 2, '1 -> 2'), (1, 3, '1 -> 3'), (2, 3, '2 -> 3'), (1, 4, '1 -> 4'), (4, 5, '4 -> 5'); +``` + +We can traverse that graph with such query: + +**Example:** Graph traversal without cycle detection +```sql +WITH RECURSIVE search_graph AS ( + SELECT from, to, label FROM graph g + UNION ALL + SELECT g.from, g.to, g.label + FROM graph g, search_graph sg + WHERE g.from = sg.to +) +SELECT DISTINCT * FROM search_graph ORDER BY from; +``` +```text +┌─from─┬─to─┬─label──┐ +│ 1 │ 4 │ 1 -> 4 │ +│ 1 │ 2 │ 1 -> 2 │ +│ 1 │ 3 │ 1 -> 3 │ +│ 2 │ 3 │ 2 -> 3 │ +│ 4 │ 5 │ 4 -> 5 │ +└──────┴────┴────────┘ +``` + +But if we add cycle in that graph, previous query will fail with `Maximum recursive CTE evaluation depth` error: + +```sql +INSERT INTO graph VALUES (5, 1, '5 -> 1'); + +WITH RECURSIVE search_graph AS ( + SELECT from, to, label FROM graph g +UNION ALL + SELECT g.from, g.to, g.label + FROM graph g, search_graph sg + WHERE g.from = sg.to +) +SELECT DISTINCT * FROM search_graph ORDER BY from; +``` + +```text +Code: 306. DB::Exception: Received from localhost:9000. DB::Exception: Maximum recursive CTE evaluation depth (1000) exceeded, during evaluation of search_graph AS (SELECT from, to, label FROM graph AS g UNION ALL SELECT g.from, g.to, g.label FROM graph AS g, search_graph AS sg WHERE g.from = sg.to). Consider raising max_recursive_cte_evaluation_depth setting.: While executing RecursiveCTESource. (TOO_DEEP_RECURSION) +``` + +The standard method for handling cycles is to compute an array of the already visited nodes: + +**Example:** Graph traversal with cycle detection +```sql +WITH RECURSIVE search_graph AS ( + SELECT from, to, label, false AS is_cycle, [tuple(g.from, g.to)] AS path FROM graph g +UNION ALL + SELECT g.from, g.to, g.label, has(path, tuple(g.from, g.to)), arrayConcat(sg.path, [tuple(g.from, g.to)]) + FROM graph g, search_graph sg + WHERE g.from = sg.to AND NOT is_cycle +) +SELECT * FROM search_graph WHERE is_cycle ORDER BY from; +``` + +```text +┌─from─┬─to─┬─label──┬─is_cycle─┬─path──────────────────────┐ +│ 1 │ 4 │ 1 -> 4 │ true │ [(1,4),(4,5),(5,1),(1,4)] │ +│ 4 │ 5 │ 4 -> 5 │ true │ [(4,5),(5,1),(1,4),(4,5)] │ +│ 5 │ 1 │ 5 -> 1 │ true │ [(5,1),(1,4),(4,5),(5,1)] │ +└──────┴────┴────────┴──────────┴───────────────────────────┘ +``` + +### Infinite queries + +It is also possible to use infinite recursive CTE queries if `LIMIT` is used in outer query: + +**Example:** Infinite recursive CTE query +```sql +WITH RECURSIVE test_table AS ( + SELECT 1 AS number +UNION ALL + SELECT number + 1 FROM test_table +) +SELECT sum(number) FROM (SELECT number FROM test_table LIMIT 100); +``` + +```text +┌─sum(number)─┐ +│ 5050 │ +└─────────────┘ +``` diff --git a/docs/en/sql-reference/statements/truncate.md b/docs/en/sql-reference/statements/truncate.md index 8cd5a6a1424..fb5d4a3055a 100644 --- a/docs/en/sql-reference/statements/truncate.md +++ b/docs/en/sql-reference/statements/truncate.md @@ -25,7 +25,7 @@ If the `alter_sync` is set to `2` and some replicas are not active for more than ## TRUNCATE ALL TABLES ``` sql -TRUNCATE ALL TABLES [IF EXISTS] db [ON CLUSTER cluster] +TRUNCATE ALL TABLES FROM [IF EXISTS] db [ON CLUSTER cluster] ``` Removes all data from all tables in a database. diff --git a/docs/ru/sql-reference/functions/null-functions.md b/docs/ru/sql-reference/functions/null-functions.md new file mode 100644 index 00000000000..7934b0f68c9 --- /dev/null +++ b/docs/ru/sql-reference/functions/null-functions.md @@ -0,0 +1,311 @@ +--- +slug: /ru/sql-reference/functions/functions-for-nulls +sidebar_position: 63 +sidebar_label: "Функции для работы с Nullable-аргументами" +--- + +# Функции для работы с Nullable-аргументами {#funktsii-dlia-raboty-s-nullable-argumentami} + +## isNull {#isnull} + +Проверяет является ли аргумент [NULL](../../sql-reference/syntax.md#null-literal). + +``` sql +isNull(x) +``` + +Синоним: `ISNULL`. + +**Аргументы** + +- `x` — значение с не составным типом данных. + +**Возвращаемое значение** + +- `1`, если `x` — `NULL`. +- `0`, если `x` — не `NULL`. + +**Пример** + +Входная таблица + +``` text +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +│ 2 │ 3 │ +└───┴──────┘ +``` + +Запрос + +``` sql +SELECT x FROM t_null WHERE isNull(y); +``` + +``` text +┌─x─┐ +│ 1 │ +└───┘ +``` + +## isNotNull {#isnotnull} + +Проверяет не является ли аргумент [NULL](../../sql-reference/syntax.md#null-literal). + +``` sql +isNotNull(x) +``` + +**Аргументы** + +- `x` — значение с не составным типом данных. + +**Возвращаемое значение** + +- `0`, если `x` — `NULL`. +- `1`, если `x` — не `NULL`. + +**Пример** + +Входная таблица + +``` text +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +│ 2 │ 3 │ +└───┴──────┘ +``` + +Запрос + +``` sql +SELECT x FROM t_null WHERE isNotNull(y); +``` + +``` text +┌─x─┐ +│ 2 │ +└───┘ +``` + +## coalesce {#coalesce} + +Последовательно слева-направо проверяет являются ли переданные аргументы `NULL` и возвращает первый не `NULL`. + +``` sql +coalesce(x,...) +``` + +**Аргументы** + +- Произвольное количество параметров не составного типа. Все параметры должны быть совместимы по типу данных. + +**Возвращаемые значения** + +- Первый не `NULL` аргумент. +- `NULL`, если все аргументы — `NULL`. + +**Пример** + +Рассмотрим адресную книгу, в которой может быть указано несколько способов связи с клиентом. + +``` text +┌─name─────┬─mail─┬─phone─────┬──icq─┐ +│ client 1 │ ᴺᵁᴸᴸ │ 123-45-67 │ 123 │ +│ client 2 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ +└──────────┴──────┴───────────┴──────┘ +``` + +Поля `mail` и `phone` имеют тип String, а поле `icq` — `UInt32`, его необходимо будет преобразовать в `String`. + +Получим из адресной книги первый доступный способ связаться с клиентом: + +``` sql +SELECT coalesce(mail, phone, CAST(icq,'Nullable(String)')) FROM aBook; +``` + +``` text +┌─name─────┬─coalesce(mail, phone, CAST(icq, 'Nullable(String)'))─┐ +│ client 1 │ 123-45-67 │ +│ client 2 │ ᴺᵁᴸᴸ │ +└──────────┴──────────────────────────────────────────────────────┘ +``` + +## ifNull {#ifnull} + +Возвращает альтернативное значение, если основной аргумент — `NULL`. + +``` sql +ifNull(x,alt) +``` + +**Аргументы** + +- `x` — значение для проверки на `NULL`, +- `alt` — значение, которое функция вернёт, если `x` — `NULL`. + +**Возвращаемые значения** + +- Значение `x`, если `x` — не `NULL`. +- Значение `alt`, если `x` — `NULL`. + +**Пример** + +``` sql +SELECT ifNull('a', 'b'); +``` + +``` text +┌─ifNull('a', 'b')─┐ +│ a │ +└──────────────────┘ +``` + +``` sql +SELECT ifNull(NULL, 'b'); +``` + +``` text +┌─ifNull(NULL, 'b')─┐ +│ b │ +└───────────────────┘ +``` + +## nullIf {#nullif} + +Возвращает `NULL`, если аргументы равны. + +``` sql +nullIf(x, y) +``` + +**Аргументы** + +`x`, `y` — значения для сравнивания. Они должны быть совместимых типов, иначе ClickHouse сгенерирует исключение. + +**Возвращаемые значения** + +- `NULL`, если аргументы равны. +- Значение `x`, если аргументы не равны. + +**Пример** + +``` sql +SELECT nullIf(1, 1); +``` + +``` text +┌─nullIf(1, 1)─┐ +│ ᴺᵁᴸᴸ │ +└──────────────┘ +``` + +``` sql +SELECT nullIf(1, 2); +``` + +``` text +┌─nullIf(1, 2)─┐ +│ 1 │ +└──────────────┘ +``` + +## assumeNotNull {#assumenotnull} + +Приводит значение типа [Nullable](../../sql-reference/functions/functions-for-nulls.md) к не `Nullable`, если значение не `NULL`. + +``` sql +assumeNotNull(x) +``` + +**Аргументы** + +- `x` — исходное значение. + +**Возвращаемые значения** + +- Исходное значение с не `Nullable` типом, если оно — не `NULL`. +- Неспецифицированный результат, зависящий от реализации, если исходное значение — `NULL`. + +**Пример** + +Рассмотрим таблицу `t_null`. + +``` sql +SHOW CREATE TABLE t_null; +``` + +``` text +┌─statement─────────────────────────────────────────────────────────────────┐ +│ CREATE TABLE default.t_null ( x Int8, y Nullable(Int8)) ENGINE = TinyLog │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +``` text +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +│ 2 │ 3 │ +└───┴──────┘ +``` + +Применим функцию `assumeNotNull` к столбцу `y`. + +``` sql +SELECT assumeNotNull(y) FROM t_null; +``` + +``` text +┌─assumeNotNull(y)─┐ +│ 0 │ +│ 3 │ +└──────────────────┘ +``` + +``` sql +SELECT toTypeName(assumeNotNull(y)) FROM t_null; +``` + +``` text +┌─toTypeName(assumeNotNull(y))─┐ +│ Int8 │ +│ Int8 │ +└──────────────────────────────┘ +``` + +## toNullable {#tonullable} + +Преобразует тип аргумента к `Nullable`. + +``` sql +toNullable(x) +``` + +**Аргументы** + +- `x` — значение произвольного не составного типа. + +**Возвращаемое значение** + +- Входное значение с типом не `Nullable`. + +**Пример** + +``` sql +SELECT toTypeName(10); +``` + +``` text +┌─toTypeName(10)─┐ +│ UInt8 │ +└────────────────┘ +``` + +``` sql +SELECT toTypeName(toNullable(10)); +``` + +``` text +┌─toTypeName(toNullable(10))─┐ +│ Nullable(UInt8) │ +└────────────────────────────┘ +``` diff --git a/docs/ru/sql-reference/functions/string-functions.md b/docs/ru/sql-reference/functions/string-functions.md index 276dfc2ef20..eeb5752c626 100644 --- a/docs/ru/sql-reference/functions/string-functions.md +++ b/docs/ru/sql-reference/functions/string-functions.md @@ -493,7 +493,7 @@ SELECT concat(key1, key2), sum(value) FROM key_val GROUP BY (key1, key2); ## base58Encode(plaintext), base58Decode(encoded_text) {#base58} -Принимает на вход строку или колонку строк и кодирует/раскодирует их с помощью схемы кодирования [Base58](https://tools.ietf.org/id/draft-msporny-base58-01.html) с использованием стандартного алфавита Bitcoin. +Принимает на вход строку или колонку строк и кодирует/раскодирует их с помощью схемы кодирования [Base58](https://datatracker.ietf.org/doc/html/draft-msporny-base58) с использованием стандартного алфавита Bitcoin. **Синтаксис** diff --git a/docs/zh/sql-reference/functions/null-functions.md b/docs/zh/sql-reference/functions/null-functions.md new file mode 100644 index 00000000000..b3dca3ac549 --- /dev/null +++ b/docs/zh/sql-reference/functions/null-functions.md @@ -0,0 +1,254 @@ +--- +slug: /zh/sql-reference/functions/functions-for-nulls +--- +# Nullable处理函数 {#nullablechu-li-han-shu} + +## isNull {#isnull} + +检查参数是否为[NULL](../../sql-reference/syntax.md#null-literal)。 + + isNull(x) + +**参数** + +- `x` — 一个非复合数据类型的值。 + +**返回值** + +- `1` 如果`x`为`NULL`。 +- `0` 如果`x`不为`NULL`。 + +**示例** + +存在以下内容的表 + +```response +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +│ 2 │ 3 │ +└───┴──────┘ +``` + +对其进行查询 + +```sql +SELECT x FROM t_null WHERE isNull(y) +``` + +```response +┌─x─┐ +│ 1 │ +└───┘ +``` + +## isNotNull {#isnotnull} + +检查参数是否不为 [NULL](../../sql-reference/syntax.md#null-literal). + + isNotNull(x) + +**参数:** + +- `x` — 一个非复合数据类型的值。 + +**返回值** + +- `0` 如果`x`为`NULL`。 +- `1` 如果`x`不为`NULL`。 + +**示例** + +存在以下内容的表 + +```response +┌─x─┬────y─┐ +│ 1 │ ᴺᵁᴸᴸ │ +│ 2 │ 3 │ +└───┴──────┘ +``` + +对其进行查询 + +```sql +SELECT x FROM t_null WHERE isNotNull(y) +``` + +```response +┌─x─┐ +│ 2 │ +└───┘ +``` + +## 合并 {#coalesce} + +检查从左到右是否传递了«NULL»参数并返回第一个非`'NULL`参数。 + + coalesce(x,...) + +**参数:** + +- 任何数量的非复合类型的参数。所有参数必须与数据类型兼容。 + +**返回值** + +- 第一个非’NULL\`参数。 +- `NULL`，如果所有参数都是’NULL\`。 + +**示例** + +考虑可以指定多种联系客户的方式的联系人列表。 + +```response +┌─name─────┬─mail─┬─phone─────┬──icq─┐ +│ client 1 │ ᴺᵁᴸᴸ │ 123-45-67 │ 123 │ +│ client 2 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ +└──────────┴──────┴───────────┴──────┘ +``` + +`mail`和`phone`字段是String类型，但`icq`字段是`UInt32`，所以它需要转换为`String`。 + +从联系人列表中获取客户的第一个可用联系方式： + +```sql +SELECT coalesce(mail, phone, CAST(icq,'Nullable(String)')) FROM aBook +``` + +```response +┌─name─────┬─coalesce(mail, phone, CAST(icq, 'Nullable(String)'))─┐ +│ client 1 │ 123-45-67 │ +│ client 2 │ ᴺᵁᴸᴸ │ +└──────────┴──────────────────────────────────────────────────────┘ +``` + +## ifNull {#ifnull} + +如果第一个参数为«NULL»，则返回第二个参数的值。 + + ifNull(x,alt) + +**参数:** + +- `x` — 要检查«NULL»的值。 +- `alt` — 如果`x`为’NULL\`，函数返回的值。 + +**返回值** + +- 价值 `x`，如果 `x` 不是 `NULL`. +- 价值 `alt`，如果 `x` 是 `NULL`. + +**示例** + + SELECT ifNull('a', 'b') + + ┌─ifNull('a', 'b')─┐ + │ a │ + └──────────────────┘ + + SELECT ifNull(NULL, 'b') + + ┌─ifNull(NULL, 'b')─┐ + │ b │ + └───────────────────┘ + +## nullIf {#nullif} + +如果参数相等，则返回`NULL`。 + + nullIf(x, y) + +**参数:** + +`x`, `y` — 用于比较的值。 它们必须是类型兼容的，否则将抛出异常。 + +**返回值** + +- 如果参数相等，则为`NULL`。 +- 如果参数不相等，则为`x`值。 + +**示例** + + SELECT nullIf(1, 1) + + ┌─nullIf(1, 1)─┐ + │ ᴺᵁᴸᴸ │ + └──────────────┘ + + SELECT nullIf(1, 2) + + ┌─nullIf(1, 2)─┐ + │ 1 │ + └──────────────┘ + +## assumeNotNull {#assumenotnull} + +将[可为空](../../sql-reference/functions/functions-for-nulls.md)类型的值转换为非`Nullable`类型的值。 + + assumeNotNull(x) + +**参数：** + +- `x` — 原始值。 + +**返回值** + +- 如果`x`不为`NULL`，返回非`Nullable`类型的原始值。 +- 如果`x`为`NULL`，则返回任意值。 + +**示例** + +存在如下`t_null`表。 + + SHOW CREATE TABLE t_null + + ┌─statement─────────────────────────────────────────────────────────────────┐ + │ CREATE TABLE default.t_null ( x Int8, y Nullable(Int8)) ENGINE = TinyLog │ + └───────────────────────────────────────────────────────────────────────────┘ + + ┌─x─┬────y─┐ + │ 1 │ ᴺᵁᴸᴸ │ + │ 2 │ 3 │ + └───┴──────┘ + +将列`y`作为`assumeNotNull`函数的参数。 + + SELECT assumeNotNull(y) FROM t_null + + ┌─assumeNotNull(y)─┐ + │ 0 │ + │ 3 │ + └──────────────────┘ + + SELECT toTypeName(assumeNotNull(y)) FROM t_null + + ┌─toTypeName(assumeNotNull(y))─┐ + │ Int8 │ + │ Int8 │ + └──────────────────────────────┘ + +## 可调整 {#tonullable} + +将参数的类型转换为`Nullable`。 + + toNullable(x) + +**参数：** + +- `x` — 任何非复合类型的值。 + +**返回值** + +- 输入的值，但其类型为`Nullable`。 + +**示例** + + SELECT toTypeName(10) + + ┌─toTypeName(10)─┐ + │ UInt8 │ + └────────────────┘ + + SELECT toTypeName(toNullable(10)) + + ┌─toTypeName(toNullable(10))─┐ + │ Nullable(UInt8) │ + └────────────────────────────┘ diff --git a/programs/git-import/git-import.cpp b/programs/git-import/git-import.cpp index fdabeacd46e..eaf85df67b1 100644 --- a/programs/git-import/git-import.cpp +++ b/programs/git-import/git-import.cpp @@ -233,7 +233,7 @@ struct Commit }; -enum class FileChangeType +enum class FileChangeType : uint8_t { Add, Delete, @@ -291,7 +291,7 @@ struct FileChange }; -enum class LineType +enum class LineType : uint8_t { Empty, Comment, diff --git a/programs/install/Install.cpp b/programs/install/Install.cpp index 6bed114238a..d6576927a20 100644 --- a/programs/install/Install.cpp +++ b/programs/install/Install.cpp @@ -323,7 +323,7 @@ int mainEntryClickHouseInstall(int argc, char ** argv) { fmt::print("Symlink {} already exists but it points to {}. Will replace the old symlink to {}.\n", main_bin_path.string(), points_to.string(), binary_self_canonical_path.string()); - fs::remove(main_bin_path); + (void)fs::remove(main_bin_path); } } } @@ -489,7 +489,7 @@ int mainEntryClickHouseInstall(int argc, char ** argv) { fmt::print("Symlink {} already exists but it points to {}. Will replace the old symlink to {}.\n", symlink_path.string(), points_to.string(), main_bin_path.string()); - fs::remove(symlink_path); + (void)fs::remove(symlink_path); } } } @@ -1006,7 +1006,7 @@ namespace else { fmt::print("{} file exists but damaged, ignoring.\n", pid_file.string()); - fs::remove(pid_file); + (void)fs::remove(pid_file); } } else @@ -1014,7 +1014,7 @@ namespace /// Create a directory for pid file. /// It's created by "install" but we also support cases when ClickHouse is already installed different way. fs::path pid_path = pid_file; - pid_path.remove_filename(); + pid_path = pid_path.remove_filename(); fs::create_directories(pid_path); /// All users are allowed to read pid file (for clickhouse status command). fs::permissions(pid_path, fs::perms::owner_all | fs::perms::group_read | fs::perms::others_read, fs::perm_options::replace); @@ -1098,7 +1098,7 @@ namespace else { fmt::print("{} file exists but damaged, ignoring.\n", pid_file.string()); - fs::remove(pid_file); + (void)fs::remove(pid_file); } } catch (const Exception & e) diff --git a/programs/keeper-client/KeeperClient.cpp b/programs/keeper-client/KeeperClient.cpp index 52d825f30e6..ebec337060c 100644 --- a/programs/keeper-client/KeeperClient.cpp +++ b/programs/keeper-client/KeeperClient.cpp @@ -86,7 +86,10 @@ std::vector KeeperClient::getCompletions(const String & prefix) const void KeeperClient::askConfirmation(const String & prompt, std::function && callback) { if (!ask_confirmation) - return callback(); + { + callback(); + return; + } std::cout << prompt << " Continue?\n"; waiting_confirmation = true; diff --git a/programs/library-bridge/LibraryBridgeHandlers.cpp b/programs/library-bridge/LibraryBridgeHandlers.cpp index 26d887cfc98..8d116e537aa 100644 --- a/programs/library-bridge/LibraryBridgeHandlers.cpp +++ b/programs/library-bridge/LibraryBridgeHandlers.cpp @@ -284,7 +284,6 @@ void ExternalDictionaryLibraryBridgeRequestHandler::handleRequest(HTTPServerRequ else if (method == "extDict_loadIds") { LOG_DEBUG(log, "Getting diciontary ids for dictionary with id: {}", dictionary_id); - String ids_string; std::vector ids = parseIdsFromBinary(request.getStream()); auto library_handler = ExternalDictionaryLibraryHandlerFactory::instance().get(dictionary_id); diff --git a/programs/library-bridge/SharedLibrary.cpp b/programs/library-bridge/SharedLibrary.cpp index d70709474b5..7423f9b89f4 100644 --- a/programs/library-bridge/SharedLibrary.cpp +++ b/programs/library-bridge/SharedLibrary.cpp @@ -14,7 +14,7 @@ namespace ErrorCodes SharedLibrary::SharedLibrary(std::string_view path, int flags) { - handle = dlopen(path.data(), flags); + handle = dlopen(path.data(), flags); // NOLINT if (!handle) throw Exception(ErrorCodes::CANNOT_DLOPEN, "Cannot dlopen: ({})", dlerror()); // NOLINT(concurrency-mt-unsafe) // MT-Safe on Linux, see man dlerror @@ -34,7 +34,7 @@ void * SharedLibrary::getImpl(std::string_view name, bool no_throw) { dlerror(); // NOLINT(concurrency-mt-unsafe) // MT-Safe on Linux, see man dlerror - auto * res = dlsym(handle, name.data()); + auto * res = dlsym(handle, name.data()); // NOLINT if (char * error = dlerror()) // NOLINT(concurrency-mt-unsafe) // MT-Safe on Linux, see man dlerror { diff --git a/programs/main.cpp b/programs/main.cpp index 9ad8b016c82..4bb73399719 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -119,7 +119,7 @@ std::pair clickhouse_short_names[] = }; -enum class InstructionFail +enum class InstructionFail : uint8_t { NONE = 0, SSE3 = 1, @@ -491,9 +491,13 @@ int main(int argc_, char ** argv_) /// clickhouse -q 'select 1' # use local /// clickhouse # spawn local /// clickhouse local # spawn local + /// clickhouse "select ..." # spawn local /// - if (main_func == printHelp && !argv.empty() && (argv.size() == 1 || argv[1][0] == '-')) + if (main_func == printHelp && !argv.empty() && (argv.size() == 1 || argv[1][0] == '-' + || std::string_view(argv[1]).contains(' '))) + { main_func = mainEntryClickHouseLocal; + } int exit_code = main_func(static_cast(argv.size()), argv.data()); diff --git a/programs/obfuscator/Obfuscator.cpp b/programs/obfuscator/Obfuscator.cpp index b2bf942af4e..8035f053b41 100644 --- a/programs/obfuscator/Obfuscator.cpp +++ b/programs/obfuscator/Obfuscator.cpp @@ -674,8 +674,7 @@ private: if (pos + length > end) length = end - pos; - if (length > sizeof(CodePoint)) - length = sizeof(CodePoint); + length = std::min(length, sizeof(CodePoint)); CodePoint res = 0; memcpy(&res, pos, length); @@ -883,9 +882,7 @@ public: throw Exception(ErrorCodes::LOGICAL_ERROR, "Logical error in markov model"); size_t offset_from_begin_of_string = pos - data; - size_t determinator_sliding_window_size = params.determinator_sliding_window_size; - if (determinator_sliding_window_size > determinator_size) - determinator_sliding_window_size = determinator_size; + size_t determinator_sliding_window_size = std::min(params.determinator_sliding_window_size, determinator_size); size_t determinator_sliding_window_overflow = offset_from_begin_of_string + determinator_sliding_window_size > determinator_size ? offset_from_begin_of_string + determinator_sliding_window_size - determinator_size : 0; diff --git a/programs/odbc-bridge/ODBCSource.cpp b/programs/odbc-bridge/ODBCSource.cpp index 7f0d47f7e2e..940970f36ab 100644 --- a/programs/odbc-bridge/ODBCSource.cpp +++ b/programs/odbc-bridge/ODBCSource.cpp @@ -119,8 +119,7 @@ void ODBCSource::insertValue( time_t time = 0; const DataTypeDateTime & datetime_type = assert_cast(*data_type); readDateTimeText(time, in, datetime_type.getTimeZone()); - if (time < 0) - time = 0; + time = std::max(time, 0); column.insert(static_cast(time)); break; } diff --git a/programs/odbc-bridge/getIdentifierQuote.cpp b/programs/odbc-bridge/getIdentifierQuote.cpp index 15bd055e615..c0c833e5b8c 100644 --- a/programs/odbc-bridge/getIdentifierQuote.cpp +++ b/programs/odbc-bridge/getIdentifierQuote.cpp @@ -37,7 +37,7 @@ std::string getIdentifierQuote(nanodbc::ConnectionHolderPtr connection_holder) IdentifierQuotingStyle getQuotingStyle(nanodbc::ConnectionHolderPtr connection) { auto identifier_quote = getIdentifierQuote(connection); - if (identifier_quote.length() == 0) + if (identifier_quote.empty()) return IdentifierQuotingStyle::None; else if (identifier_quote[0] == '`') return IdentifierQuotingStyle::Backticks; diff --git a/programs/server/dashboard.html b/programs/server/dashboard.html index 901211e8ad9..b21d4b86314 100644 --- a/programs/server/dashboard.html +++ b/programs/server/dashboard.html @@ -538,9 +538,57 @@ let params = default_params; /// Palette generation for charts function generatePalette(numColors) { + // oklch() does not work in firefox<=125 inside element so we convert it back to rgb for now. + // Based on https://github.com/color-js/color.js/blob/main/src/spaces/oklch.js + const multiplyMatrices = (A, B) => { + return [ + A[0]*B[0] + A[1]*B[1] + A[2]*B[2], + A[3]*B[0] + A[4]*B[1] + A[5]*B[2], + A[6]*B[0] + A[7]*B[1] + A[8]*B[2] + ]; + } + + const oklch2oklab = ([l, c, h]) => [ + l, + isNaN(h) ? 0 : c * Math.cos(h * Math.PI / 180), + isNaN(h) ? 0 : c * Math.sin(h * Math.PI / 180) + ] + + const srgbLinear2rgb = rgb => rgb.map(c => + Math.abs(c) > 0.0031308 ? + (c < 0 ? -1 : 1) * (1.055 * (Math.abs(c) ** (1 / 2.4)) - 0.055) : + 12.92 * c + ) + + const oklab2xyz = lab => { + const LMSg = multiplyMatrices([ + 1, 0.3963377773761749, 0.2158037573099136, + 1, -0.1055613458156586, -0.0638541728258133, + 1, -0.0894841775298119, -1.2914855480194092, + ], lab) + const LMS = LMSg.map(val => val ** 3) + return multiplyMatrices([ + 1.2268798758459243, -0.5578149944602171, 0.2813910456659647, + -0.0405757452148008, 1.1122868032803170, -0.0717110580655164, + -0.0763729366746601, -0.4214933324022432, 1.5869240198367816 + ], LMS) + } + + const xyz2rgbLinear = xyz => { + return multiplyMatrices([ + 3.2409699419045226, -1.537383177570094, -0.4986107602930034, + -0.9692436362808796, 1.8759675015077202, 0.04155505740717559, + 0.05563007969699366, -0.20397695888897652, 1.0569715142428786 + ], xyz) + } + + const oklch2rgb = lch => srgbLinear2rgb(xyz2rgbLinear(oklab2xyz(oklch2oklab(lch)))) + palette = []; for (let i = 0; i < numColors; i++) { - palette.push(`oklch(${theme != 'dark' ? 0.75 : 0.5}, 0.15, ${360 * i / numColors})`); + //palette.push(`oklch(${theme != 'dark' ? 0.75 : 0.5}, 0.15, ${360 * i / numColors})`); + let rgb = oklch2rgb([theme != 'dark' ? 0.75 : 0.5, 0.15, 360 * i / numColors]); + palette.push(`rgb(${rgb[0] * 255}, ${rgb[1] * 255}, ${rgb[2] * 255})`); } return palette; } diff --git a/programs/static-files-disk-uploader/static-files-disk-uploader.cpp b/programs/static-files-disk-uploader/static-files-disk-uploader.cpp index dfe68c819fc..f7696dd37f1 100644 --- a/programs/static-files-disk-uploader/static-files-disk-uploader.cpp +++ b/programs/static-files-disk-uploader/static-files-disk-uploader.cpp @@ -111,13 +111,11 @@ void processTableFiles(const fs::path & data_path, fs::path dst_path, bool test_ std::shared_ptr directory_meta; if (test_mode) { - auto files_root = dst_path / prefix; directory_meta = std::make_shared(HTTPConnectionGroupType::HTTP, Poco::URI(dst_path / directory_prefix / ".index"), Poco::Net::HTTPRequest::HTTP_PUT); } else { dst_path = fs::canonical(dst_path); - auto files_root = dst_path / prefix; fs::create_directories(dst_path / directory_prefix); directory_meta = std::make_shared(dst_path / directory_prefix / ".index"); } diff --git a/src/Access/AccessBackup.cpp b/src/Access/AccessBackup.cpp index 1110b9c4b21..90effdab70f 100644 --- a/src/Access/AccessBackup.cpp +++ b/src/Access/AccessBackup.cpp @@ -93,8 +93,6 @@ namespace break; } - size_t id_endpos = line.find('\t'); - String id_as_string = line.substr(0, id_endpos); UUID id = parse(line); line.clear(); diff --git a/src/Access/AccessBackup.h b/src/Access/AccessBackup.h index 1aaac589ccb..aa59d6bf201 100644 --- a/src/Access/AccessBackup.h +++ b/src/Access/AccessBackup.h @@ -8,7 +8,7 @@ namespace DB { class AccessControl; -enum class AccessEntityType; +enum class AccessEntityType : uint8_t; struct IAccessEntity; using AccessEntityPtr = std::shared_ptr; class AccessRightsElements; diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index a87e9361e8e..c10931f554c 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -233,7 +233,7 @@ namespace /** * Levels: - * 1. GLOBAL + * 1. GLOBAL * 2. DATABASE_LEVEL 2. GLOBAL_WITH_PARAMETER (parameter example: named collection) * 3. TABLE_LEVEL * 4. COLUMN_LEVEL @@ -241,11 +241,12 @@ namespace enum Level { - GLOBAL_LEVEL, - DATABASE_LEVEL, + GLOBAL_LEVEL = 0, + DATABASE_LEVEL = 1, GLOBAL_WITH_PARAMETER = DATABASE_LEVEL, - TABLE_LEVEL, - COLUMN_LEVEL, + TABLE_LEVEL = 2, + COLUMN_LEVEL = 3, + MAX = COLUMN_LEVEL, }; AccessFlags getAllGrantableFlags(Level level) @@ -520,7 +521,7 @@ public: private: AccessFlags getAllGrantableFlags() const { return ::DB::getAllGrantableFlags(level); } - AccessFlags getChildAllGrantableFlags() const { return ::DB::getAllGrantableFlags(static_cast(level + 1)); } + AccessFlags getChildAllGrantableFlags() const { return ::DB::getAllGrantableFlags(static_cast(level == Level::MAX ? level : (level + 1))); } Node * tryGetChild(std::string_view name) const { diff --git a/src/Access/AuthenticationData.cpp b/src/Access/AuthenticationData.cpp index a4c25b438e8..a32215f3d92 100644 --- a/src/Access/AuthenticationData.cpp +++ b/src/Access/AuthenticationData.cpp @@ -118,13 +118,16 @@ void AuthenticationData::setPassword(const String & password_) switch (type) { case AuthenticationType::PLAINTEXT_PASSWORD: - return setPasswordHashBinary(Util::stringToDigest(password_)); + setPasswordHashBinary(Util::stringToDigest(password_)); + return; case AuthenticationType::SHA256_PASSWORD: - return setPasswordHashBinary(Util::encodeSHA256(password_)); + setPasswordHashBinary(Util::encodeSHA256(password_)); + return; case AuthenticationType::DOUBLE_SHA1_PASSWORD: - return setPasswordHashBinary(Util::encodeDoubleSHA1(password_)); + setPasswordHashBinary(Util::encodeDoubleSHA1(password_)); + return; case AuthenticationType::BCRYPT_PASSWORD: case AuthenticationType::NO_PASSWORD: @@ -146,7 +149,7 @@ void AuthenticationData::setPasswordBcrypt(const String & password_, int workfac if (type != AuthenticationType::BCRYPT_PASSWORD) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot specify bcrypt password for authentication type {}", toString(type)); - return setPasswordHashBinary(Util::encodeBcrypt(password_, workfactor_)); + setPasswordHashBinary(Util::encodeBcrypt(password_, workfactor_)); } String AuthenticationData::getPassword() const diff --git a/src/Access/Common/AccessEntityType.h b/src/Access/Common/AccessEntityType.h index 12df7cf5538..fd8d1fe8421 100644 --- a/src/Access/Common/AccessEntityType.h +++ b/src/Access/Common/AccessEntityType.h @@ -7,7 +7,7 @@ namespace DB { /// Represents the type of an access entity (see the IAccessEntity class). -enum class AccessEntityType +enum class AccessEntityType : uint8_t { USER, ROLE, diff --git a/src/Access/Common/AccessFlags.cpp b/src/Access/Common/AccessFlags.cpp index 9d846821e42..23c52d86f4e 100644 --- a/src/Access/Common/AccessFlags.cpp +++ b/src/Access/Common/AccessFlags.cpp @@ -115,15 +115,15 @@ namespace { UNKNOWN = -2, GROUP = -1, - GLOBAL, - DATABASE, - TABLE, + GLOBAL = 0, + DATABASE = 1, + TABLE = 2, VIEW = TABLE, - COLUMN, - DICTIONARY, - NAMED_COLLECTION, - USER_NAME, - TABLE_ENGINE, + COLUMN = 3, + DICTIONARY = 4, + NAMED_COLLECTION = 5, + USER_NAME = 6, + TABLE_ENGINE = 7, }; struct Node; diff --git a/src/Access/Common/AccessRightsElement.cpp b/src/Access/Common/AccessRightsElement.cpp index 835f414df37..24ff4e7631b 100644 --- a/src/Access/Common/AccessRightsElement.cpp +++ b/src/Access/Common/AccessRightsElement.cpp @@ -245,7 +245,7 @@ bool AccessRightsElements::sameOptions() const void AccessRightsElements::eraseNonGrantable() { - boost::range::remove_erase_if(*this, [](AccessRightsElement & element) + std::erase_if(*this, [](AccessRightsElement & element) { element.eraseNonGrantable(); return element.empty(); diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index 570e36bf6db..7f0eff2184b 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -7,7 +7,7 @@ namespace DB { /// Represents an access type which can be granted on databases, tables, columns, etc. -enum class AccessType +enum class AccessType : uint8_t { /// Macro M should be defined as M(name, aliases, node_type, parent_group_name) /// where name is identifier with underscores (instead of spaces); diff --git a/src/Access/Common/AllowedClientHosts.cpp b/src/Access/Common/AllowedClientHosts.cpp index bee0cdd7264..2875f65579e 100644 --- a/src/Access/Common/AllowedClientHosts.cpp +++ b/src/Access/Common/AllowedClientHosts.cpp @@ -308,7 +308,7 @@ void AllowedClientHosts::removeAddress(const IPAddress & address) if (address.isLoopback()) local_host = false; else - boost::range::remove_erase(addresses, address); + std::erase(addresses, address); } void AllowedClientHosts::addSubnet(const IPSubnet & subnet) @@ -328,7 +328,7 @@ void AllowedClientHosts::removeSubnet(const IPSubnet & subnet) else if (subnet.isMaskAllBitsOne()) removeAddress(subnet.getPrefix()); else - boost::range::remove_erase(subnets, subnet); + std::erase(subnets, subnet); } void AllowedClientHosts::addName(const String & name) @@ -344,7 +344,7 @@ void AllowedClientHosts::removeName(const String & name) if (boost::iequals(name, "localhost")) local_host = false; else - boost::range::remove_erase(names, name); + std::erase(names, name); } void AllowedClientHosts::addNameRegexp(const String & name_regexp) @@ -364,7 +364,7 @@ void AllowedClientHosts::removeNameRegexp(const String & name_regexp) else if (name_regexp == ".*") any_host = false; else - boost::range::remove_erase(name_regexps, name_regexp); + std::erase(name_regexps, name_regexp); } void AllowedClientHosts::addLikePattern(const String & pattern) @@ -384,7 +384,7 @@ void AllowedClientHosts::removeLikePattern(const String & pattern) else if ((pattern == "%") || (pattern == "0.0.0.0/0") || (pattern == "::/0")) any_host = false; else - boost::range::remove_erase(like_patterns, pattern); + std::erase(like_patterns, pattern); } void AllowedClientHosts::addLocalHost() diff --git a/src/Access/Common/AuthenticationType.h b/src/Access/Common/AuthenticationType.h index 506c8abd3b1..a68549aff4c 100644 --- a/src/Access/Common/AuthenticationType.h +++ b/src/Access/Common/AuthenticationType.h @@ -6,7 +6,7 @@ namespace DB { -enum class AuthenticationType +enum class AuthenticationType : uint8_t { /// User doesn't have to enter password. NO_PASSWORD, diff --git a/src/Access/Common/QuotaDefs.h b/src/Access/Common/QuotaDefs.h index 6618f01c8f9..74253833cfd 100644 --- a/src/Access/Common/QuotaDefs.h +++ b/src/Access/Common/QuotaDefs.h @@ -9,7 +9,7 @@ namespace DB using QuotaValue = UInt64; /// Kinds of resource what we wish to quota. -enum class QuotaType +enum class QuotaType : uint8_t { QUERIES, /// Number of queries. QUERY_SELECTS, /// Number of select queries. @@ -45,7 +45,7 @@ struct QuotaTypeInfo /// Key to share quota consumption. /// Users with the same key share the same amount of resource. -enum class QuotaKeyType +enum class QuotaKeyType : uint8_t { NONE, /// All users share the same quota. USER_NAME, /// Connections with the same user name share the same quota. diff --git a/src/Access/Common/RowPolicyDefs.h b/src/Access/Common/RowPolicyDefs.h index bf2f632e98b..4197c58c18e 100644 --- a/src/Access/Common/RowPolicyDefs.h +++ b/src/Access/Common/RowPolicyDefs.h @@ -25,7 +25,7 @@ struct RowPolicyName /// Types of the filters of row policies. /// Currently only RowPolicyFilterType::SELECT is supported. -enum class RowPolicyFilterType +enum class RowPolicyFilterType : uint8_t { /// Filter is a SQL conditional expression used to figure out which rows should be visible /// for user or available for modification. If the expression returns NULL or false for some rows diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 2736d13e751..2a658d7aaa2 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -570,11 +570,8 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg if (params.full_access) return true; - auto access_granted = [&] + auto access_granted = [] { - if (trace_log) - LOG_TRACE(trace_log, "Access granted: {}{}", (AccessRightsElement{flags, args...}.toStringWithoutOptions()), - (grant_option ? " WITH GRANT OPTION" : "")); return true; }; @@ -582,9 +579,6 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg FormatStringHelper fmt_string [[maybe_unused]], FmtArgs && ...fmt_args [[maybe_unused]]) { - if (trace_log) - LOG_TRACE(trace_log, "Access denied: {}{}", (AccessRightsElement{flags, args...}.toStringWithoutOptions()), - (grant_option ? " WITH GRANT OPTION" : "")); if constexpr (throw_if_denied) throw Exception(error_code, std::move(fmt_string), getUserName(), std::forward(fmt_args)...); return false; diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index 3c20ef3d102..fe698b32816 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -71,7 +71,7 @@ namespace SCOPE_EXIT( { if (!succeeded) - std::filesystem::remove(tmp_file_path); + (void)std::filesystem::remove(tmp_file_path); }); /// Write the file. @@ -302,7 +302,7 @@ void DiskAccessStorage::writeLists() } /// The list files was successfully written, we don't need the 'need_rebuild_lists.mark' file any longer. - std::filesystem::remove(getNeedRebuildListsMarkFilePath(directory_path)); + (void)std::filesystem::remove(getNeedRebuildListsMarkFilePath(directory_path)); types_of_lists_to_write.clear(); } @@ -419,7 +419,7 @@ void DiskAccessStorage::removeAllExceptInMemory(const boost::container::flat_set const auto & id = it->first; ++it; /// We must go to the next element in the map `entries_by_id` here because otherwise removeNoLock() can invalidate our iterator. if (!ids_to_keep.contains(id)) - removeNoLock(id, /* throw_if_not_exists */ true, /* write_on_disk= */ false); + (void)removeNoLock(id, /* throw_if_not_exists */ true, /* write_on_disk= */ false); } } @@ -549,7 +549,7 @@ bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne if (name_collision && (id_by_name != id)) { assert(replace_if_exists); - removeNoLock(id_by_name, /* throw_if_not_exists= */ false, write_on_disk); + removeNoLock(id_by_name, /* throw_if_not_exists= */ false, write_on_disk); // NOLINT } if (id_collision) @@ -574,7 +574,7 @@ bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne return true; } - removeNoLock(id, /* throw_if_not_exists= */ false, write_on_disk); + removeNoLock(id, /* throw_if_not_exists= */ false, write_on_disk); // NOLINT } /// Do insertion. diff --git a/src/Access/GrantedRoles.cpp b/src/Access/GrantedRoles.cpp index bb07cbd6951..e1a23182cc0 100644 --- a/src/Access/GrantedRoles.cpp +++ b/src/Access/GrantedRoles.cpp @@ -161,9 +161,9 @@ void GrantedRoles::makeUnion(const GrantedRoles & other) void GrantedRoles::makeIntersection(const GrantedRoles & other) { - boost::range::remove_erase_if(roles, [&other](const UUID & id) { return other.roles.find(id) == other.roles.end(); }); + boost::range::remove_erase_if(roles, [&other](const UUID & id) { return other.roles.find(id) == other.roles.end(); }); // NOLINT - boost::range::remove_erase_if(roles_with_admin_option, [&other](const UUID & id) + boost::range::remove_erase_if(roles_with_admin_option, [&other](const UUID & id) // NOLINT { return other.roles_with_admin_option.find(id) == other.roles_with_admin_option.end(); }); diff --git a/src/Access/IAccessStorage.cpp b/src/Access/IAccessStorage.cpp index 1d6b8d99cd5..8e51481e415 100644 --- a/src/Access/IAccessStorage.cpp +++ b/src/Access/IAccessStorage.cpp @@ -583,7 +583,7 @@ void IAccessStorage::backup(BackupEntriesCollector & backup_entries_collector, c throwBackupNotAllowed(); auto entities = readAllWithIDs(type); - boost::range::remove_erase_if(entities, [](const std::pair & x) { return !x.second->isBackupAllowed(); }); + std::erase_if(entities, [](const std::pair & x) { return !x.second->isBackupAllowed(); }); if (entities.empty()) return; diff --git a/src/Access/IAccessStorage.h b/src/Access/IAccessStorage.h index b00c8bac849..4f980bf9212 100644 --- a/src/Access/IAccessStorage.h +++ b/src/Access/IAccessStorage.h @@ -24,7 +24,7 @@ namespace DB struct User; class Credentials; class ExternalAuthenticators; -enum class AuthenticationType; +enum class AuthenticationType : uint8_t; class BackupEntriesCollector; class RestorerFromBackup; diff --git a/src/Access/LDAPAccessStorage.cpp b/src/Access/LDAPAccessStorage.cpp index b1f10cd2d49..3206b20b691 100644 --- a/src/Access/LDAPAccessStorage.cpp +++ b/src/Access/LDAPAccessStorage.cpp @@ -76,7 +76,7 @@ void LDAPAccessStorage::setConfiguration(const Poco::Util::AbstractConfiguration config.keys(prefix, all_keys); for (const auto & key : all_keys) { - if (key == "role_mapping" || key.find("role_mapping[") == 0) + if (key == "role_mapping" || key.starts_with("role_mapping[")) parseLDAPRoleSearchParams(role_search_params_cfg.emplace_back(), config, prefix_str + key); } } @@ -94,7 +94,7 @@ void LDAPAccessStorage::setConfiguration(const Poco::Util::AbstractConfiguration role_change_subscription = access_control.subscribeForChanges( [this] (const UUID & id, const AccessEntityPtr & entity) { - return this->processRoleChange(id, entity); + this->processRoleChange(id, entity); } ); } @@ -200,7 +200,7 @@ void LDAPAccessStorage::applyRoleChangeNoLock(bool grant, const UUID & role_id, void LDAPAccessStorage::assignRolesNoLock(User & user, const LDAPClient::SearchResultsList & external_roles) const { const auto external_roles_hash = boost::hash{}(external_roles); - return assignRolesNoLock(user, external_roles, external_roles_hash); + assignRolesNoLock(user, external_roles, external_roles_hash); } diff --git a/src/Access/LDAPClient.h b/src/Access/LDAPClient.h index bf193bf6bb2..0bbd2c6e9c4 100644 --- a/src/Access/LDAPClient.h +++ b/src/Access/LDAPClient.h @@ -26,7 +26,7 @@ class LDAPClient public: struct SearchParams { - enum class Scope + enum class Scope : uint8_t { BASE, ONE_LEVEL, @@ -57,20 +57,20 @@ public: struct Params { - enum class ProtocolVersion + enum class ProtocolVersion : uint8_t { V2, V3 }; - enum class TLSEnable + enum class TLSEnable : uint8_t { NO, YES_STARTTLS, YES }; - enum class TLSProtocolVersion + enum class TLSProtocolVersion : uint8_t { SSL2, SSL3, @@ -79,7 +79,7 @@ public: TLS1_2 }; - enum class TLSRequireCert + enum class TLSRequireCert : uint8_t { NEVER, ALLOW, @@ -87,7 +87,7 @@ public: DEMAND }; - enum class SASLMechanism + enum class SASLMechanism : uint8_t { UNKNOWN, SIMPLE diff --git a/src/Access/MemoryAccessStorage.cpp b/src/Access/MemoryAccessStorage.cpp index 999e2c41d0b..791030b9b12 100644 --- a/src/Access/MemoryAccessStorage.cpp +++ b/src/Access/MemoryAccessStorage.cpp @@ -106,7 +106,7 @@ bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & if (name_collision && (id_by_name != id)) { assert(replace_if_exists); - removeNoLock(id_by_name, /* throw_if_not_exists= */ true); + removeNoLock(id_by_name, /* throw_if_not_exists= */ true); // NOLINT } if (id_collision) @@ -128,7 +128,7 @@ bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & } return true; } - removeNoLock(id, /* throw_if_not_exists= */ true); + removeNoLock(id, /* throw_if_not_exists= */ true); // NOLINT } /// Do insertion. @@ -238,7 +238,7 @@ void MemoryAccessStorage::removeAllExceptNoLock(const boost::container::flat_set const auto & id = it->first; ++it; /// We must go to the next element in the map `entries_by_id` here because otherwise removeNoLock() can invalidate our iterator. if (!ids_to_keep.contains(id)) - removeNoLock(id, /* throw_if_not_exists */ true); + removeNoLock(id, /* throw_if_not_exists */ true); // NOLINT } } diff --git a/src/Access/MultipleAccessStorage.cpp b/src/Access/MultipleAccessStorage.cpp index 03d690de0bd..a8b508202b5 100644 --- a/src/Access/MultipleAccessStorage.cpp +++ b/src/Access/MultipleAccessStorage.cpp @@ -238,7 +238,7 @@ void MultipleAccessStorage::moveAccessEntities(const std::vector & ids, co try { - source_storage->remove(ids); + source_storage->remove(ids); // NOLINT need_rollback = true; destination_storage->insert(to_move, ids); } diff --git a/src/Access/ReplicatedAccessStorage.cpp b/src/Access/ReplicatedAccessStorage.cpp index 275672da115..cd9a86a1bd2 100644 --- a/src/Access/ReplicatedAccessStorage.cpp +++ b/src/Access/ReplicatedAccessStorage.cpp @@ -616,7 +616,7 @@ void ReplicatedAccessStorage::setEntityNoLock(const UUID & id, const AccessEntit void ReplicatedAccessStorage::removeEntityNoLock(const UUID & id) { LOG_DEBUG(getLogger(), "Removing entity with id {}", toString(id)); - memory_storage.remove(id, /* throw_if_not_exists= */ false); + memory_storage.remove(id, /* throw_if_not_exists= */ false); // NOLINT } @@ -654,7 +654,7 @@ void ReplicatedAccessStorage::backup(BackupEntriesCollector & backup_entries_col throwBackupNotAllowed(); auto entities = readAllWithIDs(type); - boost::range::remove_erase_if(entities, [](const std::pair & x) { return !x.second->isBackupAllowed(); }); + std::erase_if(entities, [](const std::pair & x) { return !x.second->isBackupAllowed(); }); if (entities.empty()) return; diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index db805c83e17..a274f6b54f2 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -190,11 +190,11 @@ void SettingsConstraints::check(const Settings & current_settings, const Setting void SettingsConstraints::check(const Settings & current_settings, SettingsChanges & changes, SettingSource source) const { - boost::range::remove_erase_if( + std::erase_if( changes, [&](SettingChange & change) -> bool { - return !checkImpl(current_settings, const_cast(change), THROW_ON_VIOLATION, source); + return !checkImpl(current_settings, change, THROW_ON_VIOLATION, source); }); } @@ -211,7 +211,7 @@ void SettingsConstraints::check(const MergeTreeSettings & current_settings, cons void SettingsConstraints::clamp(const Settings & current_settings, SettingsChanges & changes, SettingSource source) const { - boost::range::remove_erase_if( + std::erase_if( changes, [&](SettingChange & change) -> bool { diff --git a/src/AggregateFunctions/AggregateFunctionAggThrow.cpp b/src/AggregateFunctions/AggregateFunctionAggThrow.cpp index 27432bc94ba..ebf6cabbcfd 100644 --- a/src/AggregateFunctions/AggregateFunctionAggThrow.cpp +++ b/src/AggregateFunctions/AggregateFunctionAggThrow.cpp @@ -16,7 +16,7 @@ struct Settings; namespace ErrorCodes { extern const int AGGREGATE_FUNCTION_THROW; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION; } namespace @@ -116,7 +116,7 @@ void registerAggregateFunctionAggThrow(AggregateFunctionFactory & factory) if (parameters.size() == 1) throw_probability = parameters[0].safeGet(); else if (parameters.size() > 1) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} cannot have more than one parameter", name); + throw Exception(ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION, "Aggregate function {} cannot have more than one parameter", name); return std::make_shared(argument_types, parameters, throw_probability); }); diff --git a/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp b/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp index 33e318b6c2f..204edd83215 100644 --- a/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp +++ b/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp @@ -269,9 +269,9 @@ struct AggregateFunctionFlameGraphData using Entries = HashMap; - AggregateFunctionFlameGraphTree tree; Entries entries; Entry * free_list = nullptr; + AggregateFunctionFlameGraphTree tree; Entry * alloc(Arena * arena) { diff --git a/src/AggregateFunctions/AggregateFunctionGroupArray.cpp b/src/AggregateFunctions/AggregateFunctionGroupArray.cpp index 63002652166..2b1e1a7c339 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArray.cpp +++ b/src/AggregateFunctions/AggregateFunctionGroupArray.cpp @@ -43,7 +43,7 @@ namespace ErrorCodes namespace { -enum class Sampler +enum class Sampler : uint8_t { NONE, RNG, @@ -735,14 +735,14 @@ IAggregateFunction * createWithNumericOrTimeType(const IDataType & argument_type template inline AggregateFunctionPtr createAggregateFunctionGroupArrayImpl(const DataTypePtr & argument_type, const Array & parameters, TArgs ... args) { - if (auto res = createWithNumericOrTimeType(*argument_type, argument_type, parameters, std::forward(args)...)) + if (auto res = createWithNumericOrTimeType(*argument_type, argument_type, parameters, args...)) return AggregateFunctionPtr(res); WhichDataType which(argument_type); if (which.idx == TypeIndex::String) - return std::make_shared>(argument_type, parameters, std::forward(args)...); + return std::make_shared>(argument_type, parameters, args...); - return std::make_shared>(argument_type, parameters, std::forward(args)...); + return std::make_shared>(argument_type, parameters, args...); } size_t getMaxArraySize() diff --git a/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.cpp b/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.cpp index 60e8df64283..8674aac2e90 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.cpp +++ b/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.cpp @@ -27,6 +27,7 @@ struct Settings; namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION; extern const int TOO_LARGE_ARRAY_SIZE; extern const int CANNOT_CONVERT_TYPE; extern const int ILLEGAL_TYPE_OF_ARGUMENT; @@ -74,7 +75,7 @@ public: if (!params.empty()) { if (params.size() > 2) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} requires at most two parameters.", getName()); + throw Exception(ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION, "Aggregate function {} requires at most two parameters.", getName()); default_value = params[0]; diff --git a/src/AggregateFunctions/AggregateFunctionGroupArraySorted.cpp b/src/AggregateFunctions/AggregateFunctionGroupArraySorted.cpp index c633871c737..d41d743e17a 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArraySorted.cpp +++ b/src/AggregateFunctions/AggregateFunctionGroupArraySorted.cpp @@ -47,7 +47,7 @@ namespace ErrorCodes namespace { -enum class GroupArraySortedStrategy +enum class GroupArraySortedStrategy : uint8_t { heap, sort @@ -374,10 +374,10 @@ AggregateFunctionPtr createWithNumericOrTimeType(const IDataType & argument_type template