diff --git a/.github/ISSUE_TEMPLATE/85_bug-report.md b/.github/ISSUE_TEMPLATE/85_bug-report.md index 93b2342af70..6bf265260ac 100644 --- a/.github/ISSUE_TEMPLATE/85_bug-report.md +++ b/.github/ISSUE_TEMPLATE/85_bug-report.md @@ -17,7 +17,7 @@ assignees: '' > A link to reproducer in [https://fiddle.clickhouse.com/](https://fiddle.clickhouse.com/). -**Does it reproduce on recent release?** +**Does it reproduce on the most recent release?** [The list of releases](https://github.com/ClickHouse/ClickHouse/blob/master/utils/list-versions/version_date.tsv) @@ -34,11 +34,11 @@ assignees: '' **How to reproduce** * Which ClickHouse server version to use -* Which interface to use, if matters +* Which interface to use, if it matters * Non-default settings, if any * `CREATE TABLE` statements for all tables involved * Sample data for all these tables, use [clickhouse-obfuscator](https://github.com/ClickHouse/ClickHouse/blob/master/programs/obfuscator/Obfuscator.cpp#L42-L80) if necessary -* Queries to run that lead to unexpected result +* Queries to run that lead to an unexpected result **Expected behavior** diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 5d57e6fc1d8..2471e4f9194 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -55,7 +55,6 @@ jobs: uses: ./.github/workflows/reusable_docker.yml with: data: ${{ needs.RunConfig.outputs.data }} - set_latest: true StyleCheck: needs: [RunConfig, BuildDockers] if: ${{ !failure() && !cancelled() }} @@ -362,14 +361,6 @@ jobs: test_name: Stateless tests (release) runner_type: func-tester data: ${{ needs.RunConfig.outputs.data }} - FunctionalStatelessTestReleaseDatabaseOrdinary: - needs: [RunConfig, BuilderDebRelease] - if: ${{ !failure() && !cancelled() }} - uses: ./.github/workflows/reusable_test.yml - with: - test_name: Stateless tests (release, DatabaseOrdinary) - runner_type: func-tester - data: ${{ needs.RunConfig.outputs.data }} FunctionalStatelessTestReleaseDatabaseReplicated: needs: [RunConfig, BuilderDebRelease] if: ${{ !failure() && !cancelled() }} @@ -733,7 +724,6 @@ jobs: - MarkReleaseReady - FunctionalStatelessTestDebug - FunctionalStatelessTestRelease - - FunctionalStatelessTestReleaseDatabaseOrdinary - FunctionalStatelessTestReleaseDatabaseReplicated - FunctionalStatelessTestReleaseAnalyzer - FunctionalStatelessTestReleaseS3 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 2774eae24cc..770e1ec3789 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -28,7 +28,7 @@ jobs: id: runconfig run: | echo "::group::configure CI run" - python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --skip-jobs --rebuild-all-docker --outfile ${{ runner.temp }}/ci_run_data.json + python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --skip-jobs --outfile ${{ runner.temp }}/ci_run_data.json echo "::endgroup::" echo "::group::CI run configure results" diff --git a/.github/workflows/reusable_docker.yml b/.github/workflows/reusable_docker.yml index 08a5740e7e0..3fe1a8883c6 100644 --- a/.github/workflows/reusable_docker.yml +++ b/.github/workflows/reusable_docker.yml @@ -46,7 +46,7 @@ jobs: needs: [DockerBuildAmd64, DockerBuildAarch64] runs-on: [self-hosted, style-checker] if: | - !failure() && !cancelled() && toJson(fromJson(inputs.data).docker_data.missing_multi) != '[]' + !failure() && !cancelled() && (toJson(fromJson(inputs.data).docker_data.missing_multi) != '[]' || inputs.set_latest) steps: - name: Check out repository code uses: ClickHouse/checkout@v1 @@ -55,14 +55,12 @@ jobs: - name: Build images run: | cd "$GITHUB_WORKSPACE/tests/ci" + FLAG_LATEST='' if [ "${{ inputs.set_latest }}" == "true" ]; then + FLAG_LATEST='--set-latest' echo "latest tag will be set for resulting manifests" - python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 \ - --image-tags '${{ toJson(fromJson(inputs.data).docker_data.images) }}' \ - --missing-images '${{ toJson(fromJson(inputs.data).docker_data.missing_multi) }}' \ - --set-latest - else - python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 \ - --image-tags '${{ toJson(fromJson(inputs.data).docker_data.images) }}' \ - --missing-images '${{ toJson(fromJson(inputs.data).docker_data.missing_multi) }}' fi + python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 \ + --image-tags '${{ toJson(fromJson(inputs.data).docker_data.images) }}' \ + --missing-images '${{ toJson(fromJson(inputs.data).docker_data.missing_multi) }}' \ + $FLAG_LATEST diff --git a/CHANGELOG.md b/CHANGELOG.md index 60618402174..b3e5dd709ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,6 @@ * Add `quantileDD` aggregate function as well as the corresponding `quantilesDD` and `medianDD`. It is based on the DDSketch https://www.vldb.org/pvldb/vol12/p2195-masson.pdf. ### Documentation entry for user-facing changes. [#56342](https://github.com/ClickHouse/ClickHouse/pull/56342) ([Srikanth Chekuri](https://github.com/srikanthccv)). * Allow to configure any kind of object storage with any kind of metadata type. [#58357](https://github.com/ClickHouse/ClickHouse/pull/58357) ([Kseniia Sumarokova](https://github.com/kssenii)). * Added `null_status_on_timeout_only_active` and `throw_only_active` modes for `distributed_ddl_output_mode` that allow to avoid waiting for inactive replicas. [#58350](https://github.com/ClickHouse/ClickHouse/pull/58350) ([Alexander Tokmakov](https://github.com/tavplubix)). -* Allow partitions from tables with different partition expressions to be attached when the destination table partition expression doesn't re-partition/split the part. [#39507](https://github.com/ClickHouse/ClickHouse/pull/39507) ([Arthur Passos](https://github.com/arthurpassos)). * Add function `arrayShingles` to compute subarrays, e.g. `arrayShingles([1, 2, 3, 4, 5], 3)` returns `[[1,2,3],[2,3,4],[3,4,5]]`. [#58396](https://github.com/ClickHouse/ClickHouse/pull/58396) ([Zheng Miao](https://github.com/zenmiao7)). * Added functions `punycodeEncode`, `punycodeDecode`, `idnaEncode` and `idnaDecode` which are useful for translating international domain names to an ASCII representation according to the IDNA standard. [#58454](https://github.com/ClickHouse/ClickHouse/pull/58454) ([Robert Schulze](https://github.com/rschu1ze)). * Added string similarity functions `dramerauLevenshteinDistance`, `jaroSimilarity` and `jaroWinklerSimilarity`. [#58531](https://github.com/ClickHouse/ClickHouse/pull/58531) ([Robert Schulze](https://github.com/rschu1ze)). diff --git a/README.md b/README.md index d356e429892..9ada350d173 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Keep an eye out for upcoming meetups around the world. Somewhere else you want u ## Recent Recordings * **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments" -* **Recording available**: [**v23.10 Release Webinar**](https://www.youtube.com/watch?v=PGQS6uPb970) All the features of 23.10, one convenient video! Watch it now! +* **Recording available**: [**v24.1 Release Webinar**](https://www.youtube.com/watch?v=pBF9g0wGAGs) All the features of 24.1, one convenient video! Watch it now! * **All release webinar recordings**: [YouTube playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3jAlSy1JxyP8zluvXaN3nxU) diff --git a/base/base/CMakeLists.txt b/base/base/CMakeLists.txt index 3886932d198..025687d2c59 100644 --- a/base/base/CMakeLists.txt +++ b/base/base/CMakeLists.txt @@ -17,6 +17,7 @@ set (SRCS getMemoryAmount.cpp getPageSize.cpp getThreadId.cpp + int8_to_string.cpp JSON.cpp mremap.cpp phdr_cache.cpp diff --git a/base/base/getMemoryAmount.cpp b/base/base/getMemoryAmount.cpp index a46e964c5a3..ccdc0f0f976 100644 --- a/base/base/getMemoryAmount.cpp +++ b/base/base/getMemoryAmount.cpp @@ -1,8 +1,11 @@ -#include -#include #include + #include +#include +#include +#include + #include #include #include @@ -11,6 +14,80 @@ #endif +namespace +{ + +std::optional getCgroupsV2MemoryLimit() +{ +#if defined(OS_LINUX) + const std::filesystem::path default_cgroups_mount = "/sys/fs/cgroup"; + + /// This file exists iff the host has cgroups v2 enabled. + std::ifstream controllers_file(default_cgroups_mount / "cgroup.controllers"); + if (!controllers_file.is_open()) + return {}; + + /// Make sure that the memory controller is enabled. + /// - cgroup.controllers defines which controllers *can* be enabled. + /// - cgroup.subtree_control defines which controllers *are* enabled. + /// (see https://docs.kernel.org/admin-guide/cgroup-v2.html) + /// Caveat: nested groups may disable controllers. For simplicity, check only the top-level group. + /// ReadBufferFromFile subtree_control_file(default_cgroups_mount / "cgroup.subtree_control"); + /// std::string subtree_control; + /// readString(subtree_control, subtree_control_file); + /// if (subtree_control.find("memory") == std::string::npos) + /// return {}; + std::ifstream subtree_control_file(default_cgroups_mount / "cgroup.subtree_control"); + std::stringstream subtree_control_buf; + subtree_control_buf << subtree_control_file.rdbuf(); + std::string subtree_control = subtree_control_buf.str(); + if (subtree_control.find("memory") == std::string::npos) + return {}; + + /// Identify the cgroup the process belongs to + /// All PIDs assigned to a cgroup are in /sys/fs/cgroups/{cgroup_name}/cgroup.procs + /// A simpler way to get the membership is: + std::ifstream cgroup_name_file("/proc/self/cgroup"); + if (!cgroup_name_file.is_open()) + return {}; + + std::stringstream cgroup_name_buf; + cgroup_name_buf << cgroup_name_file.rdbuf(); + std::string cgroup_name = cgroup_name_buf.str(); + if (!cgroup_name.empty() && cgroup_name.back() == '\n') + cgroup_name.pop_back(); /// remove trailing newline, if any + /// With cgroups v2, there will be a *single* line with prefix "0::/" + const std::string v2_prefix = "0::/"; + if (!cgroup_name.starts_with(v2_prefix)) + return {}; + cgroup_name = cgroup_name.substr(v2_prefix.length()); + + std::filesystem::path current_cgroup = cgroup_name.empty() ? default_cgroups_mount : (default_cgroups_mount / cgroup_name); + + /// Open the bottom-most nested memory limit setting file. If there is no such file at the current + /// level, try again at the parent level as memory settings are inherited. + while (current_cgroup != default_cgroups_mount.parent_path()) + { + std::ifstream setting_file(current_cgroup / "memory.max"); + if (setting_file.is_open()) + { + uint64_t value; + if (setting_file >> value) + return {value}; + else + return {}; /// e.g. the cgroups default "max" + } + current_cgroup = current_cgroup.parent_path(); + } + + return {}; +#else + return {}; +#endif +} + +} + /** Returns the size of physical memory (RAM) in bytes. * Returns 0 on unsupported platform */ @@ -26,34 +103,27 @@ uint64_t getMemoryAmountOrZero() uint64_t memory_amount = num_pages * page_size; -#if defined(OS_LINUX) - // Try to lookup at the Cgroup limit - - // CGroups v2 - std::ifstream cgroupv2_limit("/sys/fs/cgroup/memory.max"); - if (cgroupv2_limit.is_open()) - { - uint64_t memory_limit = 0; - cgroupv2_limit >> memory_limit; - if (memory_limit > 0 && memory_limit < memory_amount) - memory_amount = memory_limit; - } + /// Respect the memory limit set by cgroups v2. + auto limit_v2 = getCgroupsV2MemoryLimit(); + if (limit_v2.has_value() && *limit_v2 < memory_amount) + memory_amount = *limit_v2; else { - // CGroups v1 - std::ifstream cgroup_limit("/sys/fs/cgroup/memory/memory.limit_in_bytes"); - if (cgroup_limit.is_open()) + /// Cgroups v1 were replaced by v2 in 2015. The only reason we keep supporting v1 is that the transition to v2 + /// has been slow. Caveat : Hierarchical groups as in v2 are not supported for v1, the location of the memory + /// limit (virtual) file is hard-coded. + /// TODO: check at the end of 2024 if we can get rid of v1. + std::ifstream limit_file_v1("/sys/fs/cgroup/memory/memory.limit_in_bytes"); + if (limit_file_v1.is_open()) { - uint64_t memory_limit = 0; // in case of read error - cgroup_limit >> memory_limit; - if (memory_limit > 0 && memory_limit < memory_amount) - memory_amount = memory_limit; + uint64_t limit_v1; + if (limit_file_v1 >> limit_v1) + if (limit_v1 < memory_amount) + memory_amount = limit_v1; } } -#endif return memory_amount; - } diff --git a/base/base/int8_to_string.cpp b/base/base/int8_to_string.cpp new file mode 100644 index 00000000000..f74a6b8077e --- /dev/null +++ b/base/base/int8_to_string.cpp @@ -0,0 +1,9 @@ +#include + +namespace std +{ +std::string to_string(Int8 v) /// NOLINT (cert-dcl58-cpp) +{ + return to_string(int8_t{v}); +} +} diff --git a/base/base/int8_to_string.h b/base/base/int8_to_string.h new file mode 100644 index 00000000000..af0914f4312 --- /dev/null +++ b/base/base/int8_to_string.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include + +#include + +template <> +struct fmt::formatter : fmt::formatter +{ +}; + + +namespace std +{ +std::string to_string(Int8 v); /// NOLINT (cert-dcl58-cpp) +} diff --git a/base/base/types.h b/base/base/types.h index 3a7760eae91..a4874860514 100644 --- a/base/base/types.h +++ b/base/base/types.h @@ -3,14 +3,29 @@ #include #include -/// This is needed for more strict aliasing. https://godbolt.org/z/xpJBSb https://stackoverflow.com/a/57453713 +/// Using char8_t more strict aliasing (https://stackoverflow.com/a/57453713) using UInt8 = char8_t; +/// Same for using signed _BitInt(8) (there isn't a signed char8_t, which would be more convenient) +/// See https://godbolt.org/z/fafnWEnnf +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wbit-int-extension" +using Int8 = signed _BitInt(8); +#pragma clang diagnostic pop + +namespace std +{ +template <> +struct hash /// NOLINT (cert-dcl58-cpp) +{ + size_t operator()(const Int8 x) const { return std::hash()(int8_t{x}); } +}; +} + using UInt16 = uint16_t; using UInt32 = uint32_t; using UInt64 = uint64_t; -using Int8 = int8_t; using Int16 = int16_t; using Int32 = int32_t; using Int64 = int64_t; diff --git a/docs/en/sql-reference/data-types/variant.md b/docs/en/sql-reference/data-types/variant.md index 17d51878420..f027e3fe343 100644 --- a/docs/en/sql-reference/data-types/variant.md +++ b/docs/en/sql-reference/data-types/variant.md @@ -1,5 +1,5 @@ --- -slug: /en/sql-reference/data-types/json +slug: /en/sql-reference/data-types/variant sidebar_position: 55 sidebar_label: Variant --- diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 0988e1eb4a1..cc142470d7f 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -504,7 +504,7 @@ void Client::connect() << "It may lack support for new features." << std::endl << std::endl; } - else if (client_version_tuple > server_version_tuple) + else if (client_version_tuple > server_version_tuple && server_display_name != "clickhouse-cloud") { std::cout << "ClickHouse server version is older than ClickHouse client. " << "It may indicate that the server is out of date and can be upgraded." << std::endl diff --git a/src/AggregateFunctions/AggregateFunctionTopK.cpp b/src/AggregateFunctions/AggregateFunctionTopK.cpp index 660f136e4dc..dcbd5586406 100644 --- a/src/AggregateFunctions/AggregateFunctionTopK.cpp +++ b/src/AggregateFunctions/AggregateFunctionTopK.cpp @@ -234,6 +234,9 @@ public: void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override { + if (!this->data(rhs).value.size()) + return; + auto & set = this->data(place).value; if (set.capacity() != reserved) set.resize(reserved); diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 560be1ad3bf..fcff8c7cf5f 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -2766,7 +2766,13 @@ QueryTreeNodePtr QueryAnalyzer::tryResolveIdentifierFromAliases(const Identifier { if (identifier_lookup.isExpressionLookup()) { - return tryResolveIdentifierFromCompoundExpression(identifier_lookup.identifier, 1 /*identifier_bind_size*/, it->second, {}, scope); + return tryResolveIdentifierFromCompoundExpression( + identifier_lookup.identifier, + 1 /*identifier_bind_size*/, + it->second, + {} /* compound_expression_source */, + scope, + identifier_resolve_settings.allow_to_check_join_tree /* can_be_not_found */); } else if (identifier_lookup.isFunctionLookup() || identifier_lookup.isTableExpressionLookup()) { diff --git a/src/Common/ElapsedTimeProfileEventIncrement.h b/src/Common/ElapsedTimeProfileEventIncrement.h index b30afd24a4c..731295a4cfd 100644 --- a/src/Common/ElapsedTimeProfileEventIncrement.h +++ b/src/Common/ElapsedTimeProfileEventIncrement.h @@ -14,12 +14,13 @@ enum Time Seconds, }; -template