diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index fede3fe519d..85b1d460833 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -44,22 +44,35 @@ At a minimum, the following information should be added (but add more as needed) --- ### Modify your CI run: **NOTE:** If your merge the PR with modified CI you **MUST KNOW** what you are doing -**NOTE:** Set desired options before CI starts or re-push after updates +**NOTE:** Checked options will be applied if set before CI RunConfig/PrepareRunConfig step -#### Run only: -- [ ] Integration tests -- [ ] Integration tests (arm64) -- [ ] Stateless tests (release) -- [ ] Stateless tests (asan) -- [ ] Stateful tests (release) -- [ ] Stateful tests (asan) -- [ ] No sanitizers -- [ ] Tests with analyzer -- [ ] Fast tests -- [ ] Only package_debug build -- [ ] Add your CI variant description here +#### Include tests (required builds will be added automatically): +- [ ] Fast test +- [ ] Integration Tests +- [ ] Stateless tests +- [ ] Stateful tests +- [ ] Unit tests +- [ ] Performance tests +- [ ] All with ASAN +- [ ] All with TSAN +- [ ] All with Analyzer +- [ ] Add your option here -#### CI options: +#### Exclude tests: +- [ ] Fast test +- [ ] Integration Tests +- [ ] Stateless tests +- [ ] Stateful tests +- [ ] Performance tests +- [ ] All with ASAN +- [ ] All with TSAN +- [ ] All with MSAN +- [ ] All with UBSAN +- [ ] All with Coverage +- [ ] All with Aarch64 +- [ ] Add your option here + +#### Extra options: - [ ] do not test (only style check) - [ ] disable merge-commit (no merge from master before tests) - [ ] disable CI cache (job reuse) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 2853adff48a..816bdfd4f31 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -374,7 +374,7 @@ jobs: if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml with: - test_name: Stateless tests (release, analyzer, s3, DatabaseReplicated) + test_name: Stateless tests (release, old analyzer, s3, DatabaseReplicated) runner_type: func-tester data: ${{ needs.RunConfig.outputs.data }} FunctionalStatelessTestS3Debug: @@ -632,7 +632,7 @@ jobs: if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml with: - test_name: Integration tests (asan, analyzer) + test_name: Integration tests (asan, old analyzer) runner_type: stress-tester data: ${{ needs.RunConfig.outputs.data }} IntegrationTestsTsan: diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 9e95b3d3d8f..4d45c8d8d4b 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -436,7 +436,7 @@ jobs: if: ${{ !failure() && !cancelled() }} uses: ./.github/workflows/reusable_test.yml with: - test_name: Integration tests (asan, analyzer) + test_name: Integration tests (asan, old analyzer) runner_type: stress-tester data: ${{ needs.RunConfig.outputs.data }} IntegrationTestsTsan: diff --git a/.gitignore b/.gitignore index 1ea8f83dcc2..db3f77d7d1e 100644 --- a/.gitignore +++ b/.gitignore @@ -164,6 +164,9 @@ tests/queries/0_stateless/*.generated-expect tests/queries/0_stateless/*.expect.history tests/integration/**/_gen +# pytest --pdb history +.pdb_history + # rust /rust/**/target* # It is autogenerated from *.in diff --git a/CHANGELOG.md b/CHANGELOG.md index e576fb447c1..dd88f3ee2c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ # 2024 Changelog -### ClickHouse release 24.3 LTS, 2024-03-26 +### ClickHouse release 24.3 LTS, 2024-03-27 #### Upgrade Notes * The setting `allow_experimental_analyzer` is enabled by default and it switches the query analysis to a new implementation, which has better compatibility and feature completeness. The feature "analyzer" is considered beta instead of experimental. You can turn the old behavior by setting the `compatibility` to `24.2` or disabling the `allow_experimental_analyzer` setting. Watch the [video on YouTube](https://www.youtube.com/watch?v=zhrOYQpgvkk). @@ -123,7 +123,6 @@ * Something was wrong with Apache Hive, which is experimental and not supported. [#60262](https://github.com/ClickHouse/ClickHouse/pull/60262) ([shanfengp](https://github.com/Aed-p)). * An improvement for experimental parallel replicas: force reanalysis if parallel replicas changed [#60362](https://github.com/ClickHouse/ClickHouse/pull/60362) ([Raúl Marín](https://github.com/Algunenano)). * Fix usage of plain metadata type with new disks configuration option [#60396](https://github.com/ClickHouse/ClickHouse/pull/60396) ([Kseniia Sumarokova](https://github.com/kssenii)). -* Don't allow to set max_parallel_replicas to 0 as it doesn't make sense [#60430](https://github.com/ClickHouse/ClickHouse/pull/60430) ([Kruglov Pavel](https://github.com/Avogar)). * Try to fix logical error 'Cannot capture column because it has incompatible type' in mapContainsKeyLike [#60451](https://github.com/ClickHouse/ClickHouse/pull/60451) ([Kruglov Pavel](https://github.com/Avogar)). * Avoid calculation of scalar subqueries for CREATE TABLE. [#60464](https://github.com/ClickHouse/ClickHouse/pull/60464) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). * Fix deadlock in parallel parsing when lots of rows are skipped due to errors [#60516](https://github.com/ClickHouse/ClickHouse/pull/60516) ([Kruglov Pavel](https://github.com/Avogar)). diff --git a/SECURITY.md b/SECURITY.md index 86578b188d8..4701f2ec70b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -13,18 +13,16 @@ The following versions of ClickHouse server are currently being supported with s | Version | Supported | |:-|:-| +| 24.3 | ✔️ | | 24.2 | ✔️ | | 24.1 | ✔️ | -| 23.12 | ✔️ | -| 23.11 | ❌ | -| 23.10 | ❌ | -| 23.9 | ❌ | +| 23.* | ❌ | | 23.8 | ✔️ | | 23.7 | ❌ | | 23.6 | ❌ | | 23.5 | ❌ | | 23.4 | ❌ | -| 23.3 | ✔️ | +| 23.3 | ❌ | | 23.2 | ❌ | | 23.1 | ❌ | | 22.* | ❌ | diff --git a/base/poco/JSON/src/pdjson.c b/base/poco/JSON/src/pdjson.c index 18768ac96d3..563fa277439 100644 --- a/base/poco/JSON/src/pdjson.c +++ b/base/poco/JSON/src/pdjson.c @@ -314,13 +314,13 @@ static int read_unicode(json_stream *json) if (l < 0xdc00 || l > 0xdfff) { json_error(json, "invalid surrogate pair continuation \\u%04lx out " - "of range (dc00-dfff)", l); + "of range (dc00-dfff)", (unsigned long)l); return -1; } cp = ((h - 0xd800) * 0x400) + ((l - 0xdc00) + 0x10000); } else if (cp >= 0xdc00 && cp <= 0xdfff) { - json_error(json, "dangling surrogate \\u%04lx", cp); + json_error(json, "dangling surrogate \\u%04lx", (unsigned long)cp); return -1; } diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 2929c64ded8..26cb0eb23c6 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54484) +SET(VERSION_REVISION 54485) SET(VERSION_MAJOR 24) -SET(VERSION_MINOR 3) +SET(VERSION_MINOR 4) SET(VERSION_PATCH 1) -SET(VERSION_GITHASH 891689a41506d00aa169548f5b4a8774351242c4) -SET(VERSION_DESCRIBE v24.3.1.1-testing) -SET(VERSION_STRING 24.3.1.1) +SET(VERSION_GITHASH 2c5c589a882ceec35439650337b92db3e76f0081) +SET(VERSION_DESCRIBE v24.4.1.1-testing) +SET(VERSION_STRING 24.4.1.1) # end of autochange diff --git a/contrib/NuRaft b/contrib/NuRaft index 4a12f99dfc9..08ac76ea80a 160000 --- a/contrib/NuRaft +++ b/contrib/NuRaft @@ -1 +1 @@ -Subproject commit 4a12f99dfc9d47c687ff7700b927cc76856225d1 +Subproject commit 08ac76ea80a37f89b12109c805eafe9f1dc9b991 diff --git a/contrib/nuraft-cmake/CMakeLists.txt b/contrib/nuraft-cmake/CMakeLists.txt index eaca00566d6..970ca4b9ce1 100644 --- a/contrib/nuraft-cmake/CMakeLists.txt +++ b/contrib/nuraft-cmake/CMakeLists.txt @@ -32,6 +32,7 @@ set(SRCS "${LIBRARY_DIR}/src/handle_custom_notification.cxx" "${LIBRARY_DIR}/src/handle_vote.cxx" "${LIBRARY_DIR}/src/launcher.cxx" + "${LIBRARY_DIR}/src/log_entry.cxx" "${LIBRARY_DIR}/src/srv_config.cxx" "${LIBRARY_DIR}/src/snapshot_sync_req.cxx" "${LIBRARY_DIR}/src/snapshot_sync_ctx.cxx" diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index 17eee6d4287..3daa62cb212 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="24.2.2.71" +ARG VERSION="24.3.1.2672" ARG PACKAGES="clickhouse-keeper" ARG DIRECT_DOWNLOAD_URLS="" @@ -44,7 +44,10 @@ ARG DIRECT_DOWNLOAD_URLS="" # We do that in advance at the begining of Dockerfile before any packages will be # installed to prevent picking those uid / gid by some unrelated software. # The same uid / gid (101) is used both for alpine and ubuntu. - +ARG DEFAULT_UID="101" +ARG DEFAULT_GID="101" +RUN addgroup -S -g "${DEFAULT_GID}" clickhouse && \ + adduser -S -h "/var/lib/clickhouse" -s /bin/bash -G clickhouse -g "ClickHouse keeper" -u "${DEFAULT_UID}" clickhouse ARG TARGETARCH RUN arch=${TARGETARCH:-amd64} \ @@ -71,20 +74,21 @@ RUN arch=${TARGETARCH:-amd64} \ fi \ ; done \ && rm /tmp/*.tgz /install -r \ - && addgroup -S -g 101 clickhouse \ - && adduser -S -h /var/lib/clickhouse -s /bin/bash -G clickhouse -g "ClickHouse keeper" -u 101 clickhouse \ - && mkdir -p /var/lib/clickhouse /var/log/clickhouse-keeper /etc/clickhouse-keeper \ - && chown clickhouse:clickhouse /var/lib/clickhouse \ - && chown root:clickhouse /var/log/clickhouse-keeper \ && chmod +x /entrypoint.sh \ && apk add --no-cache su-exec bash tzdata \ && cp /usr/share/zoneinfo/UTC /etc/localtime \ - && echo "UTC" > /etc/timezone \ - && chmod ugo+Xrw -R /var/lib/clickhouse /var/log/clickhouse-keeper /etc/clickhouse-keeper + && echo "UTC" > /etc/timezone +ARG DEFAULT_CONFIG_DIR="/etc/clickhouse-keeper" +ARG DEFAULT_DATA_DIR="/var/lib/clickhouse-keeper" +ARG DEFAULT_LOG_DIR="/var/log/clickhouse-keeper" +RUN mkdir -p "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CONFIG_DIR}" \ + && chown clickhouse:clickhouse "${DEFAULT_DATA_DIR}" \ + && chown root:clickhouse "${DEFAULT_LOG_DIR}" \ + && chmod ugo+Xrw -R "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CONFIG_DIR}" +# /var/lib/clickhouse is necessary due to the current default configuration for Keeper +VOLUME "${DEFAULT_DATA_DIR}" /var/lib/clickhouse EXPOSE 2181 10181 44444 9181 -VOLUME /var/lib/clickhouse /var/log/clickhouse-keeper /etc/clickhouse-keeper - ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/keeper/entrypoint.sh b/docker/keeper/entrypoint.sh index 939cd941aeb..1390ad9ce74 100644 --- a/docker/keeper/entrypoint.sh +++ b/docker/keeper/entrypoint.sh @@ -80,7 +80,7 @@ if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then # so the container can't be finished by ctrl+c export CLICKHOUSE_WATCHDOG_ENABLE - cd /var/lib/clickhouse + cd "${DATA_DIR}" # There is a config file. It is already tested with gosu (if it is readably by keeper user) if [ -f "$KEEPER_CONFIG" ]; then diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index bd5fa313adc..ace01ae9a9f 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="24.2.2.71" +ARG VERSION="24.3.1.2672" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" ARG DIRECT_DOWNLOAD_URLS="" @@ -42,6 +42,10 @@ ARG DIRECT_DOWNLOAD_URLS="" # We do that in advance at the begining of Dockerfile before any packages will be # installed to prevent picking those uid / gid by some unrelated software. # The same uid / gid (101) is used both for alpine and ubuntu. +ARG DEFAULT_UID="101" +ARG DEFAULT_GID="101" +RUN addgroup -S -g "${DEFAULT_GID}" clickhouse && \ + adduser -S -h "/var/lib/clickhouse" -s /bin/bash -G clickhouse -g "ClickHouse server" -u "${DEFAULT_UID}" clickhouse RUN arch=${TARGETARCH:-amd64} \ && cd /tmp \ @@ -66,23 +70,30 @@ RUN arch=${TARGETARCH:-amd64} \ fi \ ; done \ && rm /tmp/*.tgz /install -r \ - && addgroup -S -g 101 clickhouse \ - && adduser -S -h /var/lib/clickhouse -s /bin/bash -G clickhouse -g "ClickHouse server" -u 101 clickhouse \ - && mkdir -p /var/lib/clickhouse /var/log/clickhouse-server /etc/clickhouse-server/config.d /etc/clickhouse-server/users.d /etc/clickhouse-client /docker-entrypoint-initdb.d \ - && chown clickhouse:clickhouse /var/lib/clickhouse \ - && chown root:clickhouse /var/log/clickhouse-server \ && chmod +x /entrypoint.sh \ && apk add --no-cache bash tzdata \ && cp /usr/share/zoneinfo/UTC /etc/localtime \ - && echo "UTC" > /etc/timezone \ - && chmod ugo+Xrw -R /var/lib/clickhouse /var/log/clickhouse-server /etc/clickhouse-server /etc/clickhouse-client + && echo "UTC" > /etc/timezone -# we need to allow "others" access to clickhouse folder, because docker container -# can be started with arbitrary uid (openshift usecase) +ARG DEFAULT_CLIENT_CONFIG_DIR="/etc/clickhouse-client" +ARG DEFAULT_SERVER_CONFIG_DIR="/etc/clickhouse-server" +ARG DEFAULT_DATA_DIR="/var/lib/clickhouse" +ARG DEFAULT_LOG_DIR="/var/log/clickhouse-server" +# we need to allow "others" access to ClickHouse folders, because docker containers +# can be started with arbitrary uids (OpenShift usecase) +RUN mkdir -p \ + "${DEFAULT_DATA_DIR}" \ + "${DEFAULT_LOG_DIR}" \ + "${DEFAULT_CLIENT_CONFIG_DIR}" \ + "${DEFAULT_SERVER_CONFIG_DIR}/config.d" \ + "${DEFAULT_SERVER_CONFIG_DIR}/users.d" \ + /docker-entrypoint-initdb.d \ + && chown clickhouse:clickhouse "${DEFAULT_DATA_DIR}" \ + && chown root:clickhouse "${DEFAULT_LOG_DIR}" \ + && chmod ugo+Xrw -R "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CLIENT_CONFIG_DIR}" "${DEFAULT_SERVER_CONFIG_DIR}" + +VOLUME "${DEFAULT_DATA_DIR}" EXPOSE 9000 8123 9009 -VOLUME /var/lib/clickhouse \ - /var/log/clickhouse-server - ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 256dcdc029f..e92823b686a 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -27,7 +27,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="24.2.2.71" +ARG VERSION="24.3.1.2672" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index 98da5988ad5..c2e9fdfe41d 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -25,7 +25,7 @@ azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log & config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml cache_policy="" -if [ $(( $(date +%-d) % 2 )) -eq 1 ]; then +if [ $(($RANDOM%2)) -eq 1 ]; then cache_policy="SLRU" else cache_policy="LRU" diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index bac9d8df7a9..b9ed0561a48 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -257,10 +257,10 @@ do echo "$err" [[ "0" != "${#err}" ]] && failed_to_save_logs=1 if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then - err=$( { clickhouse-client -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst; } 2>&1 ) + err=$( { clickhouse-client --port 19000 -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst; } 2>&1 ) echo "$err" [[ "0" != "${#err}" ]] && failed_to_save_logs=1 - err=$( { clickhouse-client -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.2.tsv.zst; } 2>&1 ) + err=$( { clickhouse-client --port 29000 -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.2.tsv.zst; } 2>&1 ) echo "$err" [[ "0" != "${#err}" ]] && failed_to_save_logs=1 fi diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index ea7e3aece1d..6c6caf872e9 100644 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -72,7 +72,7 @@ mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/c # Randomize cache policies. cache_policy="" -if [ $(( $(date +%-d) % 2 )) -eq 1 ]; then +if [ $(($RANDOM%2)) -eq 1 ]; then cache_policy="SLRU" else cache_policy="LRU" @@ -87,6 +87,25 @@ if [ "$cache_policy" = "SLRU" ]; then mv /etc/clickhouse-server/config.d/storage_conf.xml.tmp /etc/clickhouse-server/config.d/storage_conf.xml fi +# Disable experimental WINDOW VIEW tests for stress tests, since they may be +# created with old analyzer and then, after server restart it will refuse to +# start. +# FIXME: remove once the support for WINDOW VIEW will be implemented in analyzer. +sudo cat /etc/clickhouse-server/users.d/stress_tests_overrides.xml < + + + false + + + + + + + + +EOL + start_server clickhouse-client --query "SHOW TABLES FROM datasets" diff --git a/docs/changelogs/v24.3.1.2672-lts.md b/docs/changelogs/v24.3.1.2672-lts.md new file mode 100644 index 00000000000..e5d008680a8 --- /dev/null +++ b/docs/changelogs/v24.3.1.2672-lts.md @@ -0,0 +1,537 @@ +--- +sidebar_position: 1 +sidebar_label: 2024 +--- + +# 2024 Changelog + +### ClickHouse release v24.3.1.2672-lts (2c5c589a882) FIXME as compared to v24.2.1.2248-stable (891689a4150) + +#### Backward Incompatible Change +* Don't allow to set max_parallel_replicas to 0 as it doesn't make sense. Setting it to 0 could lead to unexpected logical errors. Closes [#60140](https://github.com/ClickHouse/ClickHouse/issues/60140). [#60430](https://github.com/ClickHouse/ClickHouse/pull/60430) ([Kruglov Pavel](https://github.com/Avogar)). +* Change the column name from `duration_ms` to `duration_microseconds` in the `system.zookeeper` table to reflect the reality that the duration is in the microsecond resolution. [#60774](https://github.com/ClickHouse/ClickHouse/pull/60774) ([Duc Canh Le](https://github.com/canhld94)). +* Reject incoming INSERT queries in case when query-level settings `async_insert` and `deduplicate_blocks_in_dependent_materialized_views` are enabled together. This behaviour is controlled by a setting `throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert` and enabled by default. This is a continuation of https://github.com/ClickHouse/ClickHouse/pull/59699 needed to unblock https://github.com/ClickHouse/ClickHouse/pull/59915. [#60888](https://github.com/ClickHouse/ClickHouse/pull/60888) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Utility `clickhouse-copier` is moved to a separate repository on GitHub: https://github.com/ClickHouse/copier. It is no longer included in the bundle but is still available as a separate download. This closes: [#60734](https://github.com/ClickHouse/ClickHouse/issues/60734) This closes: [#60540](https://github.com/ClickHouse/ClickHouse/issues/60540) This closes: [#60250](https://github.com/ClickHouse/ClickHouse/issues/60250) This closes: [#52917](https://github.com/ClickHouse/ClickHouse/issues/52917) This closes: [#51140](https://github.com/ClickHouse/ClickHouse/issues/51140) This closes: [#47517](https://github.com/ClickHouse/ClickHouse/issues/47517) This closes: [#47189](https://github.com/ClickHouse/ClickHouse/issues/47189) This closes: [#46598](https://github.com/ClickHouse/ClickHouse/issues/46598) This closes: [#40257](https://github.com/ClickHouse/ClickHouse/issues/40257) This closes: [#36504](https://github.com/ClickHouse/ClickHouse/issues/36504) This closes: [#35485](https://github.com/ClickHouse/ClickHouse/issues/35485) This closes: [#33702](https://github.com/ClickHouse/ClickHouse/issues/33702) This closes: [#26702](https://github.com/ClickHouse/ClickHouse/issues/26702) ### Documentation entry for user-facing changes. [#61058](https://github.com/ClickHouse/ClickHouse/pull/61058) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* To increase compatibility with MySQL, function `locate` now accepts arguments `(needle, haystack[, start_pos])` by default. The previous behavior `(haystack, needle, [, start_pos])` can be restored by setting `function_locate_has_mysql_compatible_argument_order = 0`. [#61092](https://github.com/ClickHouse/ClickHouse/pull/61092) ([Robert Schulze](https://github.com/rschu1ze)). +* The obsolete in-memory data parts have been deprecated since version 23.5 and have not been supported since version 23.10. Now the remaining code is removed. Continuation of [#55186](https://github.com/ClickHouse/ClickHouse/issues/55186) and [#45409](https://github.com/ClickHouse/ClickHouse/issues/45409). It is unlikely that you have used in-memory data parts because they were available only before version 23.5 and only when you enabled them manually by specifying the corresponding SETTINGS for a MergeTree table. To check if you have in-memory data parts, run the following query: `SELECT part_type, count() FROM system.parts GROUP BY part_type ORDER BY part_type`. To disable the usage of in-memory data parts, do `ALTER TABLE ... MODIFY SETTING min_bytes_for_compact_part = DEFAULT, min_rows_for_compact_part = DEFAULT`. Before upgrading from old ClickHouse releases, first check that you don't have in-memory data parts. If there are in-memory data parts, disable them first, then wait while there are no in-memory data parts and continue the upgrade. [#61127](https://github.com/ClickHouse/ClickHouse/pull/61127) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Forbid `SimpleAggregateFunction` in `ORDER BY` of `MergeTree` tables (like `AggregateFunction` is forbidden, but they are forbidden because they are not comparable) by default (use `allow_suspicious_primary_key` to allow them). [#61399](https://github.com/ClickHouse/ClickHouse/pull/61399) ([Azat Khuzhin](https://github.com/azat)). +* ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. This is controlled by the settings, `output_format_parquet_string_as_string`, `output_format_orc_string_as_string`, `output_format_arrow_string_as_string`. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases. Parquet/ORC/Arrow supports many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools lack support for the faster `lz4` compression method, that's why we set `zstd` by default. This is controlled by the settings `output_format_parquet_compression_method`, `output_format_orc_compression_method`, and `output_format_arrow_compression_method`. We changed the default to `zstd` for Parquet and ORC, but not Arrow (it is emphasized for low-level usages). [#61817](https://github.com/ClickHouse/ClickHouse/pull/61817) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* In the new ClickHouse version, the functions `geoDistance`, `greatCircleDistance`, and `greatCircleAngle` will use 64-bit double precision floating point data type for internal calculations and return type if all the arguments are Float64. This closes [#58476](https://github.com/ClickHouse/ClickHouse/issues/58476). In previous versions, the function always used Float32. You can switch to the old behavior by setting `geo_distance_returns_float64_on_float64_arguments` to `false` or setting `compatibility` to `24.2` or earlier. [#61848](https://github.com/ClickHouse/ClickHouse/pull/61848) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### New Feature +* Topk/topkweighed support mode, which return count of values and it's error. [#54508](https://github.com/ClickHouse/ClickHouse/pull/54508) ([UnamedRus](https://github.com/UnamedRus)). +* Add generate_series as a table function. This function generates table with an arithmetic progression with natural numbers. [#59390](https://github.com/ClickHouse/ClickHouse/pull/59390) ([divanik](https://github.com/divanik)). +* Support reading and writing backups as tar archives. [#59535](https://github.com/ClickHouse/ClickHouse/pull/59535) ([josh-hildred](https://github.com/josh-hildred)). +* Implemented support for S3Express buckets. [#59965](https://github.com/ClickHouse/ClickHouse/pull/59965) ([Nikita Taranov](https://github.com/nickitat)). +* Allow to attach parts from a different disk * attach partition from the table on other disks using copy instead of hard link (such as instant table) * attach partition using copy when the hard link fails even on the same disk. [#60112](https://github.com/ClickHouse/ClickHouse/pull/60112) ([Unalian](https://github.com/Unalian)). +* Added function `toMillisecond` which returns the millisecond component for values of type`DateTime` or `DateTime64`. [#60281](https://github.com/ClickHouse/ClickHouse/pull/60281) ([Shaun Struwig](https://github.com/Blargian)). +* Make all format names case insensitive, like Tsv, or TSV, or tsv, or even rowbinary. [#60420](https://github.com/ClickHouse/ClickHouse/pull/60420) ([豪肥肥](https://github.com/HowePa)). +* Add four properties to the `StorageMemory` (memory-engine) `min_bytes_to_keep, max_bytes_to_keep, min_rows_to_keep` and `max_rows_to_keep` - Add tests to reflect new changes - Update `memory.md` documentation - Add table `context` property to `MemorySink` to enable access to table parameter bounds. [#60612](https://github.com/ClickHouse/ClickHouse/pull/60612) ([Jake Bamrah](https://github.com/JakeBamrah)). +* Added function `toMillisecond` which returns the millisecond component for values of type`DateTime` or `DateTime64`. [#60649](https://github.com/ClickHouse/ClickHouse/pull/60649) ([Robert Schulze](https://github.com/rschu1ze)). +* Separate limits on number of waiting and executing queries. Added new server setting `max_waiting_queries` that limits the number of queries waiting due to `async_load_databases`. Existing limits on number of executing queries no longer count waiting queries. [#61053](https://github.com/ClickHouse/ClickHouse/pull/61053) ([Sergei Trifonov](https://github.com/serxa)). +* Add support for `ATTACH PARTITION ALL`. [#61107](https://github.com/ClickHouse/ClickHouse/pull/61107) ([Kirill Nikiforov](https://github.com/allmazz)). +* Add a new function, `getClientHTTPHeader`. This closes [#54665](https://github.com/ClickHouse/ClickHouse/issues/54665). Co-authored with @lingtaolf. [#61820](https://github.com/ClickHouse/ClickHouse/pull/61820) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Performance Improvement +* Improve the performance of serialized aggregation method when involving multiple [nullable] columns. This is a general version of [#51399](https://github.com/ClickHouse/ClickHouse/issues/51399) that doesn't compromise on abstraction integrity. [#55809](https://github.com/ClickHouse/ClickHouse/pull/55809) ([Amos Bird](https://github.com/amosbird)). +* Lazy build join output to improve performance of ALL join. [#58278](https://github.com/ClickHouse/ClickHouse/pull/58278) ([LiuNeng](https://github.com/liuneng1994)). +* Improvements to aggregate functions ArgMin / ArgMax / any / anyLast / anyHeavy, as well as `ORDER BY {u8/u16/u32/u64/i8/i16/u32/i64) LIMIT 1` queries. [#58640](https://github.com/ClickHouse/ClickHouse/pull/58640) ([Raúl Marín](https://github.com/Algunenano)). +* Trivial optimize on column filter. Avoid those filter columns whoes underlying data type is not number being filtered with `result_size_hint = -1`. Peak memory can be reduced to 44% of the original in some cases. [#59698](https://github.com/ClickHouse/ClickHouse/pull/59698) ([李扬](https://github.com/taiyang-li)). +* If the table's primary key contains mostly useless columns, don't keep them in memory. This is controlled by a new setting `primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns` with the value `0.9` by default, which means: for a composite primary key, if a column changes its value for at least 0.9 of all the times, the next columns after it will be not loaded. [#60255](https://github.com/ClickHouse/ClickHouse/pull/60255) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Execute multiIf function columnarly when result_type's underlying type is number. [#60384](https://github.com/ClickHouse/ClickHouse/pull/60384) ([李扬](https://github.com/taiyang-li)). +* Faster (almost 2x) mutexes (was slower due to ThreadFuzzer). [#60823](https://github.com/ClickHouse/ClickHouse/pull/60823) ([Azat Khuzhin](https://github.com/azat)). +* Move connection drain from prepare to work, and drain multiple connections in parallel. [#60845](https://github.com/ClickHouse/ClickHouse/pull/60845) ([lizhuoyu5](https://github.com/lzydmxy)). +* Optimize insertManyFrom of nullable number or nullable string. [#60846](https://github.com/ClickHouse/ClickHouse/pull/60846) ([李扬](https://github.com/taiyang-li)). +* Optimized function `dotProduct` to omit unnecessary and expensive memory copies. [#60928](https://github.com/ClickHouse/ClickHouse/pull/60928) ([Robert Schulze](https://github.com/rschu1ze)). +* Operations with the filesystem cache will suffer less from the lock contention. [#61066](https://github.com/ClickHouse/ClickHouse/pull/61066) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Optimize ColumnString::replicate and prevent memcpySmallAllowReadWriteOverflow15Impl from being optimized to built-in memcpy. Close [#61074](https://github.com/ClickHouse/ClickHouse/issues/61074). ColumnString::replicate speeds up by 2.46x on x86-64. [#61075](https://github.com/ClickHouse/ClickHouse/pull/61075) ([李扬](https://github.com/taiyang-li)). +* 30x faster printing for 256-bit integers. [#61100](https://github.com/ClickHouse/ClickHouse/pull/61100) ([Raúl Marín](https://github.com/Algunenano)). +* If a query with a syntax error contained COLUMNS matcher with a regular expression, the regular expression was compiled each time during the parser's backtracking, instead of being compiled once. This was a fundamental error. The compiled regexp was put to AST. But the letter A in AST means "abstract" which means it should not contain heavyweight objects. Parts of AST can be created and discarded during parsing, including a large number of backtracking. This leads to slowness on the parsing side and consequently allows DoS by a readonly user. But the main problem is that it prevents progress in fuzzers. [#61543](https://github.com/ClickHouse/ClickHouse/pull/61543) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a new analyzer pass to optimize in single value. [#61564](https://github.com/ClickHouse/ClickHouse/pull/61564) ([LiuNeng](https://github.com/liuneng1994)). + +#### Improvement +* While running the MODIFY COLUMN query for materialized views, check the inner table's structure to ensure every column exists. [#47427](https://github.com/ClickHouse/ClickHouse/pull/47427) ([sunny](https://github.com/sunny19930321)). +* Added table `system.keywords` which contains all the keywords from parser. Mostly needed and will be used for better fuzzing and syntax highlighting. [#51808](https://github.com/ClickHouse/ClickHouse/pull/51808) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Ordinary database engine is deprecated. You will receive a warning in clickhouse-client if your server is using it. This closes [#52229](https://github.com/ClickHouse/ClickHouse/issues/52229). [#56942](https://github.com/ClickHouse/ClickHouse/pull/56942) ([shabroo](https://github.com/shabroo)). +* All zero copy locks related to a table have to be dropped when the table is dropped. The directory which contains these locks has to be removed also. [#57575](https://github.com/ClickHouse/ClickHouse/pull/57575) ([Sema Checherinda](https://github.com/CheSema)). +* Allow declaring enum in external table structure. [#57857](https://github.com/ClickHouse/ClickHouse/pull/57857) ([Duc Canh Le](https://github.com/canhld94)). +* Consider lightweight deleted rows when selecting parts to merge. [#58223](https://github.com/ClickHouse/ClickHouse/pull/58223) ([Zhuo Qiu](https://github.com/jewelzqiu)). +* This PR makes http/https connections reusable for all uses cases. Even when response is 3xx or 4xx. [#58845](https://github.com/ClickHouse/ClickHouse/pull/58845) ([Sema Checherinda](https://github.com/CheSema)). +* Added comments for columns for more system tables. Continuation of https://github.com/ClickHouse/ClickHouse/pull/58356. [#59016](https://github.com/ClickHouse/ClickHouse/pull/59016) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Now we can use virtual columns in PREWHERE. It's worthwhile for non-const virtual columns like `_part_offset`. [#59033](https://github.com/ClickHouse/ClickHouse/pull/59033) ([Amos Bird](https://github.com/amosbird)). +* Add ability to skip read-only replicas for INSERT into Distributed engine (Controlled with `distributed_insert_skip_read_only_replicas` setting, by default OFF - backward compatible). [#59176](https://github.com/ClickHouse/ClickHouse/pull/59176) ([Azat Khuzhin](https://github.com/azat)). +* Instead using a constant key, now object storage generates key for determining remove objects capability. [#59495](https://github.com/ClickHouse/ClickHouse/pull/59495) ([Sema Checherinda](https://github.com/CheSema)). +* Add positional pread in libhdfs3. If you want to call positional read in libhdfs3, use the hdfsPread function in hdfs.h as follows. `tSize hdfsPread(hdfsFS fs, hdfsFile file, void * buffer, tSize length, tOffset position);`. [#59624](https://github.com/ClickHouse/ClickHouse/pull/59624) ([M1eyu](https://github.com/M1eyu2018)). +* Add asynchronous WriteBuffer for AzureBlobStorage similar to S3. [#59929](https://github.com/ClickHouse/ClickHouse/pull/59929) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Allow "local" as object storage type instead of "local_blob_storage". [#60165](https://github.com/ClickHouse/ClickHouse/pull/60165) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Improved overall usability of virtual columns. Now it is allowed to use virtual columns in `PREWHERE` (it's worthwhile for non-const virtual columns like `_part_offset`). Now a builtin documentation is available for virtual columns as a comment of column in `DESCRIBE` query with enabled setting `describe_include_virtual_columns`. [#60205](https://github.com/ClickHouse/ClickHouse/pull/60205) ([Anton Popov](https://github.com/CurtizJ)). +* Parallel flush of pending INSERT blocks of Distributed engine on `DETACH`/server shutdown and `SYSTEM FLUSH DISTRIBUTED` (Parallelism will work only if you have multi disk policy for table (like everything in Distributed engine right now)). [#60225](https://github.com/ClickHouse/ClickHouse/pull/60225) ([Azat Khuzhin](https://github.com/azat)). +* Filter setting is improper in `joinRightColumnsSwitchNullability`, resolve [#59625](https://github.com/ClickHouse/ClickHouse/issues/59625). [#60259](https://github.com/ClickHouse/ClickHouse/pull/60259) ([lgbo](https://github.com/lgbo-ustc)). +* Add a setting to force read-through cache for merges. [#60308](https://github.com/ClickHouse/ClickHouse/pull/60308) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Issue [#57598](https://github.com/ClickHouse/ClickHouse/issues/57598) mentions a variant behaviour regarding transaction handling. An issued COMMIT/ROLLBACK when no transaction is active is reported as an error contrary to MySQL behaviour. [#60338](https://github.com/ClickHouse/ClickHouse/pull/60338) ([PapaToemmsn](https://github.com/PapaToemmsn)). +* Added `none_only_active` mode for `distributed_ddl_output_mode` setting. [#60340](https://github.com/ClickHouse/ClickHouse/pull/60340) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Allow configuring HTTP redirect handlers for clickhouse-server. For example, you can make `/` redirect to the Play UI. [#60390](https://github.com/ClickHouse/ClickHouse/pull/60390) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The advanced dashboard has slightly better colors for multi-line graphs. [#60391](https://github.com/ClickHouse/ClickHouse/pull/60391) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Function `substring` now has a new alias `byteSlice`. [#60494](https://github.com/ClickHouse/ClickHouse/pull/60494) ([Robert Schulze](https://github.com/rschu1ze)). +* Renamed server setting `dns_cache_max_size` to `dns_cache_max_entries` to reduce ambiguity. [#60500](https://github.com/ClickHouse/ClickHouse/pull/60500) ([Kirill Nikiforov](https://github.com/allmazz)). +* `SHOW INDEX | INDEXES | INDICES | KEYS` no longer sorts by the primary key columns (which was unintuitive). [#60514](https://github.com/ClickHouse/ClickHouse/pull/60514) ([Robert Schulze](https://github.com/rschu1ze)). +* Keeper improvement: abort during startup if an invalid snapshot is detected to avoid data loss. [#60537](https://github.com/ClickHouse/ClickHouse/pull/60537) ([Antonio Andelic](https://github.com/antonio2368)). +* Added MergeTree read split ranges into intersecting and non intersecting fault injection using `merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_fault_probability` setting. [#60548](https://github.com/ClickHouse/ClickHouse/pull/60548) ([Maksim Kita](https://github.com/kitaisreal)). +* The Advanced dashboard now has controls always visible on scrolling. This allows you to add a new chart without scrolling up. [#60692](https://github.com/ClickHouse/ClickHouse/pull/60692) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* String types and Enums can be used in the same context, such as: arrays, UNION queries, conditional expressions. This closes [#60726](https://github.com/ClickHouse/ClickHouse/issues/60726). [#60727](https://github.com/ClickHouse/ClickHouse/pull/60727) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update tzdata to 2024a. [#60768](https://github.com/ClickHouse/ClickHouse/pull/60768) ([Raúl Marín](https://github.com/Algunenano)). +* Support files without format extension in Filesystem database. [#60795](https://github.com/ClickHouse/ClickHouse/pull/60795) ([Kruglov Pavel](https://github.com/Avogar)). +* Keeper improvement: support `leadership_expiry_ms` in Keeper's settings. [#60806](https://github.com/ClickHouse/ClickHouse/pull/60806) ([Brokenice0415](https://github.com/Brokenice0415)). +* Always infer exponential numbers in JSON formats regardless of the setting `input_format_try_infer_exponent_floats`. Add setting `input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects` that allows to use String type for ambiguous paths instead of an exception during named Tuples inference from JSON objects. [#60808](https://github.com/ClickHouse/ClickHouse/pull/60808) ([Kruglov Pavel](https://github.com/Avogar)). +* Add support for `START TRANSACTION` syntax typically used in MySQL syntax, resolving https://github.com/ClickHouse/ClickHouse/discussions/60865. [#60886](https://github.com/ClickHouse/ClickHouse/pull/60886) ([Zach Naimon](https://github.com/ArctypeZach)). +* Add a flag for SMJ to treat null as biggest/smallest. So the behavior can be compitable with other SQL systems, like Apache Spark. [#60896](https://github.com/ClickHouse/ClickHouse/pull/60896) ([loudongfeng](https://github.com/loudongfeng)). +* Clickhouse version has been added to docker labels. Closes [#54224](https://github.com/ClickHouse/ClickHouse/issues/54224). [#60949](https://github.com/ClickHouse/ClickHouse/pull/60949) ([Nikolay Monkov](https://github.com/nikmonkov)). +* Add a setting `parallel_replicas_allow_in_with_subquery = 1` which allows subqueries for IN work with parallel replicas. [#60950](https://github.com/ClickHouse/ClickHouse/pull/60950) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* DNSResolver shuffles set of resolved IPs. [#60965](https://github.com/ClickHouse/ClickHouse/pull/60965) ([Sema Checherinda](https://github.com/CheSema)). +* Support detect output format by file exctension in `clickhouse-client` and `clickhouse-local`. [#61036](https://github.com/ClickHouse/ClickHouse/pull/61036) ([豪肥肥](https://github.com/HowePa)). +* Check memory limit update periodically. [#61049](https://github.com/ClickHouse/ClickHouse/pull/61049) ([Han Fei](https://github.com/hanfei1991)). +* Enable processors profiling (time spent/in and out bytes for sorting, aggregation, ...) by default. [#61096](https://github.com/ClickHouse/ClickHouse/pull/61096) ([Azat Khuzhin](https://github.com/azat)). +* Add the function `toUInt128OrZero`, which was missed by mistake (the mistake is related to https://github.com/ClickHouse/ClickHouse/pull/945). The compatibility aliases `FROM_UNIXTIME` and `DATE_FORMAT` (they are not ClickHouse-native and only exist for MySQL compatibility) have been made case insensitive, as expected for SQL-compatibility aliases. [#61114](https://github.com/ClickHouse/ClickHouse/pull/61114) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improvements for the access checks, allowing to revoke of unpossessed rights in case the target user doesn't have the revoking grants either. Example: ```sql GRANT SELECT ON *.* TO user1; REVOKE SELECT ON system.* FROM user1;. [#61115](https://github.com/ClickHouse/ClickHouse/pull/61115) ([pufit](https://github.com/pufit)). +* Fix an error in previeous opt: https://github.com/ClickHouse/ClickHouse/pull/59698: remove break to make sure the first filtered column has minimum size cc @jsc0218. [#61145](https://github.com/ClickHouse/ClickHouse/pull/61145) ([李扬](https://github.com/taiyang-li)). +* Fix `has()` function with `Nullable` column (fixes [#60214](https://github.com/ClickHouse/ClickHouse/issues/60214)). [#61249](https://github.com/ClickHouse/ClickHouse/pull/61249) ([Mikhail Koviazin](https://github.com/mkmkme)). +* Now it's possible to specify attribute `merge="true"` in config substitutions for subtrees ``. In case this attribute specified, clickhouse will merge subtree with existing configuration, otherwise default behavior is append new content to configuration. [#61299](https://github.com/ClickHouse/ClickHouse/pull/61299) ([alesapin](https://github.com/alesapin)). +* Add async metrics for virtual memory mappings: VMMaxMapCount & VMNumMaps. Closes [#60662](https://github.com/ClickHouse/ClickHouse/issues/60662). [#61354](https://github.com/ClickHouse/ClickHouse/pull/61354) ([Tuan Pham Anh](https://github.com/tuanpavn)). +* Use `temporary_files_codec` setting in all places where we create temporary data, for example external memory sorting and external memory GROUP BY. Before it worked only in `partial_merge` JOIN algorithm. [#61456](https://github.com/ClickHouse/ClickHouse/pull/61456) ([Maksim Kita](https://github.com/kitaisreal)). +* Remove duplicated check `containing_part.empty()`, It's already being checked here: https://github.com/ClickHouse/ClickHouse/blob/1296dac3c7e47670872c15e3f5e58f869e0bd2f2/src/Storages/MergeTree/MergeTreeData.cpp#L6141. [#61467](https://github.com/ClickHouse/ClickHouse/pull/61467) ([William Schoeffel](https://github.com/wiledusc)). +* Add a new setting `max_parser_backtracks` which allows to limit the complexity of query parsing. [#61502](https://github.com/ClickHouse/ClickHouse/pull/61502) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support parallel reading for azure blob storage. [#61503](https://github.com/ClickHouse/ClickHouse/pull/61503) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Less contention during dynamic resize of filesystem cache. [#61524](https://github.com/ClickHouse/ClickHouse/pull/61524) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Disallow sharded mode of StorageS3 queue, because it will be rewritten. [#61537](https://github.com/ClickHouse/ClickHouse/pull/61537) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fixed typo: from `use_leagcy_max_level` to `use_legacy_max_level`. [#61545](https://github.com/ClickHouse/ClickHouse/pull/61545) ([William Schoeffel](https://github.com/wiledusc)). +* Remove some duplicate entries in blob_storage_log. [#61622](https://github.com/ClickHouse/ClickHouse/pull/61622) ([YenchangChan](https://github.com/YenchangChan)). +* Enable `allow_experimental_analyzer` setting by default. [#61652](https://github.com/ClickHouse/ClickHouse/pull/61652) ([Dmitry Novik](https://github.com/novikd)). +* Added `current_user` function as a compatibility alias for MySQL. [#61770](https://github.com/ClickHouse/ClickHouse/pull/61770) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* Use managed identity for backups IO when using Azure Blob Storage. Add a setting to prevent ClickHouse from attempting to create a non-existent container, which requires permissions at the storage account level. [#61785](https://github.com/ClickHouse/ClickHouse/pull/61785) ([Daniel Pozo Escalona](https://github.com/danipozo)). +* Enable `output_format_pretty_row_numbers` by default. It is better for usability. [#61791](https://github.com/ClickHouse/ClickHouse/pull/61791) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* In the previous version, some numbers in Pretty formats were not pretty enough. [#61794](https://github.com/ClickHouse/ClickHouse/pull/61794) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* A long value in Pretty formats won't be cut if it is the single value in the resultset, such as in the result of the `SHOW CREATE TABLE` query. [#61795](https://github.com/ClickHouse/ClickHouse/pull/61795) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Similarly to `clickhouse-local`, `clickhouse-client` will accept the `--output-format` option as a synonym to the `--format` option. This closes [#59848](https://github.com/ClickHouse/ClickHouse/issues/59848). [#61797](https://github.com/ClickHouse/ClickHouse/pull/61797) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* If stdout is a terminal and the output format is not specified, `clickhouse-client` and similar tools will use `PrettyCompact` by default, similarly to the interactive mode. `clickhouse-client` and `clickhouse-local` will handle command line arguments for input and output formats in a unified fashion. This closes [#61272](https://github.com/ClickHouse/ClickHouse/issues/61272). [#61800](https://github.com/ClickHouse/ClickHouse/pull/61800) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Underscore digit groups in Pretty formats for better readability. This is controlled by a new setting, `output_format_pretty_highlight_digit_groups`. [#61802](https://github.com/ClickHouse/ClickHouse/pull/61802) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add ability to override initial INSERT SETTINGS via SYSTEM FLUSH DISTRIBUTED. [#61832](https://github.com/ClickHouse/ClickHouse/pull/61832) ([Azat Khuzhin](https://github.com/azat)). +* Fixed grammar from "a" to "the" in the warning message. There is only one Atomic engine, so it should be "to the new Atomic engine" instead of "to a new Atomic engine". [#61952](https://github.com/ClickHouse/ClickHouse/pull/61952) ([shabroo](https://github.com/shabroo)). + +#### Build/Testing/Packaging Improvement +* Update sccache to the latest version; significantly reduce images size by reshaking the dependency trees; use the latest working odbc driver. [#59953](https://github.com/ClickHouse/ClickHouse/pull/59953) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update python related style checkers. Continue the [#50174](https://github.com/ClickHouse/ClickHouse/issues/50174). [#60408](https://github.com/ClickHouse/ClickHouse/pull/60408) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Upgrade `prqlc` to 0.11.3. [#60616](https://github.com/ClickHouse/ClickHouse/pull/60616) ([Maximilian Roos](https://github.com/max-sixty)). +* Attach gdb to running fuzzer process. [#60654](https://github.com/ClickHouse/ClickHouse/pull/60654) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Use explicit template instantiation more aggressively. Get rid of templates in favor of overloaded functions in some places. [#60730](https://github.com/ClickHouse/ClickHouse/pull/60730) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* The real-time query profiler now works on AArch64. In previous versions, it worked only when a program didn't spend time inside a syscall. [#60807](https://github.com/ClickHouse/ClickHouse/pull/60807) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* ... Too big translation unit in `Aggregator`. [#61211](https://github.com/ClickHouse/ClickHouse/pull/61211) ([lgbo](https://github.com/lgbo-ustc)). +* Fixed flakiness of 01603_insert_select_too_many_parts test. Closes [#61158](https://github.com/ClickHouse/ClickHouse/issues/61158). [#61259](https://github.com/ClickHouse/ClickHouse/pull/61259) ([Ilya Yatsishin](https://github.com/qoega)). +* Now it possible to use `chassert(expression, comment)` in the codebase. [#61263](https://github.com/ClickHouse/ClickHouse/pull/61263) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Teach the fuzzer to use other numeric types. [#61317](https://github.com/ClickHouse/ClickHouse/pull/61317) ([Raúl Marín](https://github.com/Algunenano)). +* Increase memory limit for coverage builds. [#61405](https://github.com/ClickHouse/ClickHouse/pull/61405) ([Raúl Marín](https://github.com/Algunenano)). +* Add generic query text fuzzer in `clickhouse-local`. [#61508](https://github.com/ClickHouse/ClickHouse/pull/61508) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Fix function execution over const and LowCardinality with GROUP BY const for analyzer [#59986](https://github.com/ClickHouse/ClickHouse/pull/59986) ([Azat Khuzhin](https://github.com/azat)). +* Fix finished_mutations_to_keep=0 for MergeTree (as docs says 0 is to keep everything) [#60031](https://github.com/ClickHouse/ClickHouse/pull/60031) ([Azat Khuzhin](https://github.com/azat)). +* PartsSplitter invalid ranges for the same part [#60041](https://github.com/ClickHouse/ClickHouse/pull/60041) ([Maksim Kita](https://github.com/kitaisreal)). +* Azure Blob Storage : Fix issues endpoint and prefix [#60251](https://github.com/ClickHouse/ClickHouse/pull/60251) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* fix LRUResource Cache bug (Hive cache) [#60262](https://github.com/ClickHouse/ClickHouse/pull/60262) ([shanfengp](https://github.com/Aed-p)). +* Force reanalysis if parallel replicas changed [#60362](https://github.com/ClickHouse/ClickHouse/pull/60362) ([Raúl Marín](https://github.com/Algunenano)). +* Fix usage of plain metadata type with new disks configuration option [#60396](https://github.com/ClickHouse/ClickHouse/pull/60396) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Try to fix logical error 'Cannot capture column because it has incompatible type' in mapContainsKeyLike [#60451](https://github.com/ClickHouse/ClickHouse/pull/60451) ([Kruglov Pavel](https://github.com/Avogar)). +* Try to avoid calculation of scalar subqueries for CREATE TABLE. [#60464](https://github.com/ClickHouse/ClickHouse/pull/60464) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix deadlock in parallel parsing when lots of rows are skipped due to errors [#60516](https://github.com/ClickHouse/ClickHouse/pull/60516) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix_max_query_size_for_kql_compound_operator: [#60534](https://github.com/ClickHouse/ClickHouse/pull/60534) ([Yong Wang](https://github.com/kashwy)). +* Keeper fix: add timeouts when waiting for commit logs [#60544](https://github.com/ClickHouse/ClickHouse/pull/60544) ([Antonio Andelic](https://github.com/antonio2368)). +* Reduce the number of read rows from `system.numbers` [#60546](https://github.com/ClickHouse/ClickHouse/pull/60546) ([JackyWoo](https://github.com/JackyWoo)). +* Don't output number tips for date types [#60577](https://github.com/ClickHouse/ClickHouse/pull/60577) ([Raúl Marín](https://github.com/Algunenano)). +* Fix reading from MergeTree with non-deterministic functions in filter [#60586](https://github.com/ClickHouse/ClickHouse/pull/60586) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix logical error on bad compatibility setting value type [#60596](https://github.com/ClickHouse/ClickHouse/pull/60596) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix inconsistent aggregate function states in mixed x86-64 / ARM clusters [#60610](https://github.com/ClickHouse/ClickHouse/pull/60610) ([Harry Lee](https://github.com/HarryLeeIBM)). +* fix(prql): Robust panic handler [#60615](https://github.com/ClickHouse/ClickHouse/pull/60615) ([Maximilian Roos](https://github.com/max-sixty)). +* Fix `intDiv` for decimal and date arguments [#60672](https://github.com/ClickHouse/ClickHouse/pull/60672) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* Fix: expand CTE in alter modify query [#60682](https://github.com/ClickHouse/ClickHouse/pull/60682) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Fix system.parts for non-Atomic/Ordinary database engine (i.e. Memory) [#60689](https://github.com/ClickHouse/ClickHouse/pull/60689) ([Azat Khuzhin](https://github.com/azat)). +* Fix "Invalid storage definition in metadata file" for parameterized views [#60708](https://github.com/ClickHouse/ClickHouse/pull/60708) ([Azat Khuzhin](https://github.com/azat)). +* Fix buffer overflow in CompressionCodecMultiple [#60731](https://github.com/ClickHouse/ClickHouse/pull/60731) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove nonsense from SQL/JSON [#60738](https://github.com/ClickHouse/ClickHouse/pull/60738) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove wrong sanitize checking in aggregate function quantileGK [#60740](https://github.com/ClickHouse/ClickHouse/pull/60740) ([李扬](https://github.com/taiyang-li)). +* Fix insert-select + insert_deduplication_token bug by setting streams to 1 [#60745](https://github.com/ClickHouse/ClickHouse/pull/60745) ([Jordi Villar](https://github.com/jrdi)). +* Prevent setting custom metadata headers on unsupported multipart upload operations [#60748](https://github.com/ClickHouse/ClickHouse/pull/60748) ([Francisco J. Jurado Moreno](https://github.com/Beetelbrox)). +* Fix toStartOfInterval [#60763](https://github.com/ClickHouse/ClickHouse/pull/60763) ([Andrey Zvonov](https://github.com/zvonand)). +* Fix crash in arrayEnumerateRanked [#60764](https://github.com/ClickHouse/ClickHouse/pull/60764) ([Raúl Marín](https://github.com/Algunenano)). +* Fix crash when using input() in INSERT SELECT JOIN [#60765](https://github.com/ClickHouse/ClickHouse/pull/60765) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix crash with different allow_experimental_analyzer value in subqueries [#60770](https://github.com/ClickHouse/ClickHouse/pull/60770) ([Dmitry Novik](https://github.com/novikd)). +* Remove recursion when reading from S3 [#60849](https://github.com/ClickHouse/ClickHouse/pull/60849) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix possible stuck on error in HashedDictionaryParallelLoader [#60926](https://github.com/ClickHouse/ClickHouse/pull/60926) ([vdimir](https://github.com/vdimir)). +* Fix async RESTORE with Replicated database [#60934](https://github.com/ClickHouse/ClickHouse/pull/60934) ([Antonio Andelic](https://github.com/antonio2368)). +* fix csv format not support tuple [#60994](https://github.com/ClickHouse/ClickHouse/pull/60994) ([shuai.xu](https://github.com/shuai-xu)). +* Fix deadlock in async inserts to `Log` tables via native protocol [#61055](https://github.com/ClickHouse/ClickHouse/pull/61055) ([Anton Popov](https://github.com/CurtizJ)). +* Fix lazy execution of default argument in dictGetOrDefault for RangeHashedDictionary [#61196](https://github.com/ClickHouse/ClickHouse/pull/61196) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix multiple bugs in groupArraySorted [#61203](https://github.com/ClickHouse/ClickHouse/pull/61203) ([Raúl Marín](https://github.com/Algunenano)). +* Fix Keeper reconfig for standalone binary [#61233](https://github.com/ClickHouse/ClickHouse/pull/61233) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix usage of session_token in S3 engine [#61234](https://github.com/ClickHouse/ClickHouse/pull/61234) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix possible incorrect result of aggregate function `uniqExact` [#61257](https://github.com/ClickHouse/ClickHouse/pull/61257) ([Anton Popov](https://github.com/CurtizJ)). +* Fix bugs in show database [#61269](https://github.com/ClickHouse/ClickHouse/pull/61269) ([Raúl Marín](https://github.com/Algunenano)). +* Fix logical error in RabbitMQ storage with MATERIALIZED columns [#61320](https://github.com/ClickHouse/ClickHouse/pull/61320) ([vdimir](https://github.com/vdimir)). +* Fix CREATE OR REPLACE DICTIONARY [#61356](https://github.com/ClickHouse/ClickHouse/pull/61356) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix crash in ObjectJson parsing array with nulls [#61364](https://github.com/ClickHouse/ClickHouse/pull/61364) ([vdimir](https://github.com/vdimir)). +* Fix ATTACH query with external ON CLUSTER [#61365](https://github.com/ClickHouse/ClickHouse/pull/61365) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix consecutive keys optimization for nullable keys [#61393](https://github.com/ClickHouse/ClickHouse/pull/61393) ([Anton Popov](https://github.com/CurtizJ)). +* fix issue of actions dag split [#61458](https://github.com/ClickHouse/ClickHouse/pull/61458) ([Raúl Marín](https://github.com/Algunenano)). +* Fix finishing a failed RESTORE [#61466](https://github.com/ClickHouse/ClickHouse/pull/61466) ([Vitaly Baranov](https://github.com/vitlibar)). +* Disable async_insert_use_adaptive_busy_timeout correctly with compatibility settings [#61468](https://github.com/ClickHouse/ClickHouse/pull/61468) ([Raúl Marín](https://github.com/Algunenano)). +* Allow queuing in restore pool [#61475](https://github.com/ClickHouse/ClickHouse/pull/61475) ([Nikita Taranov](https://github.com/nickitat)). +* Fix bug when reading system.parts using UUID (issue 61220). [#61479](https://github.com/ClickHouse/ClickHouse/pull/61479) ([Dan Wu](https://github.com/wudanzy)). +* Fix ALTER QUERY MODIFY SQL SECURITY [#61480](https://github.com/ClickHouse/ClickHouse/pull/61480) ([pufit](https://github.com/pufit)). +* Fix crash in window view [#61526](https://github.com/ClickHouse/ClickHouse/pull/61526) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix `repeat` with non native integers [#61527](https://github.com/ClickHouse/ClickHouse/pull/61527) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix client `-s` argument [#61530](https://github.com/ClickHouse/ClickHouse/pull/61530) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Reset part level upon attach from disk on MergeTree [#61536](https://github.com/ClickHouse/ClickHouse/pull/61536) ([Arthur Passos](https://github.com/arthurpassos)). +* Fix crash in arrayPartialReverseSort [#61539](https://github.com/ClickHouse/ClickHouse/pull/61539) ([Raúl Marín](https://github.com/Algunenano)). +* Fix string search with const position [#61547](https://github.com/ClickHouse/ClickHouse/pull/61547) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix addDays cause an error when used datetime64 [#61561](https://github.com/ClickHouse/ClickHouse/pull/61561) ([Shuai li](https://github.com/loneylee)). +* disallow LowCardinality input type for JSONExtract [#61617](https://github.com/ClickHouse/ClickHouse/pull/61617) ([Julia Kartseva](https://github.com/jkartseva)). +* Fix `system.part_log` for async insert with deduplication [#61620](https://github.com/ClickHouse/ClickHouse/pull/61620) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix Non-ready set for system.parts. [#61666](https://github.com/ClickHouse/ClickHouse/pull/61666) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Don't allow the same expression in ORDER BY with and without WITH FILL [#61667](https://github.com/ClickHouse/ClickHouse/pull/61667) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix actual_part_name for REPLACE_RANGE (`Entry actual part isn't empty yet`) [#61675](https://github.com/ClickHouse/ClickHouse/pull/61675) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix columns after executing MODIFY QUERY for a materialized view with internal table [#61734](https://github.com/ClickHouse/ClickHouse/pull/61734) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix crash in `multiSearchAllPositionsCaseInsensitiveUTF8` for incorrect UTF-8 [#61749](https://github.com/ClickHouse/ClickHouse/pull/61749) ([pufit](https://github.com/pufit)). +* Fix RANGE frame is not supported for Nullable columns. [#61766](https://github.com/ClickHouse/ClickHouse/pull/61766) ([YuanLiu](https://github.com/ditgittube)). +* Revert "Revert "Fix bug when reading system.parts using UUID (issue 61220)."" [#61779](https://github.com/ClickHouse/ClickHouse/pull/61779) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). + +#### CI Fix or Improvement (changelog entry is not required) + +* Decoupled changes from [#60408](https://github.com/ClickHouse/ClickHouse/issues/60408). [#60553](https://github.com/ClickHouse/ClickHouse/pull/60553) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Eliminates the need to provide input args to docker server jobs to clean yml files. [#60602](https://github.com/ClickHouse/ClickHouse/pull/60602) ([Max K.](https://github.com/maxknv)). +* Debug and fix markreleaseready. [#60611](https://github.com/ClickHouse/ClickHouse/pull/60611) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix build_report job so that it's defined by ci_config only (not yml file). [#60613](https://github.com/ClickHouse/ClickHouse/pull/60613) ([Max K.](https://github.com/maxknv)). +* Do not await ci pending jobs on release branches decrease wait timeout to fit into gh job timeout. [#60652](https://github.com/ClickHouse/ClickHouse/pull/60652) ([Max K.](https://github.com/maxknv)). +* Set limited number of builds for "special build check" report in backports. [#60850](https://github.com/ClickHouse/ClickHouse/pull/60850) ([Max K.](https://github.com/maxknv)). +* ... [#60935](https://github.com/ClickHouse/ClickHouse/pull/60935) ([Max K.](https://github.com/maxknv)). +* ... [#60947](https://github.com/ClickHouse/ClickHouse/pull/60947) ([Max K.](https://github.com/maxknv)). +* ... [#60952](https://github.com/ClickHouse/ClickHouse/pull/60952) ([Max K.](https://github.com/maxknv)). +* ... [#60958](https://github.com/ClickHouse/ClickHouse/pull/60958) ([Max K.](https://github.com/maxknv)). +* ... [#61022](https://github.com/ClickHouse/ClickHouse/pull/61022) ([Max K.](https://github.com/maxknv)). +* Just a preparation for the merge queue support. [#61099](https://github.com/ClickHouse/ClickHouse/pull/61099) ([Max K.](https://github.com/maxknv)). +* ... [#61133](https://github.com/ClickHouse/ClickHouse/pull/61133) ([Max K.](https://github.com/maxknv)). +* In PRs: - run typos, aspell check - always - run pylint, mypy - only if py file(s) changed in PRs - run basic source files style check - only if not all changes in py files. [#61148](https://github.com/ClickHouse/ClickHouse/pull/61148) ([Max K.](https://github.com/maxknv)). +* ... [#61172](https://github.com/ClickHouse/ClickHouse/pull/61172) ([Max K.](https://github.com/maxknv)). +* ... [#61183](https://github.com/ClickHouse/ClickHouse/pull/61183) ([Han Fei](https://github.com/hanfei1991)). +* ... [#61185](https://github.com/ClickHouse/ClickHouse/pull/61185) ([Max K.](https://github.com/maxknv)). +* TBD. [#61197](https://github.com/ClickHouse/ClickHouse/pull/61197) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* ... [#61214](https://github.com/ClickHouse/ClickHouse/pull/61214) ([Max K.](https://github.com/maxknv)). +* ... [#61441](https://github.com/ClickHouse/ClickHouse/pull/61441) ([Max K.](https://github.com/maxknv)). +* ![Screenshot_20240323_025055](https://github.com/ClickHouse/ClickHouse/assets/18581488/ccaab212-a1d3-4dfb-8d56-b1991760b6bf). [#61801](https://github.com/ClickHouse/ClickHouse/pull/61801) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* ... [#61877](https://github.com/ClickHouse/ClickHouse/pull/61877) ([Max K.](https://github.com/maxknv)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Revert "Revert "Use `MergeTree` as a default table engine""'. [#60524](https://github.com/ClickHouse/ClickHouse/pull/60524) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Revert "Revert "Support resource request canceling""'. [#60558](https://github.com/ClickHouse/ClickHouse/pull/60558) ([Sergei Trifonov](https://github.com/serxa)). +* NO CL ENTRY: 'Revert "Add `toMillisecond` function"'. [#60644](https://github.com/ClickHouse/ClickHouse/pull/60644) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Revert "Synchronize parsers"'. [#60759](https://github.com/ClickHouse/ClickHouse/pull/60759) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Revert "Fix wacky primary key sorting in `SHOW INDEX`"'. [#60898](https://github.com/ClickHouse/ClickHouse/pull/60898) ([Antonio Andelic](https://github.com/antonio2368)). +* NO CL ENTRY: 'Revert "CI: make style check faster"'. [#61142](https://github.com/ClickHouse/ClickHouse/pull/61142) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Revert "Don't allow to set max_parallel_replicas to 0 as it doesn't make sense"'. [#61200](https://github.com/ClickHouse/ClickHouse/pull/61200) ([Kruglov Pavel](https://github.com/Avogar)). +* NO CL ENTRY: 'Revert "Fix usage of session_token in S3 engine"'. [#61359](https://github.com/ClickHouse/ClickHouse/pull/61359) ([Antonio Andelic](https://github.com/antonio2368)). +* NO CL ENTRY: 'Revert "Revert "Fix usage of session_token in S3 engine""'. [#61362](https://github.com/ClickHouse/ClickHouse/pull/61362) ([Kruglov Pavel](https://github.com/Avogar)). +* NO CL ENTRY: 'Reorder hidden and shown checks in comment, change url of Mergeable check'. [#61373](https://github.com/ClickHouse/ClickHouse/pull/61373) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* NO CL ENTRY: 'Remove unnecessary layers from clickhouse/cctools'. [#61374](https://github.com/ClickHouse/ClickHouse/pull/61374) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* NO CL ENTRY: 'Revert "Updated format settings references in the docs (datetime.md)"'. [#61435](https://github.com/ClickHouse/ClickHouse/pull/61435) ([Kruglov Pavel](https://github.com/Avogar)). +* NO CL ENTRY: 'Revert "CI: ARM integration tests: disable tests with HDFS "'. [#61449](https://github.com/ClickHouse/ClickHouse/pull/61449) ([Max K.](https://github.com/maxknv)). +* NO CL ENTRY: 'Revert "Analyzer: Fix virtual columns in StorageMerge"'. [#61518](https://github.com/ClickHouse/ClickHouse/pull/61518) ([Antonio Andelic](https://github.com/antonio2368)). +* NO CL ENTRY: 'Revert "Revert "Analyzer: Fix virtual columns in StorageMerge""'. [#61528](https://github.com/ClickHouse/ClickHouse/pull/61528) ([Dmitry Novik](https://github.com/novikd)). +* NO CL ENTRY: 'Improve build_download_helper'. [#61592](https://github.com/ClickHouse/ClickHouse/pull/61592) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* NO CL ENTRY: 'Revert "Un-flake `test_undrop_query`"'. [#61668](https://github.com/ClickHouse/ClickHouse/pull/61668) ([Robert Schulze](https://github.com/rschu1ze)). +* NO CL ENTRY: 'Fix flaky tests (stateless, integration)'. [#61816](https://github.com/ClickHouse/ClickHouse/pull/61816) ([Nikita Fomichev](https://github.com/fm4v)). +* NO CL ENTRY: 'Better usability of "expect" tests: less trouble with running directly'. [#61818](https://github.com/ClickHouse/ClickHouse/pull/61818) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Revert "Fix flaky `02122_parallel_formatting_Template`"'. [#61868](https://github.com/ClickHouse/ClickHouse/pull/61868) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Revert "Add --now option to enable and start the service" #job_Install_packages_amd64'. [#61878](https://github.com/ClickHouse/ClickHouse/pull/61878) ([Max K.](https://github.com/maxknv)). +* NO CL ENTRY: 'Revert "disallow LowCardinality input type for JSONExtract"'. [#61960](https://github.com/ClickHouse/ClickHouse/pull/61960) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Improve query performance in case of very small blocks [#58879](https://github.com/ClickHouse/ClickHouse/pull/58879) ([Azat Khuzhin](https://github.com/azat)). +* Analyzer: fixes for JOIN columns resolution [#59007](https://github.com/ClickHouse/ClickHouse/pull/59007) ([vdimir](https://github.com/vdimir)). +* Fix race on `Context::async_insert_queue` [#59082](https://github.com/ClickHouse/ClickHouse/pull/59082) ([Alexander Tokmakov](https://github.com/tavplubix)). +* CI: support batch specification in commit message [#59738](https://github.com/ClickHouse/ClickHouse/pull/59738) ([Max K.](https://github.com/maxknv)). +* Update storing-data.md [#60024](https://github.com/ClickHouse/ClickHouse/pull/60024) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Make max_insert_delayed_streams_for_parallel_write actually work [#60079](https://github.com/ClickHouse/ClickHouse/pull/60079) ([alesapin](https://github.com/alesapin)). +* Analyzer: support join using column from select list [#60182](https://github.com/ClickHouse/ClickHouse/pull/60182) ([vdimir](https://github.com/vdimir)). +* test for [#60223](https://github.com/ClickHouse/ClickHouse/issues/60223) [#60258](https://github.com/ClickHouse/ClickHouse/pull/60258) ([Denny Crane](https://github.com/den-crane)). +* Analyzer: Refactor execution name for ConstantNode [#60313](https://github.com/ClickHouse/ClickHouse/pull/60313) ([Dmitry Novik](https://github.com/novikd)). +* Fix database iterator waiting code [#60314](https://github.com/ClickHouse/ClickHouse/pull/60314) ([Sergei Trifonov](https://github.com/serxa)). +* QueryCache: Don't acquire the query count mutex if not necessary [#60348](https://github.com/ClickHouse/ClickHouse/pull/60348) ([zhongyuankai](https://github.com/zhongyuankai)). +* Fix bugfix check (due to unknown commit_logs_cache_size_threshold) [#60375](https://github.com/ClickHouse/ClickHouse/pull/60375) ([Azat Khuzhin](https://github.com/azat)). +* Enable testing with `io_uring` back [#60383](https://github.com/ClickHouse/ClickHouse/pull/60383) ([Nikita Taranov](https://github.com/nickitat)). +* Analyzer - improve hiding secret arguments. [#60386](https://github.com/ClickHouse/ClickHouse/pull/60386) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* CI: make workflow yml abstract [#60421](https://github.com/ClickHouse/ClickHouse/pull/60421) ([Max K.](https://github.com/maxknv)). +* Improve test test_reload_clusters_config [#60426](https://github.com/ClickHouse/ClickHouse/pull/60426) ([Kruglov Pavel](https://github.com/Avogar)). +* Revert "Revert "Merge pull request [#56864](https://github.com/ClickHouse/ClickHouse/issues/56864) from ClickHouse/broken-projections-better-handling"" [#60452](https://github.com/ClickHouse/ClickHouse/pull/60452) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Do not check to and from files existence in metadata_storage because it does not see uncommitted changes [#60462](https://github.com/ClickHouse/ClickHouse/pull/60462) ([Alexander Gololobov](https://github.com/davenger)). +* Fix option ambiguous in `clickhouse-local` [#60475](https://github.com/ClickHouse/ClickHouse/pull/60475) ([豪肥肥](https://github.com/HowePa)). +* Fix: test_parallel_replicas_custom_key_load_balancing [#60485](https://github.com/ClickHouse/ClickHouse/pull/60485) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix: progress bar for *Cluster table functions [#60491](https://github.com/ClickHouse/ClickHouse/pull/60491) ([Igor Nikonov](https://github.com/devcrafter)). +* Analyzer: Support different ObjectJSON on shards [#60497](https://github.com/ClickHouse/ClickHouse/pull/60497) ([Dmitry Novik](https://github.com/novikd)). +* Cancel PipelineExecutor properly in case of exception in spawnThreads [#60499](https://github.com/ClickHouse/ClickHouse/pull/60499) ([Kruglov Pavel](https://github.com/Avogar)). +* Refactor StorageSystemOneBlock [#60510](https://github.com/ClickHouse/ClickHouse/pull/60510) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Simple cleanup while fixing progress bar [#60513](https://github.com/ClickHouse/ClickHouse/pull/60513) ([Igor Nikonov](https://github.com/devcrafter)). +* PullingAsyncPipelineExecutor cleanup [#60515](https://github.com/ClickHouse/ClickHouse/pull/60515) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix bad error message [#60518](https://github.com/ClickHouse/ClickHouse/pull/60518) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Synchronize Access [#60519](https://github.com/ClickHouse/ClickHouse/pull/60519) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Synchronize metrics and Keeper [#60520](https://github.com/ClickHouse/ClickHouse/pull/60520) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Enforce clang-tidy in `programs/` and `utils/` headers [#60521](https://github.com/ClickHouse/ClickHouse/pull/60521) ([Robert Schulze](https://github.com/rschu1ze)). +* Synchronize parsers [#60522](https://github.com/ClickHouse/ClickHouse/pull/60522) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix a bunch of clang-tidy warnings in headers [#60523](https://github.com/ClickHouse/ClickHouse/pull/60523) ([Robert Schulze](https://github.com/rschu1ze)). +* General sanity in function `seriesOutliersDetectTukey` [#60535](https://github.com/ClickHouse/ClickHouse/pull/60535) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update Chinese document for max_query_size, max_parser_depth and optimize_functions_to_subcolumns [#60541](https://github.com/ClickHouse/ClickHouse/pull/60541) ([Alex Cheng](https://github.com/Alex-Cheng)). +* Userspace page cache again [#60552](https://github.com/ClickHouse/ClickHouse/pull/60552) ([Michael Kolupaev](https://github.com/al13n321)). +* Traverse shadow directory for system.remote_data_paths [#60585](https://github.com/ClickHouse/ClickHouse/pull/60585) ([Aleksei Filatov](https://github.com/aalexfvk)). +* Add test for [#58906](https://github.com/ClickHouse/ClickHouse/issues/58906) [#60597](https://github.com/ClickHouse/ClickHouse/pull/60597) ([Raúl Marín](https://github.com/Algunenano)). +* Use python zipfile to have x-platform idempotent lambda packages [#60603](https://github.com/ClickHouse/ClickHouse/pull/60603) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* tests: suppress data-race in librdkafka statistics code [#60604](https://github.com/ClickHouse/ClickHouse/pull/60604) ([Azat Khuzhin](https://github.com/azat)). +* Update version after release [#60605](https://github.com/ClickHouse/ClickHouse/pull/60605) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update version_date.tsv and changelogs after v24.2.1.2248-stable [#60607](https://github.com/ClickHouse/ClickHouse/pull/60607) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Addition to changelog [#60609](https://github.com/ClickHouse/ClickHouse/pull/60609) ([Anton Popov](https://github.com/CurtizJ)). +* internal: Refine rust prql code [#60617](https://github.com/ClickHouse/ClickHouse/pull/60617) ([Maximilian Roos](https://github.com/max-sixty)). +* fix(rust): Fix skim's panic handler [#60621](https://github.com/ClickHouse/ClickHouse/pull/60621) ([Maximilian Roos](https://github.com/max-sixty)). +* Resubmit "Analyzer: compute ALIAS columns right after reading" [#60641](https://github.com/ClickHouse/ClickHouse/pull/60641) ([vdimir](https://github.com/vdimir)). +* Analyzer: Fix bug with join_use_nulls and PREWHERE [#60655](https://github.com/ClickHouse/ClickHouse/pull/60655) ([vdimir](https://github.com/vdimir)). +* Add test for [#59891](https://github.com/ClickHouse/ClickHouse/issues/59891) [#60657](https://github.com/ClickHouse/ClickHouse/pull/60657) ([Raúl Marín](https://github.com/Algunenano)). +* Fix missed entries in system.part_log in case of fetch preferred over merges/mutations [#60659](https://github.com/ClickHouse/ClickHouse/pull/60659) ([Azat Khuzhin](https://github.com/azat)). +* Always apply first minmax index among available skip indices [#60675](https://github.com/ClickHouse/ClickHouse/pull/60675) ([Igor Nikonov](https://github.com/devcrafter)). +* Remove bad test `02152_http_external_tables_memory_tracking` [#60690](https://github.com/ClickHouse/ClickHouse/pull/60690) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix questionable behavior in the `parseDateTimeBestEffort` function. [#60691](https://github.com/ClickHouse/ClickHouse/pull/60691) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix flaky checks [#60694](https://github.com/ClickHouse/ClickHouse/pull/60694) ([Azat Khuzhin](https://github.com/azat)). +* Resubmit http_external_tables_memory_tracking test [#60695](https://github.com/ClickHouse/ClickHouse/pull/60695) ([Azat Khuzhin](https://github.com/azat)). +* Fix bugfix and upgrade checks (due to "Unknown handler type 'redirect'" error) [#60696](https://github.com/ClickHouse/ClickHouse/pull/60696) ([Azat Khuzhin](https://github.com/azat)). +* Fix test_grant_and_revoke/test.py::test_grant_all_on_table (after syncing with cloud) [#60699](https://github.com/ClickHouse/ClickHouse/pull/60699) ([Azat Khuzhin](https://github.com/azat)). +* Remove unit test for ColumnObject [#60709](https://github.com/ClickHouse/ClickHouse/pull/60709) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve unit tests [#60710](https://github.com/ClickHouse/ClickHouse/pull/60710) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix scheduler fairness test [#60712](https://github.com/ClickHouse/ClickHouse/pull/60712) ([Sergei Trifonov](https://github.com/serxa)). +* Do not retry queries if container is down in integration tests (resubmit) [#60714](https://github.com/ClickHouse/ClickHouse/pull/60714) ([Azat Khuzhin](https://github.com/azat)). +* Mark one setting as obsolete [#60715](https://github.com/ClickHouse/ClickHouse/pull/60715) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix a test with Analyzer [#60723](https://github.com/ClickHouse/ClickHouse/pull/60723) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Two tests are fixed with Analyzer [#60724](https://github.com/ClickHouse/ClickHouse/pull/60724) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove old code [#60728](https://github.com/ClickHouse/ClickHouse/pull/60728) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove more code from LIVE VIEW [#60729](https://github.com/ClickHouse/ClickHouse/pull/60729) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix `test_keeper_back_to_back/test.py::test_concurrent_watches` [#60749](https://github.com/ClickHouse/ClickHouse/pull/60749) ([Antonio Andelic](https://github.com/antonio2368)). +* Catch exceptions on finalize in `InterserverIOHTTPHandler` [#60769](https://github.com/ClickHouse/ClickHouse/pull/60769) ([Antonio Andelic](https://github.com/antonio2368)). +* Reduce flakiness of 02932_refreshable_materialized_views [#60771](https://github.com/ClickHouse/ClickHouse/pull/60771) ([Michael Kolupaev](https://github.com/al13n321)). +* Use 64-bit capabilities if available [#60775](https://github.com/ClickHouse/ClickHouse/pull/60775) ([Azat Khuzhin](https://github.com/azat)). +* Include multiline logs in fuzzer fatal.log report [#60796](https://github.com/ClickHouse/ClickHouse/pull/60796) ([Raúl Marín](https://github.com/Algunenano)). +* Add missing clone calls related to compression [#60810](https://github.com/ClickHouse/ClickHouse/pull/60810) ([Raúl Marín](https://github.com/Algunenano)). +* New private runners [#60811](https://github.com/ClickHouse/ClickHouse/pull/60811) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Move userspace page cache settings to the correct section of SettingsChangeHistory.h [#60812](https://github.com/ClickHouse/ClickHouse/pull/60812) ([Michael Kolupaev](https://github.com/al13n321)). +* Update version_date.tsv and changelogs after v23.8.10.43-lts [#60851](https://github.com/ClickHouse/ClickHouse/pull/60851) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Fix fuzzer report [#60853](https://github.com/ClickHouse/ClickHouse/pull/60853) ([Raúl Marín](https://github.com/Algunenano)). +* Update version_date.tsv and changelogs after v23.3.20.27-lts [#60857](https://github.com/ClickHouse/ClickHouse/pull/60857) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Refactor OptimizeDateOrDateTimeConverterWithPreimageVisitor [#60875](https://github.com/ClickHouse/ClickHouse/pull/60875) ([Zhiguo Zhou](https://github.com/ZhiguoZh)). +* Fix race in PageCache [#60878](https://github.com/ClickHouse/ClickHouse/pull/60878) ([Michael Kolupaev](https://github.com/al13n321)). +* Small changes in async inserts code [#60885](https://github.com/ClickHouse/ClickHouse/pull/60885) ([Nikita Taranov](https://github.com/nickitat)). +* Remove useless verbose logging from AWS library [#60921](https://github.com/ClickHouse/ClickHouse/pull/60921) ([alesapin](https://github.com/alesapin)). +* Throw on query timeout in ZooKeeperRetries [#60922](https://github.com/ClickHouse/ClickHouse/pull/60922) ([Antonio Andelic](https://github.com/antonio2368)). +* Bring clickhouse-test changes from private [#60924](https://github.com/ClickHouse/ClickHouse/pull/60924) ([Raúl Marín](https://github.com/Algunenano)). +* Add debug info to exceptions in `IMergeTreeDataPart::checkConsistency()` [#60981](https://github.com/ClickHouse/ClickHouse/pull/60981) ([Nikita Taranov](https://github.com/nickitat)). +* Fix a typo [#60987](https://github.com/ClickHouse/ClickHouse/pull/60987) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Replace some header includes with forward declarations [#61003](https://github.com/ClickHouse/ClickHouse/pull/61003) ([Amos Bird](https://github.com/amosbird)). +* Speed up cctools building [#61011](https://github.com/ClickHouse/ClickHouse/pull/61011) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix ASTRenameQuery::clone [#61013](https://github.com/ClickHouse/ClickHouse/pull/61013) ([vdimir](https://github.com/vdimir)). +* Update README.md [#61021](https://github.com/ClickHouse/ClickHouse/pull/61021) ([Tyler Hannan](https://github.com/tylerhannan)). +* Fix TableFunctionExecutable::skipAnalysisForArguments [#61037](https://github.com/ClickHouse/ClickHouse/pull/61037) ([Dmitry Novik](https://github.com/novikd)). +* Fix: parallel replicas with PREWHERE (ubsan) [#61052](https://github.com/ClickHouse/ClickHouse/pull/61052) ([Igor Nikonov](https://github.com/devcrafter)). +* Fast fix tests. [#61056](https://github.com/ClickHouse/ClickHouse/pull/61056) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix `test_placement_info` [#61057](https://github.com/ClickHouse/ClickHouse/pull/61057) ([Konstantin Bogdanov](https://github.com/thevar1able)). +* Fix: parallel replicas with CTEs, crash in EXPLAIN SYNTAX with analyzer [#61059](https://github.com/ClickHouse/ClickHouse/pull/61059) ([Igor Nikonov](https://github.com/devcrafter)). +* Debug fuzzer failures [#61062](https://github.com/ClickHouse/ClickHouse/pull/61062) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add regression tests for fixed issues [#61076](https://github.com/ClickHouse/ClickHouse/pull/61076) ([Antonio Andelic](https://github.com/antonio2368)). +* Analyzer: Fix 01244_optimize_distributed_group_by_sharding_key [#61089](https://github.com/ClickHouse/ClickHouse/pull/61089) ([Dmitry Novik](https://github.com/novikd)). +* Use global scalars cache with analyzer [#61104](https://github.com/ClickHouse/ClickHouse/pull/61104) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix removing is_active node after re-creation [#61105](https://github.com/ClickHouse/ClickHouse/pull/61105) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Update 02962_system_sync_replica_lightweight_from_modifier.sh [#61110](https://github.com/ClickHouse/ClickHouse/pull/61110) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Simplify bridges [#61118](https://github.com/ClickHouse/ClickHouse/pull/61118) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* update cppkafka to v0.4.1 [#61119](https://github.com/ClickHouse/ClickHouse/pull/61119) ([Ilya Golshtein](https://github.com/ilejn)). +* CI: add wf class in ci_config [#61122](https://github.com/ClickHouse/ClickHouse/pull/61122) ([Max K.](https://github.com/maxknv)). +* QueryFuzzer: replace element randomly when AST part buffer is full [#61124](https://github.com/ClickHouse/ClickHouse/pull/61124) ([Tomer Shafir](https://github.com/tomershafir)). +* CI: make style check fast [#61125](https://github.com/ClickHouse/ClickHouse/pull/61125) ([Max K.](https://github.com/maxknv)). +* Better gitignore [#61128](https://github.com/ClickHouse/ClickHouse/pull/61128) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix something strange [#61129](https://github.com/ClickHouse/ClickHouse/pull/61129) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update check-large-objects.sh to be language neutral [#61130](https://github.com/ClickHouse/ClickHouse/pull/61130) ([Dan Wu](https://github.com/wudanzy)). +* Throw memory limit exceptions to avoid OOM in some places [#61132](https://github.com/ClickHouse/ClickHouse/pull/61132) ([alesapin](https://github.com/alesapin)). +* Fix test_distributed_directory_monitor_split_batch_on_failure flakienss [#61136](https://github.com/ClickHouse/ClickHouse/pull/61136) ([Azat Khuzhin](https://github.com/azat)). +* Fix llvm symbolizer on CI [#61147](https://github.com/ClickHouse/ClickHouse/pull/61147) ([Azat Khuzhin](https://github.com/azat)). +* Some clang-tidy fixes [#61150](https://github.com/ClickHouse/ClickHouse/pull/61150) ([Robert Schulze](https://github.com/rschu1ze)). +* Revive "Less contention in the cache, part 2" [#61152](https://github.com/ClickHouse/ClickHouse/pull/61152) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Enable black back [#61159](https://github.com/ClickHouse/ClickHouse/pull/61159) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* CI: fix nightly job issue [#61160](https://github.com/ClickHouse/ClickHouse/pull/61160) ([Max K.](https://github.com/maxknv)). +* Split `RangeHashedDictionary` [#61162](https://github.com/ClickHouse/ClickHouse/pull/61162) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* Remove a few templates from Aggregator.cpp [#61171](https://github.com/ClickHouse/ClickHouse/pull/61171) ([Raúl Marín](https://github.com/Algunenano)). +* Avoid some logical errors in experimantal Object type [#61173](https://github.com/ClickHouse/ClickHouse/pull/61173) ([Kruglov Pavel](https://github.com/Avogar)). +* Update ReadSettings.h [#61174](https://github.com/ClickHouse/ClickHouse/pull/61174) ([Kseniia Sumarokova](https://github.com/kssenii)). +* CI: ARM integration tests: disable tests with HDFS [#61182](https://github.com/ClickHouse/ClickHouse/pull/61182) ([Max K.](https://github.com/maxknv)). +* Disable sanitizers with 02784_parallel_replicas_automatic_decision_join [#61184](https://github.com/ClickHouse/ClickHouse/pull/61184) ([Raúl Marín](https://github.com/Algunenano)). +* Fix `02887_mutations_subcolumns` test flakiness [#61198](https://github.com/ClickHouse/ClickHouse/pull/61198) ([Nikita Taranov](https://github.com/nickitat)). +* Make variant tests a bit faster [#61199](https://github.com/ClickHouse/ClickHouse/pull/61199) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix strange log message [#61206](https://github.com/ClickHouse/ClickHouse/pull/61206) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix 01603_insert_select_too_many_parts flakiness [#61218](https://github.com/ClickHouse/ClickHouse/pull/61218) ([Azat Khuzhin](https://github.com/azat)). +* Make every style-checker runner types scaling-out very quickly [#61231](https://github.com/ClickHouse/ClickHouse/pull/61231) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Improve `test_failed_mutations` [#61235](https://github.com/ClickHouse/ClickHouse/pull/61235) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* Fix `test_merge_tree_load_parts/test.py::test_merge_tree_load_parts_corrupted` [#61236](https://github.com/ClickHouse/ClickHouse/pull/61236) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* fix `forget_partition` test [#61237](https://github.com/ClickHouse/ClickHouse/pull/61237) ([Sergei Trifonov](https://github.com/serxa)). +* Print more info in `02572_system_logs_materialized_views_ignore_errors` to debug [#61246](https://github.com/ClickHouse/ClickHouse/pull/61246) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* Fix runtime error in AST Fuzzer [#61248](https://github.com/ClickHouse/ClickHouse/pull/61248) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* Add retries to `02908_many_requests_to_system_replicas` [#61253](https://github.com/ClickHouse/ClickHouse/pull/61253) ([Nikita Taranov](https://github.com/nickitat)). +* Followup fix ASTRenameQuery::clone [#61254](https://github.com/ClickHouse/ClickHouse/pull/61254) ([vdimir](https://github.com/vdimir)). +* Disable test 02998_primary_key_skip_columns.sql in sanitizer builds as it can be slow [#61256](https://github.com/ClickHouse/ClickHouse/pull/61256) ([Kruglov Pavel](https://github.com/Avogar)). +* Update curl to curl with data race fix [#61264](https://github.com/ClickHouse/ClickHouse/pull/61264) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Fix `01417_freeze_partition_verbose` [#61266](https://github.com/ClickHouse/ClickHouse/pull/61266) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Free memory earlier in inserts [#61267](https://github.com/ClickHouse/ClickHouse/pull/61267) ([Anton Popov](https://github.com/CurtizJ)). +* Fixing test_build_sets_from_multiple_threads/test.py::test_set [#61286](https://github.com/ClickHouse/ClickHouse/pull/61286) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Analyzer: Fix virtual columns in StorageMerge [#61298](https://github.com/ClickHouse/ClickHouse/pull/61298) ([Dmitry Novik](https://github.com/novikd)). +* Fix 01952_optimize_distributed_group_by_sharding_key with analyzer. [#61301](https://github.com/ClickHouse/ClickHouse/pull/61301) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* fix data race in poco tcp server [#61309](https://github.com/ClickHouse/ClickHouse/pull/61309) ([Sema Checherinda](https://github.com/CheSema)). +* Don't use default cluster in test test_distibuted_settings [#61314](https://github.com/ClickHouse/ClickHouse/pull/61314) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix false positive assertion in cache [#61319](https://github.com/ClickHouse/ClickHouse/pull/61319) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix test test_input_format_parallel_parsing_memory_tracking [#61322](https://github.com/ClickHouse/ClickHouse/pull/61322) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix 01761_cast_to_enum_nullable with analyzer. [#61323](https://github.com/ClickHouse/ClickHouse/pull/61323) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add zookeeper retries for exists check in forcefullyRemoveBrokenOutdatedPartFromZooKeeper [#61324](https://github.com/ClickHouse/ClickHouse/pull/61324) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Minor changes in stress and fuzzer reports [#61333](https://github.com/ClickHouse/ClickHouse/pull/61333) ([Raúl Marín](https://github.com/Algunenano)). +* Un-flake `test_undrop_query` [#61348](https://github.com/ClickHouse/ClickHouse/pull/61348) ([Robert Schulze](https://github.com/rschu1ze)). +* Tiny improvement for replication.lib [#61361](https://github.com/ClickHouse/ClickHouse/pull/61361) ([alesapin](https://github.com/alesapin)). +* Fix bugfix check (due to "unknown object storage type: azure") [#61363](https://github.com/ClickHouse/ClickHouse/pull/61363) ([Azat Khuzhin](https://github.com/azat)). +* Fix `01599_multiline_input_and_singleline_comments` 3 minute wait [#61371](https://github.com/ClickHouse/ClickHouse/pull/61371) ([Sergei Trifonov](https://github.com/serxa)). +* Terminate EC2 on spot event if runner isn't running [#61377](https://github.com/ClickHouse/ClickHouse/pull/61377) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Try fix docs check [#61378](https://github.com/ClickHouse/ClickHouse/pull/61378) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix `heap-use-after-free` for Merge table with alias [#61380](https://github.com/ClickHouse/ClickHouse/pull/61380) ([Antonio Andelic](https://github.com/antonio2368)). +* Disable `optimize_rewrite_sum_if_to_count_if` if return type is nullable (new analyzer) [#61389](https://github.com/ClickHouse/ClickHouse/pull/61389) ([Antonio Andelic](https://github.com/antonio2368)). +* Analyzer: Fix planner context for subquery in StorageMerge [#61392](https://github.com/ClickHouse/ClickHouse/pull/61392) ([Dmitry Novik](https://github.com/novikd)). +* Fix `test_failed_async_inserts` [#61394](https://github.com/ClickHouse/ClickHouse/pull/61394) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix test test_system_clusters_actual_information flakiness [#61395](https://github.com/ClickHouse/ClickHouse/pull/61395) ([Kruglov Pavel](https://github.com/Avogar)). +* Remove default cluster from default config from test config [#61396](https://github.com/ClickHouse/ClickHouse/pull/61396) ([Raúl Marín](https://github.com/Algunenano)). +* Enable clang-tidy in headers [#61406](https://github.com/ClickHouse/ClickHouse/pull/61406) ([Robert Schulze](https://github.com/rschu1ze)). +* Add sanity check for poll_max_batch_size FileLog setting [#61408](https://github.com/ClickHouse/ClickHouse/pull/61408) ([Kruglov Pavel](https://github.com/Avogar)). +* ThreadFuzzer: randomize sleep time [#61410](https://github.com/ClickHouse/ClickHouse/pull/61410) ([Tomer Shafir](https://github.com/tomershafir)). +* Update version_date.tsv and changelogs after v23.8.11.28-lts [#61416](https://github.com/ClickHouse/ClickHouse/pull/61416) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v23.3.21.26-lts [#61418](https://github.com/ClickHouse/ClickHouse/pull/61418) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v24.1.7.18-stable [#61419](https://github.com/ClickHouse/ClickHouse/pull/61419) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v24.2.2.71-stable [#61420](https://github.com/ClickHouse/ClickHouse/pull/61420) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v23.12.5.81-stable [#61421](https://github.com/ClickHouse/ClickHouse/pull/61421) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Restore automerge for approved PRs [#61433](https://github.com/ClickHouse/ClickHouse/pull/61433) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Disable broken SonarCloud [#61434](https://github.com/ClickHouse/ClickHouse/pull/61434) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix `01599_multiline_input_and_singleline_comments` properly [#61440](https://github.com/ClickHouse/ClickHouse/pull/61440) ([Sergei Trifonov](https://github.com/serxa)). +* Convert test 02998_system_dns_cache_table to smoke and mirrors [#61443](https://github.com/ClickHouse/ClickHouse/pull/61443) ([vdimir](https://github.com/vdimir)). +* Check boundaries for some settings in parallel replicas [#61455](https://github.com/ClickHouse/ClickHouse/pull/61455) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Use SHARD_LOAD_QUEUE_BACKLOG for dictionaries in tests [#61462](https://github.com/ClickHouse/ClickHouse/pull/61462) ([vdimir](https://github.com/vdimir)). +* Split `02125_lz4_compression_bug` [#61465](https://github.com/ClickHouse/ClickHouse/pull/61465) ([Antonio Andelic](https://github.com/antonio2368)). +* Correctly process last stacktrace in `postprocess-traces.pl` [#61470](https://github.com/ClickHouse/ClickHouse/pull/61470) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix test `test_polymorphic_parts` [#61477](https://github.com/ClickHouse/ClickHouse/pull/61477) ([Anton Popov](https://github.com/CurtizJ)). +* A definitive guide to CAST [#61491](https://github.com/ClickHouse/ClickHouse/pull/61491) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Minor rename in FileCache [#61494](https://github.com/ClickHouse/ClickHouse/pull/61494) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Remove useless code [#61498](https://github.com/ClickHouse/ClickHouse/pull/61498) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix fuzzers [#61499](https://github.com/ClickHouse/ClickHouse/pull/61499) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update jdbc.md [#61506](https://github.com/ClickHouse/ClickHouse/pull/61506) ([San](https://github.com/santrancisco)). +* Fix error in clickhouse-client [#61507](https://github.com/ClickHouse/ClickHouse/pull/61507) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix clang-tidy build [#61519](https://github.com/ClickHouse/ClickHouse/pull/61519) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix infinite loop in function `hop` [#61523](https://github.com/ClickHouse/ClickHouse/pull/61523) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Improve tests 00159_parallel_formatting_* to to avoid timeouts [#61532](https://github.com/ClickHouse/ClickHouse/pull/61532) ([Kruglov Pavel](https://github.com/Avogar)). +* Refactoring of reading from compact parts [#61535](https://github.com/ClickHouse/ClickHouse/pull/61535) ([Anton Popov](https://github.com/CurtizJ)). +* Don't run 01459_manual_write_to_replicas in debug build as it's too slow [#61538](https://github.com/ClickHouse/ClickHouse/pull/61538) ([Kruglov Pavel](https://github.com/Avogar)). +* CI: ARM integration test - skip hdfs, kerberos, kafka [#61542](https://github.com/ClickHouse/ClickHouse/pull/61542) ([Max K.](https://github.com/maxknv)). +* More logging for loading of tables [#61546](https://github.com/ClickHouse/ClickHouse/pull/61546) ([Sergei Trifonov](https://github.com/serxa)). +* Fixing 01584_distributed_buffer_cannot_find_column with analyzer. [#61550](https://github.com/ClickHouse/ClickHouse/pull/61550) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Wait for done mutation with more logs and asserts [#61554](https://github.com/ClickHouse/ClickHouse/pull/61554) ([alesapin](https://github.com/alesapin)). +* Fix read_rows count with external group by [#61555](https://github.com/ClickHouse/ClickHouse/pull/61555) ([Alexander Tokmakov](https://github.com/tavplubix)). +* queries-file should be used to specify file [#61557](https://github.com/ClickHouse/ClickHouse/pull/61557) ([danila-ermakov](https://github.com/danila-ermakov)). +* Fix `02481_async_insert_dedup_token` [#61568](https://github.com/ClickHouse/ClickHouse/pull/61568) ([Antonio Andelic](https://github.com/antonio2368)). +* Add a comment after [#61458](https://github.com/ClickHouse/ClickHouse/issues/61458) [#61580](https://github.com/ClickHouse/ClickHouse/pull/61580) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix clickhouse-test client option and CLICKHOUSE_URL_PARAMS interference [#61596](https://github.com/ClickHouse/ClickHouse/pull/61596) ([vdimir](https://github.com/vdimir)). +* CI: remove compose files from integration test docker [#61597](https://github.com/ClickHouse/ClickHouse/pull/61597) ([Max K.](https://github.com/maxknv)). +* Fix 01244_optimize_distributed_group_by_sharding_key by ordering output [#61602](https://github.com/ClickHouse/ClickHouse/pull/61602) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Remove some tests from analyzer_tech_debt [#61603](https://github.com/ClickHouse/ClickHouse/pull/61603) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Reduce header dependencies [#61604](https://github.com/ClickHouse/ClickHouse/pull/61604) ([Raúl Marín](https://github.com/Algunenano)). +* Remove some magic_enum from headers [#61606](https://github.com/ClickHouse/ClickHouse/pull/61606) ([Raúl Marín](https://github.com/Algunenano)). +* Fix configs for upgrade and bugfix [#61607](https://github.com/ClickHouse/ClickHouse/pull/61607) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add tests for multiple fuzzer issues [#61614](https://github.com/ClickHouse/ClickHouse/pull/61614) ([Raúl Marín](https://github.com/Algunenano)). +* Try to fix `02908_many_requests_to_system_replicas` again [#61616](https://github.com/ClickHouse/ClickHouse/pull/61616) ([Nikita Taranov](https://github.com/nickitat)). +* Verbose error message about analyzer_compatibility_join_using_top_level_identifier [#61631](https://github.com/ClickHouse/ClickHouse/pull/61631) ([vdimir](https://github.com/vdimir)). +* Fix 00223_shard_distributed_aggregation_memory_efficient with analyzer [#61649](https://github.com/ClickHouse/ClickHouse/pull/61649) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Better fuzzer logs [#61650](https://github.com/ClickHouse/ClickHouse/pull/61650) ([Raúl Marín](https://github.com/Algunenano)). +* Fix flaky `02122_parallel_formatting_Template` [#61651](https://github.com/ClickHouse/ClickHouse/pull/61651) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix Aggregator when data is empty [#61654](https://github.com/ClickHouse/ClickHouse/pull/61654) ([Antonio Andelic](https://github.com/antonio2368)). +* Restore poco SUN files [#61655](https://github.com/ClickHouse/ClickHouse/pull/61655) ([Andy Fiddaman](https://github.com/citrus-it)). +* Another fix for `SumIfToCountIfPass` [#61656](https://github.com/ClickHouse/ClickHouse/pull/61656) ([Antonio Andelic](https://github.com/antonio2368)). +* Keeper: fix data race during snapshot destructor call [#61657](https://github.com/ClickHouse/ClickHouse/pull/61657) ([Antonio Andelic](https://github.com/antonio2368)). +* CI: integration tests: use runner as py module [#61658](https://github.com/ClickHouse/ClickHouse/pull/61658) ([Max K.](https://github.com/maxknv)). +* Fix logging of autoscaling lambda, add test for effective_capacity [#61662](https://github.com/ClickHouse/ClickHouse/pull/61662) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Small change in `DatabaseOnDisk::iterateMetadataFiles()` [#61664](https://github.com/ClickHouse/ClickHouse/pull/61664) ([Nikita Taranov](https://github.com/nickitat)). +* Build improvements by removing magic enum from header and apply some explicit template instantiation [#61665](https://github.com/ClickHouse/ClickHouse/pull/61665) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* Update the dictionary for OSSFuzz [#61672](https://github.com/ClickHouse/ClickHouse/pull/61672) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Inhibit randomization in some tests and exclude some long tests from debug runs [#61676](https://github.com/ClickHouse/ClickHouse/pull/61676) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#61669](https://github.com/ClickHouse/ClickHouse/issues/61669) [#61678](https://github.com/ClickHouse/ClickHouse/pull/61678) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix use-of-uninitialized-value in HedgedConnections [#61679](https://github.com/ClickHouse/ClickHouse/pull/61679) ([Nikolay Degterinsky](https://github.com/evillique)). +* Remove clickhouse-diagnostics from the package [#61681](https://github.com/ClickHouse/ClickHouse/pull/61681) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix use-of-uninitialized-value in parseDateTimeBestEffort [#61694](https://github.com/ClickHouse/ClickHouse/pull/61694) ([Nikolay Degterinsky](https://github.com/evillique)). +* poco foundation: add illumos support [#61701](https://github.com/ClickHouse/ClickHouse/pull/61701) ([Andy Fiddaman](https://github.com/citrus-it)). +* contrib/c-ares: add illumos as a platform [#61702](https://github.com/ClickHouse/ClickHouse/pull/61702) ([Andy Fiddaman](https://github.com/citrus-it)). +* contrib/curl: Add illumos support [#61704](https://github.com/ClickHouse/ClickHouse/pull/61704) ([Andy Fiddaman](https://github.com/citrus-it)). +* Fuzzer: Try a different way to wait for the server [#61706](https://github.com/ClickHouse/ClickHouse/pull/61706) ([Raúl Marín](https://github.com/Algunenano)). +* Disable some tests for SMT [#61708](https://github.com/ClickHouse/ClickHouse/pull/61708) ([Raúl Marín](https://github.com/Algunenano)). +* Fix signal handler for sanitizer signals [#61709](https://github.com/ClickHouse/ClickHouse/pull/61709) ([Antonio Andelic](https://github.com/antonio2368)). +* Avoid `IsADirectoryError: Is a directory contrib/azure` [#61710](https://github.com/ClickHouse/ClickHouse/pull/61710) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Analyzer: fix group_by_use_nulls [#61717](https://github.com/ClickHouse/ClickHouse/pull/61717) ([Dmitry Novik](https://github.com/novikd)). +* Analyzer: Clear list of broken integration tests [#61718](https://github.com/ClickHouse/ClickHouse/pull/61718) ([Dmitry Novik](https://github.com/novikd)). +* CI: modify CI from PR body [#61725](https://github.com/ClickHouse/ClickHouse/pull/61725) ([Max K.](https://github.com/maxknv)). +* Add test for [#57820](https://github.com/ClickHouse/ClickHouse/issues/57820) [#61726](https://github.com/ClickHouse/ClickHouse/pull/61726) ([Dmitry Novik](https://github.com/novikd)). +* Revert "Revert "Un-flake test_undrop_query"" [#61727](https://github.com/ClickHouse/ClickHouse/pull/61727) ([Yarik Briukhovetskyi](https://github.com/yariks5s)). +* FunctionsConversion: Start simplifying templates [#61733](https://github.com/ClickHouse/ClickHouse/pull/61733) ([Raúl Marín](https://github.com/Algunenano)). +* CI: modify it [#61735](https://github.com/ClickHouse/ClickHouse/pull/61735) ([Max K.](https://github.com/maxknv)). +* Fix segfault in SquashingTransform [#61736](https://github.com/ClickHouse/ClickHouse/pull/61736) ([Michael Kolupaev](https://github.com/al13n321)). +* Fix DWARF format failing to skip DW_FORM_strx3 attributes [#61737](https://github.com/ClickHouse/ClickHouse/pull/61737) ([Michael Kolupaev](https://github.com/al13n321)). +* There is no such thing as broken tests [#61739](https://github.com/ClickHouse/ClickHouse/pull/61739) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Process removed files, decouple _check_mime [#61751](https://github.com/ClickHouse/ClickHouse/pull/61751) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Keeper fix: destroy `KeeperDispatcher` first [#61752](https://github.com/ClickHouse/ClickHouse/pull/61752) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix flaky `03014_async_with_dedup_part_log_rmt` [#61757](https://github.com/ClickHouse/ClickHouse/pull/61757) ([Antonio Andelic](https://github.com/antonio2368)). +* FunctionsConversion: Remove another batch of bad templates [#61773](https://github.com/ClickHouse/ClickHouse/pull/61773) ([Raúl Marín](https://github.com/Algunenano)). +* Revert "Fix bug when reading system.parts using UUID (issue 61220)." [#61774](https://github.com/ClickHouse/ClickHouse/pull/61774) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)). +* CI: disable grpc tests on ARM [#61778](https://github.com/ClickHouse/ClickHouse/pull/61778) ([Max K.](https://github.com/maxknv)). +* Fix more tests with virtual columns in StorageMerge. [#61787](https://github.com/ClickHouse/ClickHouse/pull/61787) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove already not flaky tests with analyzer. [#61788](https://github.com/ClickHouse/ClickHouse/pull/61788) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Analyzer: Fix assert in JOIN with Distributed table [#61789](https://github.com/ClickHouse/ClickHouse/pull/61789) ([vdimir](https://github.com/vdimir)). +* A test can be slow in debug build [#61796](https://github.com/ClickHouse/ClickHouse/pull/61796) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Updated clang-19 to master. [#61798](https://github.com/ClickHouse/ClickHouse/pull/61798) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix test "00002_log_and_exception_messages_formatting" [#61821](https://github.com/ClickHouse/ClickHouse/pull/61821) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* A test is too slow for debug [#61822](https://github.com/ClickHouse/ClickHouse/pull/61822) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove DataStreams [#61824](https://github.com/ClickHouse/ClickHouse/pull/61824) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Better message for logging errors [#61827](https://github.com/ClickHouse/ClickHouse/pull/61827) ([Azat Khuzhin](https://github.com/azat)). +* Fix sanitizers suppressions [#61828](https://github.com/ClickHouse/ClickHouse/pull/61828) ([Azat Khuzhin](https://github.com/azat)). +* Remove unused code [#61830](https://github.com/ClickHouse/ClickHouse/pull/61830) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove DataStreams (2) [#61831](https://github.com/ClickHouse/ClickHouse/pull/61831) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update xxhash to v0.8.2 [#61838](https://github.com/ClickHouse/ClickHouse/pull/61838) ([Shubham Ranjan](https://github.com/shubhamranjan)). +* Fix: DISTINCT in subquery with analyzer [#61847](https://github.com/ClickHouse/ClickHouse/pull/61847) ([Igor Nikonov](https://github.com/devcrafter)). +* Analyzer: fix limit/offset on shards [#61849](https://github.com/ClickHouse/ClickHouse/pull/61849) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Remove PoolBase::AllocateNewBypassingPool [#61866](https://github.com/ClickHouse/ClickHouse/pull/61866) ([Azat Khuzhin](https://github.com/azat)). +* Try to fix 02901_parallel_replicas_rollup with analyzer. [#61875](https://github.com/ClickHouse/ClickHouse/pull/61875) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Add test for [#57808](https://github.com/ClickHouse/ClickHouse/issues/57808) [#61879](https://github.com/ClickHouse/ClickHouse/pull/61879) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* CI: merge queue support [#61881](https://github.com/ClickHouse/ClickHouse/pull/61881) ([Max K.](https://github.com/maxknv)). +* Update create.sql [#61885](https://github.com/ClickHouse/ClickHouse/pull/61885) ([Kseniia Sumarokova](https://github.com/kssenii)). +* no smaller unit in date_trunc [#61888](https://github.com/ClickHouse/ClickHouse/pull/61888) ([jsc0218](https://github.com/jsc0218)). +* Move KQL trash where it is supposed to be [#61903](https://github.com/ClickHouse/ClickHouse/pull/61903) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Changelog for 24.3 [#61909](https://github.com/ClickHouse/ClickHouse/pull/61909) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update version_date.tsv and changelogs after v23.3.22.3-lts [#61914](https://github.com/ClickHouse/ClickHouse/pull/61914) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v23.8.12.13-lts [#61915](https://github.com/ClickHouse/ClickHouse/pull/61915) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* No "please" [#61916](https://github.com/ClickHouse/ClickHouse/pull/61916) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update version_date.tsv and changelogs after v23.12.6.19-stable [#61917](https://github.com/ClickHouse/ClickHouse/pull/61917) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v24.1.8.22-stable [#61918](https://github.com/ClickHouse/ClickHouse/pull/61918) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Fix flaky test_broken_projestions/test.py::test_broken_ignored_replic… [#61932](https://github.com/ClickHouse/ClickHouse/pull/61932) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Check is Rust avaiable for build, if not, suggest a way to disable Rust support [#61938](https://github.com/ClickHouse/ClickHouse/pull/61938) ([Azat Khuzhin](https://github.com/azat)). +* CI: new ci menu in PR body [#61948](https://github.com/ClickHouse/ClickHouse/pull/61948) ([Max K.](https://github.com/maxknv)). +* Remove flaky test `01193_metadata_loading` [#61961](https://github.com/ClickHouse/ClickHouse/pull/61961) ([Nikita Taranov](https://github.com/nickitat)). + +#### Packaging Improvement + +* Adding the `--now` option to enable and start service automatically when installing the database server completely. [#60656](https://github.com/ClickHouse/ClickHouse/pull/60656) ([Chun-Sheng, Li](https://github.com/peter279k)). + diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index a76bb01ce9e..b6235fd1182 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -1270,12 +1270,13 @@ SELECT * FROM json_each_row_nested - [input_format_json_read_arrays_as_strings](/docs/en/operations/settings/settings-formats.md/#input_format_json_read_arrays_as_strings) - allow to parse JSON arrays as strings in JSON input formats. Default value - `true`. - [input_format_json_read_objects_as_strings](/docs/en/operations/settings/settings-formats.md/#input_format_json_read_objects_as_strings) - allow to parse JSON objects as strings in JSON input formats. Default value - `true`. - [input_format_json_named_tuples_as_objects](/docs/en/operations/settings/settings-formats.md/#input_format_json_named_tuples_as_objects) - parse named tuple columns as JSON objects. Default value - `true`. -- [input_format_json_try_infer_numbers_from_strings](/docs/en/operations/settings/settings-formats.md/#input_format_json_try_infer_numbers_from_strings) - Try to infer numbers from string fields while schema inference. Default value - `false`. +- [input_format_json_try_infer_numbers_from_strings](/docs/en/operations/settings/settings-formats.md/#input_format_json_try_infer_numbers_from_strings) - try to infer numbers from string fields while schema inference. Default value - `false`. - [input_format_json_try_infer_named_tuples_from_objects](/docs/en/operations/settings/settings-formats.md/#input_format_json_try_infer_named_tuples_from_objects) - try to infer named tuple from JSON objects during schema inference. Default value - `true`. - [input_format_json_infer_incomplete_types_as_strings](/docs/en/operations/settings/settings-formats.md/#input_format_json_infer_incomplete_types_as_strings) - use type String for keys that contains only Nulls or empty objects/arrays during schema inference in JSON input formats. Default value - `true`. - [input_format_json_defaults_for_missing_elements_in_named_tuple](/docs/en/operations/settings/settings-formats.md/#input_format_json_defaults_for_missing_elements_in_named_tuple) - insert default values for missing elements in JSON object while parsing named tuple. Default value - `true`. -- [input_format_json_ignore_unknown_keys_in_named_tuple](/docs/en/operations/settings/settings-formats.md/#input_format_json_ignore_unknown_keys_in_named_tuple) - Ignore unknown keys in json object for named tuples. Default value - `false`. +- [input_format_json_ignore_unknown_keys_in_named_tuple](/docs/en/operations/settings/settings-formats.md/#input_format_json_ignore_unknown_keys_in_named_tuple) - ignore unknown keys in json object for named tuples. Default value - `false`. - [input_format_json_compact_allow_variable_number_of_columns](/docs/en/operations/settings/settings-formats.md/#input_format_json_compact_allow_variable_number_of_columns) - allow variable number of columns in JSONCompact/JSONCompactEachRow format, ignore extra columns and use default values on missing columns. Default value - `false`. +- [input_format_json_throw_on_bad_escape_sequence](/docs/en/operations/settings/settings-formats.md/#input_format_json_throw_on_bad_escape_sequence) - throw an exception if JSON string contains bad escape sequence. If disabled, bad escape sequences will remain as is in the data. Default value - `true`. - [output_format_json_quote_64bit_integers](/docs/en/operations/settings/settings-formats.md/#output_format_json_quote_64bit_integers) - controls quoting of 64-bit integers in JSON output format. Default value - `true`. - [output_format_json_quote_64bit_floats](/docs/en/operations/settings/settings-formats.md/#output_format_json_quote_64bit_floats) - controls quoting of 64-bit floats in JSON output format. Default value - `false`. - [output_format_json_quote_denormals](/docs/en/operations/settings/settings-formats.md/#output_format_json_quote_denormals) - enables '+nan', '-nan', '+inf', '-inf' outputs in JSON output format. Default value - `false`. @@ -2356,7 +2357,7 @@ You can select data from a ClickHouse table and save them into some file in the $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Arrow" > {filename.arrow} ``` -### Arrow format settings {#parquet-format-settings} +### Arrow format settings {#arrow-format-settings} - [output_format_arrow_low_cardinality_as_dictionary](/docs/en/operations/settings/settings-formats.md/#output_format_arrow_low_cardinality_as_dictionary) - enable output ClickHouse LowCardinality type as Dictionary Arrow type. Default value - `false`. - [output_format_arrow_use_64_bit_indexes_for_dictionary](/docs/en/operations/settings/settings-formats.md/#output_format_arrow_use_64_bit_indexes_for_dictionary) - use 64-bit integer type for Dictionary indexes. Default value - `false`. diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 63fbd9d1964..436321c8fe8 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -945,9 +945,9 @@ Hard limit is configured via system tools ## database_atomic_delay_before_drop_table_sec {#database_atomic_delay_before_drop_table_sec} -Sets the delay before remove table data in seconds. If the query has `SYNC` modifier, this setting is ignored. +The delay during which a dropped table can be restored using the [UNDROP](/docs/en/sql-reference/statements/undrop.md) statement. If `DROP TABLE` ran with a `SYNC` modifier, the setting is ignored. -Default value: `480` (8 minute). +Default value: `480` (8 minutes). ## database_catalog_unused_dir_hide_timeout_sec {#database_catalog_unused_dir_hide_timeout_sec} @@ -1354,6 +1354,7 @@ Keys: - `count` – The number of archived log files that ClickHouse stores. - `console` – Send `log` and `errorlog` to the console instead of file. To enable, set to `1` or `true`. - `stream_compress` – Compress `log` and `errorlog` with `lz4` stream compression. To enable, set to `1` or `true`. +- `formatting` – Specify log format to be printed in console log (currently only `json` supported). Both log and error log file names (only file names, not directories) support date and time format specifiers. @@ -1422,6 +1423,8 @@ Writing to the console can be configured. Config example: ``` +### syslog + Writing to the syslog is also supported. Config example: ``` xml @@ -1445,6 +1448,52 @@ Keys for syslog: Default value: `LOG_USER` if `address` is specified, `LOG_DAEMON` otherwise. - format – Message format. Possible values: `bsd` and `syslog.` +### Log formats + +You can specify the log format that will be outputted in the console log. Currently, only JSON is supported. Here is an example of an output JSON log: + +```json +{ + "date_time": "1650918987.180175", + "thread_name": "#1", + "thread_id": "254545", + "level": "Trace", + "query_id": "", + "logger_name": "BaseDaemon", + "message": "Received signal 2", + "source_file": "../base/daemon/BaseDaemon.cpp; virtual void SignalListener::run()", + "source_line": "192" +} +``` +To enable JSON logging support, use the following snippet: + +```xml + + + json + + date_time + thread_name + thread_id + level + query_id + logger_name + message + source_file + source_line + + + +``` + +**Renaming keys for JSON logs** + +Key names can be modified by changing tag values inside the `` tag. For example, to change `DATE_TIME` to `MY_DATE_TIME`, you can use `MY_DATE_TIME`. + +**Omitting keys for JSON logs** + +Log properties can be omitted by commenting out the property. For example, if you do not want your log to print `query_id`, you can comment out the `` tag. + ## send_crash_reports {#send_crash_reports} Settings for opt-in sending crash reports to the ClickHouse core developers team via [Sentry](https://sentry.io). diff --git a/docs/en/operations/settings/settings-formats.md b/docs/en/operations/settings/settings-formats.md index 477fbf94625..831c7094114 100644 --- a/docs/en/operations/settings/settings-formats.md +++ b/docs/en/operations/settings/settings-formats.md @@ -651,6 +651,12 @@ This setting works only when setting `input_format_json_named_tuples_as_objects` Enabled by default. +## input_format_json_throw_on_bad_escape_sequence {#input_format_json_throw_on_bad_escape_sequence} + +Throw an exception if JSON string contains bad escape sequence in JSON input formats. If disabled, bad escape sequences will remain as is in the data. + +Enabled by default. + ## output_format_json_array_of_rows {#output_format_json_array_of_rows} Enables the ability to output all rows as a JSON array in the [JSONEachRow](../../interfaces/formats.md/#jsoneachrow) format. @@ -1367,7 +1373,7 @@ Default value: `1'000'000`. While importing data, when column is not found in schema default value will be used instead of error. -Disabled by default. +Enabled by default. ### input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference {#input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference} diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 6c427442e49..e4e7be83f7d 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1776,7 +1776,7 @@ Default value: 0 (no restriction). ## insert_quorum {#insert_quorum} :::note -This setting is not applicable to SharedMergeTree, see [SharedMergeTree consistency](/docs/en/cloud/reference/shared-merge-tree/#consistency) for more information. +This setting is not applicable to SharedMergeTree, see [SharedMergeTree consistency](/docs/en/cloud/reference/shared-merge-tree/#consistency) for more information. ::: Enables the quorum writes. @@ -1819,7 +1819,7 @@ See also: ## insert_quorum_parallel {#insert_quorum_parallel} :::note -This setting is not applicable to SharedMergeTree, see [SharedMergeTree consistency](/docs/en/cloud/reference/shared-merge-tree/#consistency) for more information. +This setting is not applicable to SharedMergeTree, see [SharedMergeTree consistency](/docs/en/cloud/reference/shared-merge-tree/#consistency) for more information. ::: Enables or disables parallelism for quorum `INSERT` queries. If enabled, additional `INSERT` queries can be sent while previous queries have not yet finished. If disabled, additional writes to the same table will be rejected. @@ -1840,7 +1840,7 @@ See also: ## select_sequential_consistency {#select_sequential_consistency} :::note -This setting differ in behavior between SharedMergeTree and ReplicatedMergeTree, see [SharedMergeTree consistency](/docs/en/cloud/reference/shared-merge-tree/#consistency) for more information about the behavior of `select_sequential_consistency` in SharedMergeTree. +This setting differ in behavior between SharedMergeTree and ReplicatedMergeTree, see [SharedMergeTree consistency](/docs/en/cloud/reference/shared-merge-tree/#consistency) for more information about the behavior of `select_sequential_consistency` in SharedMergeTree. ::: Enables or disables sequential consistency for `SELECT` queries. Requires `insert_quorum_parallel` to be disabled (enabled by default). @@ -2817,6 +2817,17 @@ Possible values: Default value: 0. +## distributed_insert_skip_read_only_replicas {#distributed_insert_skip_read_only_replicas} + +Enables skipping read-only replicas for INSERT queries into Distributed. + +Possible values: + +- 0 — INSERT was as usual, if it will go to read-only replica it will fail +- 1 — Initiator will skip read-only replicas before sending data to shards. + +Default value: `0` + ## distributed_foreground_insert {#distributed_foreground_insert} Enables or disables synchronous data insertion into a [Distributed](../../engines/table-engines/special/distributed.md/#distributed) table. @@ -5442,3 +5453,7 @@ Enabling this setting can lead to incorrect result as in case of evolved schema ::: Default value: 'false'. + +## allow_suspicious_primary_key {#allow_suspicious_primary_key} + +Allow suspicious `PRIMARY KEY`/`ORDER BY` for MergeTree (i.e. SimpleAggregateFunction). diff --git a/docs/en/operations/storing-data.md b/docs/en/operations/storing-data.md index fd81bc197d1..9ffbb64c1ed 100644 --- a/docs/en/operations/storing-data.md +++ b/docs/en/operations/storing-data.md @@ -520,13 +520,13 @@ Example of configuration for versions later or equal to 22.8: - +
cache
- + ``` @@ -546,13 +546,13 @@ Example of configuration for versions earlier than 22.8: - +
s3
- + ``` diff --git a/docs/en/operations/system-tables/index.md b/docs/en/operations/system-tables/index.md index eaf79d035a9..d9800e05ff9 100644 --- a/docs/en/operations/system-tables/index.md +++ b/docs/en/operations/system-tables/index.md @@ -47,7 +47,7 @@ An example: ENGINE = MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024 --> 7500 - 1048576 + 1048576 8192 524288 false diff --git a/docs/en/sql-reference/aggregate-functions/parametric-functions.md b/docs/en/sql-reference/aggregate-functions/parametric-functions.md index 3654cd157e9..38dca6b7071 100644 --- a/docs/en/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/en/sql-reference/aggregate-functions/parametric-functions.md @@ -483,7 +483,7 @@ Where: - `r1`- the number of unique visitors who visited the site during 2020-01-01 (the `cond1` condition). - `r2`- the number of unique visitors who visited the site during a specific time period between 2020-01-01 and 2020-01-02 (`cond1` and `cond2` conditions). -- `r3`- the number of unique visitors who visited the site during a specific time period between 2020-01-01 and 2020-01-03 (`cond1` and `cond3` conditions). +- `r3`- the number of unique visitors who visited the site during a specific time period on 2020-01-01 and 2020-01-03 (`cond1` and `cond3` conditions). ## uniqUpTo(N)(x) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index ba7695af3fa..4c419004344 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -1670,7 +1670,7 @@ Like [fromDaysSinceYearZero](#fromDaysSinceYearZero) but returns a [Date32](../. ## age -Returns the `unit` component of the difference between `startdate` and `enddate`. The difference is calculated using a precision of 1 microsecond. +Returns the `unit` component of the difference between `startdate` and `enddate`. The difference is calculated using a precision of 1 nanosecond. E.g. the difference between `2021-12-29` and `2022-01-01` is 3 days for `day` unit, 0 months for `month` unit, 0 years for `year` unit. For an alternative to `age`, see function `date\_diff`. @@ -1686,16 +1686,17 @@ age('unit', startdate, enddate, [timezone]) - `unit` — The type of interval for result. [String](../../sql-reference/data-types/string.md). Possible values: - - `microsecond` `microseconds` `us` `u` - - `millisecond` `milliseconds` `ms` - - `second` `seconds` `ss` `s` - - `minute` `minutes` `mi` `n` - - `hour` `hours` `hh` `h` - - `day` `days` `dd` `d` - - `week` `weeks` `wk` `ww` - - `month` `months` `mm` `m` - - `quarter` `quarters` `qq` `q` - - `year` `years` `yyyy` `yy` + - `nanosecond`, `nanoseconds`, `ns` + - `microsecond`, `microseconds`, `us`, `u` + - `millisecond`, `milliseconds`, `ms` + - `second`, `seconds`, `ss`, `s` + - `minute`, `minutes`, `mi`, `n` + - `hour`, `hours`, `hh`, `h` + - `day`, `days`, `dd`, `d` + - `week`, `weeks`, `wk`, `ww` + - `month`, `months`, `mm`, `m` + - `quarter`, `quarters`, `qq`, `q` + - `year`, `years`, `yyyy`, `yy` - `startdate` — The first time value to subtract (the subtrahend). [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). @@ -1763,16 +1764,17 @@ Aliases: `dateDiff`, `DATE_DIFF`, `timestampDiff`, `timestamp_diff`, `TIMESTAMP_ - `unit` — The type of interval for result. [String](../../sql-reference/data-types/string.md). Possible values: - - `microsecond` `microseconds` `us` `u` - - `millisecond` `milliseconds` `ms` - - `second` `seconds` `ss` `s` - - `minute` `minutes` `mi` `n` - - `hour` `hours` `hh` `h` - - `day` `days` `dd` `d` - - `week` `weeks` `wk` `ww` - - `month` `months` `mm` `m` - - `quarter` `quarters` `qq` `q` - - `year` `years` `yyyy` `yy` + - `nanosecond`, `nanoseconds`, `ns` + - `microsecond`, `microseconds`, `us`, `u` + - `millisecond`, `milliseconds`, `ms` + - `second`, `seconds`, `ss`, `s` + - `minute`, `minutes`, `mi`, `n` + - `hour`, `hours`, `hh`, `h` + - `day`, `days`, `dd`, `d` + - `week`, `weeks`, `wk`, `ww` + - `month`, `months`, `mm`, `m` + - `quarter`, `quarters`, `qq`, `q` + - `year`, `years`, `yyyy`, `yy` - `startdate` — The first time value to subtract (the subtrahend). [Date](../../sql-reference/data-types/date.md), [Date32](../../sql-reference/data-types/date32.md), [DateTime](../../sql-reference/data-types/datetime.md) or [DateTime64](../../sql-reference/data-types/datetime64.md). diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 82cc68133da..e9f8bc6e547 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -543,12 +543,64 @@ You can get similar result by using the [ternary operator](../../sql-reference/f Returns 1 if the Float32 and Float64 argument is NaN, otherwise this function 0. -## hasColumnInTable(\[‘hostname’\[, ‘username’\[, ‘password’\]\],\] ‘database’, ‘table’, ‘column’) +## hasColumnInTable + +Given the database name, the table name, and the column name as constant strings, returns 1 if the given column exists, otherwise 0. + +**Syntax** + +```sql +hasColumnInTable(\[‘hostname’\[, ‘username’\[, ‘password’\]\],\] ‘database’, ‘table’, ‘column’) +``` + +**Parameters** + +- `database` : name of the database. [String literal](../syntax#syntax-string-literal) +- `table` : name of the table. [String literal](../syntax#syntax-string-literal) +- `column` : name of the column. [String literal](../syntax#syntax-string-literal) +- `hostname` : remote server name to perform the check on. [String literal](../syntax#syntax-string-literal) +- `username` : username for remote server. [String literal](../syntax#syntax-string-literal) +- `password` : password for remote server. [String literal](../syntax#syntax-string-literal) + +**Returned value** + +- `1` if the given column exists. +- `0`, otherwise. + +**Implementation details** -Given the database name, the table name, and the column name as constant strings, returns 1 if the given column exists, otherwise 0. If parameter `hostname` is given, the check is performed on a remote server. -If the table does not exist, an exception is thrown. For elements in a nested data structure, the function checks for the existence of a column. For the nested data structure itself, the function returns 0. +**Example** + +Query: + +```sql +SELECT hasColumnInTable('system','metrics','metric') +``` + +```response +1 +``` + +```sql +SELECT hasColumnInTable('system','metrics','non-existing_column') +``` + +```response +0 +``` + +## hasThreadFuzzer + +Returns whether Thread Fuzzer is effective. It can be used in tests to prevent runs from being too long. + +**Syntax** + +```sql +hasThreadFuzzer(); +``` + ## bar Builds a bar chart. diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md index b4e2adbed3c..573790f7ff7 100644 --- a/docs/en/sql-reference/functions/string-functions.md +++ b/docs/en/sql-reference/functions/string-functions.md @@ -99,7 +99,7 @@ Alias: `OCTET_LENGTH` Returns the length of a string in Unicode code points (not: in bytes or characters). It assumes that the string contains valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined. Alias: -- `CHAR_LENGTH`` +- `CHAR_LENGTH` - `CHARACTER_LENGTH` ## leftPad diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 1b03f220db2..117e3818dc6 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -74,6 +74,8 @@ SELECT position('Hello, world!', 'o', 7) ``` +Result: + ``` text ┌─position('Hello, world!', 'o', 1)─┬─position('Hello, world!', 'o', 7)─┐ │ 5 │ 9 │ @@ -479,9 +481,9 @@ Alias: `haystack NOT ILIKE pattern` (operator) ## ngramDistance -Calculates the 4-gram distance between a `haystack` string and a `needle` string. For that, it counts the symmetric difference between two multisets of 4-grams and normalizes it by the sum of their cardinalities. Returns a Float32 between 0 and 1. The smaller the result is, the more strings are similar to each other. Throws an exception if constant `needle` or `haystack` arguments are more than 32Kb in size. If any of non-constant `haystack` or `needle` arguments is more than 32Kb in size, the distance is always 1. +Calculates the 4-gram distance between a `haystack` string and a `needle` string. For this, it counts the symmetric difference between two multisets of 4-grams and normalizes it by the sum of their cardinalities. Returns a [Float32](../../sql-reference/data-types/float.md/#float32-float64) between 0 and 1. The smaller the result is, the more similar the strings are to each other. -Functions `ngramDistanceCaseInsensitive, ngramDistanceUTF8, ngramDistanceCaseInsensitiveUTF8` provide case-insensitive and/or UTF-8 variants of this function. +Functions [`ngramDistanceCaseInsensitive`](#ngramdistancecaseinsensitive), [`ngramDistanceUTF8`](#ngramdistanceutf8), [`ngramDistanceCaseInsensitiveUTF8`](#ngramdistancecaseinsensitiveutf8) provide case-insensitive and/or UTF-8 variants of this function. **Syntax** @@ -489,15 +491,170 @@ Functions `ngramDistanceCaseInsensitive, ngramDistanceUTF8, ngramDistanceCaseIns ngramDistance(haystack, needle) ``` +**Parameters** + +- `haystack`: First comparison string. [String literal](../syntax#string) +- `needle`: Second comparison string. [String literal](../syntax#string) + +**Returned value** + +- Value between 0 and 1 representing the similarity between the two strings. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +**Implementation details** + +This function will throw an exception if constant `needle` or `haystack` arguments are more than 32Kb in size. If any non-constant `haystack` or `needle` arguments are more than 32Kb in size, then the distance is always 1. + +**Examples** + +The more similar two strings are to each other, the closer the result will be to 0 (identical). + +Query: + +```sql +SELECT ngramDistance('ClickHouse','ClickHouse!'); +``` + +Result: + +```response +0.06666667 +``` + +The less similar two strings are to each, the larger the result will be. + + +Query: + +```sql +SELECT ngramDistance('ClickHouse','House'); +``` + +Result: + +```response +0.5555556 +``` + +## ngramDistanceCaseInsensitive + +Provides a case-insensitive variant of [ngramDistance](#ngramdistance). + +**Syntax** + +```sql +ngramDistanceCaseInsensitive(haystack, needle) +``` + +**Parameters** + +- `haystack`: First comparison string. [String literal](../syntax#string) +- `needle`: Second comparison string. [String literal](../syntax#string) + +**Returned value** + +- Value between 0 and 1 representing the similarity between the two strings. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +**Examples** + +With [ngramDistance](#ngramdistance) differences in case will affect the similarity value: + +Query: + +```sql +SELECT ngramDistance('ClickHouse','clickhouse'); +``` + +Result: + +```response +0.71428573 +``` + +With [ngramDistanceCaseInsensitive](#ngramdistancecaseinsensitive) case is ignored so two identical strings differing only in case will now return a low similarity value: + +Query: + +```sql +SELECT ngramDistanceCaseInsensitive('ClickHouse','clickhouse'); +``` + +Result: + +```response +0 +``` + +## ngramDistanceUTF8 + +Provides a UTF-8 variant of [ngramDistance](#ngramdistance). Assumes that `needle` and `haystack` strings are UTF-8 encoded strings. + +**Syntax** + +```sql +ngramDistanceUTF8(haystack, needle) +``` + +**Parameters** + +- `haystack`: First UTF-8 encoded comparison string. [String literal](../syntax#string) +- `needle`: Second UTF-8 encoded comparison string. [String literal](../syntax#string) + +**Returned value** + +- Value between 0 and 1 representing the similarity between the two strings. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +**Example** + +Query: + +```sql +SELECT ngramDistanceUTF8('abcde','cde'); +``` + +Result: + +```response +0.5 +``` + +## ngramDistanceCaseInsensitiveUTF8 + +Provides a case-insensitive variant of [ngramDistanceUTF8](#ngramdistanceutf8). + +**Syntax** + +```sql +ngramDistanceCaseInsensitiveUTF8(haystack, needle) +``` + +**Parameters** + +- `haystack`: First UTF-8 encoded comparison string. [String literal](../syntax#string) +- `needle`: Second UTF-8 encoded comparison string. [String literal](../syntax#string) + +**Returned value** + +- Value between 0 and 1 representing the similarity between the two strings. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +**Example** + +Query: + +```sql +SELECT ngramDistanceCaseInsensitiveUTF8('abcde','CDE'); +``` + +Result: + +```response +0.5 +``` + ## ngramSearch -Like `ngramDistance` but calculates the non-symmetric difference between a `needle` string and a `haystack` string, i.e. the number of n-grams from needle minus the common number of n-grams normalized by the number of `needle` n-grams. Returns a Float32 between 0 and 1. The bigger the result is, the more likely `needle` is in the `haystack`. This function is useful for fuzzy string search. Also see function `soundex`. +Like `ngramDistance` but calculates the non-symmetric difference between a `needle` string and a `haystack` string, i.e. the number of n-grams from the needle minus the common number of n-grams normalized by the number of `needle` n-grams. Returns a [Float32](../../sql-reference/data-types/float.md/#float32-float64) between 0 and 1. The bigger the result is, the more likely `needle` is in the `haystack`. This function is useful for fuzzy string search. Also see function [`soundex`](../../sql-reference/functions/string-functions#soundex). -Functions `ngramSearchCaseInsensitive, ngramSearchUTF8, ngramSearchCaseInsensitiveUTF8` provide case-insensitive and/or UTF-8 variants of this function. - -:::note -The UTF-8 variants use the 3-gram distance. These are not perfectly fair n-gram distances. We use 2-byte hashes to hash n-grams and then calculate the (non-)symmetric difference between these hash tables – collisions may occur. With UTF-8 case-insensitive format we do not use fair `tolower` function – we zero the 5-th bit (starting from zero) of each codepoint byte and first bit of zeroth byte if bytes more than one – this works for Latin and mostly for all Cyrillic letters. -::: +Functions [`ngramSearchCaseInsensitive`](#ngramsearchcaseinsensitive), [`ngramSearchUTF8`](#ngramsearchutf8), [`ngramSearchCaseInsensitiveUTF8`](#ngramsearchcaseinsensitiveutf8) provide case-insensitive and/or UTF-8 variants of this function. **Syntax** @@ -505,6 +662,140 @@ The UTF-8 variants use the 3-gram distance. These are not perfectly fair n-gram ngramSearch(haystack, needle) ``` +**Parameters** + +- `haystack`: First comparison string. [String literal](../syntax#string) +- `needle`: Second comparison string. [String literal](../syntax#string) + +**Returned value** + +- Value between 0 and 1 representing the likelihood of the `needle` being in the `haystack`. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +**Implementation details** + +:::note +The UTF-8 variants use the 3-gram distance. These are not perfectly fair n-gram distances. We use 2-byte hashes to hash n-grams and then calculate the (non-)symmetric difference between these hash tables – collisions may occur. With UTF-8 case-insensitive format we do not use fair `tolower` function – we zero the 5-th bit (starting from zero) of each codepoint byte and first bit of zeroth byte if bytes more than one – this works for Latin and mostly for all Cyrillic letters. +::: + +**Example** + +Query: + +```sql +SELECT ngramSearch('Hello World','World Hello'); +``` + +Result: + +```response +0.5 +``` + +## ngramSearchCaseInsensitive + +Provides a case-insensitive variant of [ngramSearch](#ngramSearch). + +**Syntax** + +```sql +ngramSearchCaseInsensitive(haystack, needle) +``` + +**Parameters** + +- `haystack`: First comparison string. [String literal](../syntax#string) +- `needle`: Second comparison string. [String literal](../syntax#string) + +**Returned value** + +- Value between 0 and 1 representing the likelihood of the `needle` being in the `haystack`. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +The bigger the result is, the more likely `needle` is in the `haystack`. + +**Example** + +Query: + +```sql +SELECT ngramSearchCaseInsensitive('Hello World','hello'); +``` + +Result: + +```response +1 +``` + +## ngramSearchUTF8 + +Provides a UTF-8 variant of [ngramSearch](#ngramsearch) in which `needle` and `haystack` are assumed to be UTF-8 encoded strings. + +**Syntax** + +```sql +ngramSearchUTF8(haystack, needle) +``` + +**Parameters** + +- `haystack`: First UTF-8 encoded comparison string. [String literal](../syntax#string) +- `needle`: Second UTF-8 encoded comparison string. [String literal](../syntax#string) + +**Returned value** + +- Value between 0 and 1 representing the likelihood of the `needle` being in the `haystack`. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +The bigger the result is, the more likely `needle` is in the `haystack`. + +**Example** + +Query: + +```sql +SELECT ngramSearchUTF8('абвгдеёжз', 'гдеёзд'); +``` + +Result: + +```response +0.5 +``` + +## ngramSearchCaseInsensitiveUTF8 + +Provides a case-insensitive variant of [ngramSearchUTF8](#ngramsearchutf8) in which `needle` and `haystack`. + +**Syntax** + +```sql +ngramSearchCaseInsensitiveUTF8(haystack, needle) +``` + +**Parameters** + +- `haystack`: First UTF-8 encoded comparison string. [String literal](../syntax#string) +- `needle`: Second UTF-8 encoded comparison string. [String literal](../syntax#string) + +**Returned value** + +- Value between 0 and 1 representing the likelihood of the `needle` being in the `haystack`. [Float32](../../sql-reference/data-types/float.md/#float32-float64) + +The bigger the result is, the more likely `needle` is in the `haystack`. + +**Example** + +Query: + +```sql +SELECT ngramSearchCaseInsensitiveUTF8('абвГДЕёжз', 'АбвгдЕЁжз'); +``` + +Result: + +```response +0.57142854 +``` + ## countSubstrings Returns how often substring `needle` occurs in string `haystack`. @@ -610,7 +901,7 @@ Like `countMatches(haystack, pattern)` but matching ignores the case. ## regexpExtract -Extracts the first string in haystack that matches the regexp pattern and corresponds to the regex group index. +Extracts the first string in `haystack` that matches the regexp pattern and corresponds to the regex group index. **Syntax** @@ -652,7 +943,7 @@ Result: ## hasSubsequence -Returns 1 if needle is a subsequence of haystack, or 0 otherwise. +Returns 1 if `needle` is a subsequence of `haystack`, or 0 otherwise. A subsequence of a string is a sequence that can be derived from the given string by deleting zero or more elements without changing the order of the remaining elements. @@ -676,8 +967,10 @@ Type: `UInt8`. **Examples** +Query: + ``` sql -SELECT hasSubsequence('garbage', 'arg') ; +SELECT hasSubsequence('garbage', 'arg'); ``` Result: @@ -692,10 +985,263 @@ Result: Like [hasSubsequence](#hasSubsequence) but searches case-insensitively. +**Syntax** + +``` sql +hasSubsequenceCaseInsensitive(haystack, needle) +``` + +**Arguments** + +- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Subsequence to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal). + +**Returned values** + +- 1, if needle is a subsequence of haystack. +- 0, otherwise. + +Type: `UInt8`. + +**Examples** + +Query: + +``` sql +SELECT hasSubsequenceCaseInsensitive('garbage', 'ARG'); +``` + +Result: + +``` text +┌─hasSubsequenceCaseInsensitive('garbage', 'ARG')─┐ +│ 1 │ +└─────────────────────────────────────────────────┘ +``` + ## hasSubsequenceUTF8 Like [hasSubsequence](#hasSubsequence) but assumes `haystack` and `needle` are UTF-8 encoded strings. +**Syntax** + +``` sql +hasSubsequenceUTF8(haystack, needle) +``` + +**Arguments** + +- `haystack` — String in which the search is performed. UTF-8 encoded [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Subsequence to be searched. UTF-8 encoded [String](../../sql-reference/syntax.md#syntax-string-literal). + +**Returned values** + +- 1, if needle is a subsequence of haystack. +- 0, otherwise. + +Type: `UInt8`. + +Query: + +**Examples** + +``` sql +select hasSubsequenceUTF8('ClickHouse - столбцовая система управления базами данных', 'система'); +``` + +Result: + +``` text +┌─hasSubsequenceUTF8('ClickHouse - столбцовая система управления базами данных', 'система')─┐ +│ 1 │ +└───────────────────────────────────────────────────────────────────────────────────────────┘ +``` + ## hasSubsequenceCaseInsensitiveUTF8 Like [hasSubsequenceUTF8](#hasSubsequenceUTF8) but searches case-insensitively. + +**Syntax** + +``` sql +hasSubsequenceCaseInsensitiveUTF8(haystack, needle) +``` + +**Arguments** + +- `haystack` — String in which the search is performed. UTF-8 encoded [String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — Subsequence to be searched. UTF-8 encoded [String](../../sql-reference/syntax.md#syntax-string-literal). + +**Returned values** + +- 1, if needle is a subsequence of haystack. +- 0, otherwise. + +Type: `UInt8`. + +**Examples** + +Query: + +``` sql +select hasSubsequenceCaseInsensitiveUTF8('ClickHouse - столбцовая система управления базами данных', 'СИСТЕМА'); +``` + +Result: + +``` text +┌─hasSubsequenceCaseInsensitiveUTF8('ClickHouse - столбцовая система управления базами данных', 'СИСТЕМА')─┐ +│ 1 │ +└──────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +## hasToken + +Returns 1 if a given token is present in a haystack, or 0 otherwise. + +**Syntax** + +```sql +hasToken(haystack, token) +``` + +**Parameters** + +- `haystack`: String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `token`: Maximal length substring between two non alphanumeric ASCII characters (or boundaries of haystack). + +**Returned value** + +- 1, if the token is present in the haystack. +- 0, if the token is not present. + +**Implementation details** + +Token must be a constant string. Supported by tokenbf_v1 index specialization. + +**Example** + +Query: + +```sql +SELECT hasToken('Hello World','Hello'); +``` + +```response +1 +``` + +## hasTokenOrNull + +Returns 1 if a given token is present, 0 if not present, and null if the token is ill-formed. + +**Syntax** + +```sql +hasTokenOrNull(haystack, token) +``` + +**Parameters** + +- `haystack`: String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `token`: Maximal length substring between two non alphanumeric ASCII characters (or boundaries of haystack). + +**Returned value** + +- 1, if the token is present in the haystack. +- 0, if the token is not present in the haystack. +- null, if the token is ill-formed. + +**Implementation details** + +Token must be a constant string. Supported by tokenbf_v1 index specialization. + +**Example** + +Where `hasToken` would throw an error for an ill-formed token, `hasTokenOrNull` returns `null` for an ill-formed token. + +Query: + +```sql +SELECT hasTokenOrNull('Hello World','Hello,World'); +``` + +```response +null +``` + +## hasTokenCaseInsensitive + +Returns 1 if a given token is present in a haystack, 0 otherwise. Ignores case. + +**Syntax** + +```sql +hasTokenCaseInsensitive(haystack, token) +``` + +**Parameters** + +- `haystack`: String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `token`: Maximal length substring between two non alphanumeric ASCII characters (or boundaries of haystack). + +**Returned value** + +- 1, if the token is present in the haystack. +- 0, otherwise. + +**Implementation details** + +Token must be a constant string. Supported by tokenbf_v1 index specialization. + +**Example** + +Query: + +```sql +SELECT hasTokenCaseInsensitive('Hello World','hello'); +``` + +```response +1 +``` + +## hasTokenCaseInsensitiveOrNull + +Returns 1 if a given token is present in a haystack, 0 otherwise. Ignores case and returns null if the token is ill-formed. + +**Syntax** + +```sql +hasTokenCaseInsensitiveOrNull(haystack, token) +``` + +**Parameters** + +- `haystack`: String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal). +- `token`: Maximal length substring between two non alphanumeric ASCII characters (or boundaries of haystack). + +**Returned value** + +- 1, if the token is present in the haystack. +- 0, if token is not present. +- null, if the token is ill-formed. + +**Implementation details** + +Token must be a constant string. Supported by tokenbf_v1 index specialization. + +**Example** + + +Where `hasTokenCaseInsensitive` would throw an error for an ill-formed token, `hasTokenCaseInsensitiveOrNull` returns `null` for an ill-formed token. + +Query: + +```sql +SELECT hasTokenCaseInsensitiveOrNull('Hello World','hello,world'); +``` + +```response +null +``` \ No newline at end of file diff --git a/docs/en/sql-reference/functions/url-functions.md b/docs/en/sql-reference/functions/url-functions.md index ac81815b47f..a0b0170721c 100644 --- a/docs/en/sql-reference/functions/url-functions.md +++ b/docs/en/sql-reference/functions/url-functions.md @@ -128,9 +128,9 @@ Returns the part of the domain that includes top-level subdomains up to the “f For example: -- `cutToFirstSignificantSubdomain('https://news.clickhouse.com.tr/') = 'clickhouse.com.tr'`. -- `cutToFirstSignificantSubdomain('www.tr') = 'www.tr'`. -- `cutToFirstSignificantSubdomain('tr') = ''`. +- `cutToFirstSignificantSubdomainWithWWW('https://news.clickhouse.com.tr/') = 'clickhouse.com.tr'`. +- `cutToFirstSignificantSubdomainWithWWW('www.tr') = 'www.tr'`. +- `cutToFirstSignificantSubdomainWithWWW('tr') = ''`. ### cutToFirstSignificantSubdomainCustom diff --git a/docs/en/sql-reference/statements/alter/index.md b/docs/en/sql-reference/statements/alter/index.md index dc6668c7983..7961315c193 100644 --- a/docs/en/sql-reference/statements/alter/index.md +++ b/docs/en/sql-reference/statements/alter/index.md @@ -56,7 +56,9 @@ Entries for finished mutations are not deleted right away (the number of preserv For non-replicated tables, all `ALTER` queries are performed synchronously. For replicated tables, the query just adds instructions for the appropriate actions to `ZooKeeper`, and the actions themselves are performed as soon as possible. However, the query can wait for these actions to be completed on all the replicas. -For all `ALTER` queries, you can use the [alter_sync](/docs/en/operations/settings/settings.md/#alter-sync) setting to set up waiting. +For `ALTER` queries that creates mutations (e.g.: including, but not limited to `UPDATE`, `DELETE`, `MATERIALIZE INDEX`, `MATERIALIZE PROJECTION`, `MATERIALIZE COLUMN`, `APPLY DELETED MASK`, `CLEAR STATISTIC`, `MATERIALIZE STATISTIC`) the synchronicity is defined by the [mutations_sync](/docs/en/operations/settings/settings.md/#mutations_sync) setting. + +For other `ALTER` queries which only modify the metadata, you can use the [alter_sync](/docs/en/operations/settings/settings.md/#alter-sync) setting to set up waiting. You can specify how long (in seconds) to wait for inactive replicas to execute all `ALTER` queries with the [replication_wait_for_inactive_replica_timeout](/docs/en/operations/settings/settings.md/#replication-wait-for-inactive-replica-timeout) setting. @@ -64,8 +66,6 @@ You can specify how long (in seconds) to wait for inactive replicas to execute a For all `ALTER` queries, if `alter_sync = 2` and some replicas are not active for more than the time, specified in the `replication_wait_for_inactive_replica_timeout` setting, then an exception `UNFINISHED` is thrown. ::: -For `ALTER TABLE ... UPDATE|DELETE|MATERIALIZE INDEX|MATERIALIZE PROJECTION|MATERIALIZE COLUMN` queries the synchronicity is defined by the [mutations_sync](/docs/en/operations/settings/settings.md/#mutations_sync) setting. - ## Related content - Blog: [Handling Updates and Deletes in ClickHouse](https://clickhouse.com/blog/handling-updates-and-deletes-in-clickhouse) diff --git a/docs/en/sql-reference/statements/drop.md b/docs/en/sql-reference/statements/drop.md index 8ed00f625d6..159ab09ab94 100644 --- a/docs/en/sql-reference/statements/drop.md +++ b/docs/en/sql-reference/statements/drop.md @@ -30,9 +30,11 @@ Also see [UNDROP TABLE](/docs/en/sql-reference/statements/undrop.md) Syntax: ``` sql -DROP [TEMPORARY] TABLE [IF EXISTS] [IF EMPTY] [db.]name [ON CLUSTER cluster] [SYNC] +DROP [TEMPORARY] TABLE [IF EXISTS] [IF EMPTY] [db1.]name_1[, [db2.]name_2, ...] [ON CLUSTER cluster] [SYNC] ``` +Note that deleting multiple tables at the same time is a non-atomic deletion. If a table fails to be deleted, subsequent tables will not be deleted. + ## DROP DICTIONARY Deletes the dictionary. diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index 1dee2eac698..b35e9426297 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -64,6 +64,14 @@ RELOAD FUNCTIONS [ON CLUSTER cluster_name] RELOAD FUNCTION [ON CLUSTER cluster_name] function_name ``` +## RELOAD ASYNCHRONOUS METRICS + +Re-calculates all [asynchronous metrics](../../operations/system-tables/asynchronous_metrics.md). Since asynchronous metrics are periodically updated based on setting [asynchronous_metrics_update_period_s](../../operations/server-configuration-parameters/settings.md), updating them manually using this statement is typically not necessary. + +```sql +RELOAD ASYNCHRONOUS METRICS [ON CLUSTER cluster_name] +``` + ## DROP DNS CACHE Clears ClickHouse’s internal DNS cache. Sometimes (for old ClickHouse versions) it is necessary to use this command when changing the infrastructure (changing the IP address of another ClickHouse server or the server used by dictionaries). diff --git a/docs/en/sql-reference/statements/truncate.md b/docs/en/sql-reference/statements/truncate.md index 029815a4392..8cd5a6a1424 100644 --- a/docs/en/sql-reference/statements/truncate.md +++ b/docs/en/sql-reference/statements/truncate.md @@ -23,9 +23,16 @@ You can specify how long (in seconds) to wait for inactive replicas to execute ` If the `alter_sync` is set to `2` and some replicas are not active for more than the time, specified by the `replication_wait_for_inactive_replica_timeout` setting, then an exception `UNFINISHED` is thrown. ::: +## TRUNCATE ALL TABLES +``` sql +TRUNCATE ALL TABLES [IF EXISTS] db [ON CLUSTER cluster] +``` + +Removes all data from all tables in a database. + ## TRUNCATE DATABASE ``` sql -TRUNCATE DATABASE [IF EXISTS] [db.]name [ON CLUSTER cluster] +TRUNCATE DATABASE [IF EXISTS] db [ON CLUSTER cluster] ``` Removes all tables from a database but keeps the database itself. When the clause `IF EXISTS` is omitted, the query returns an error if the database does not exist. diff --git a/docs/en/sql-reference/statements/undrop.md b/docs/en/sql-reference/statements/undrop.md index 40ac1ab4f99..4b138bfe679 100644 --- a/docs/en/sql-reference/statements/undrop.md +++ b/docs/en/sql-reference/statements/undrop.md @@ -13,13 +13,6 @@ a system table called `system.dropped_tables`. If you have a materialized view without a `TO` clause associated with the dropped table, then you will also have to UNDROP the inner table of that view. -:::note -UNDROP TABLE is experimental. To use it add this setting: -```sql -set allow_experimental_undrop_table_query = 1; -``` -::: - :::tip Also see [DROP TABLE](/docs/en/sql-reference/statements/drop.md) ::: @@ -32,60 +25,53 @@ UNDROP TABLE [db.]name [UUID ''] [ON CLUSTER cluster] **Example** -``` sql -set allow_experimental_undrop_table_query = 1; -``` - ```sql -CREATE TABLE undropMe +CREATE TABLE tab ( `id` UInt8 ) ENGINE = MergeTree -ORDER BY id -``` +ORDER BY id; + +DROP TABLE tab; -```sql -DROP TABLE undropMe -``` -```sql SELECT * FROM system.dropped_tables -FORMAT Vertical +FORMAT Vertical; ``` + ```response Row 1: ────── index: 0 database: default -table: undropMe +table: tab uuid: aa696a1a-1d70-4e60-a841-4c80827706cc engine: MergeTree -metadata_dropped_path: /var/lib/clickhouse/metadata_dropped/default.undropMe.aa696a1a-1d70-4e60-a841-4c80827706cc.sql +metadata_dropped_path: /var/lib/clickhouse/metadata_dropped/default.tab.aa696a1a-1d70-4e60-a841-4c80827706cc.sql table_dropped_time: 2023-04-05 14:12:12 1 row in set. Elapsed: 0.001 sec. ``` + ```sql -UNDROP TABLE undropMe -``` -```response -Ok. -``` -```sql +UNDROP TABLE tab; + SELECT * FROM system.dropped_tables -FORMAT Vertical -``` +FORMAT Vertical; + ```response Ok. 0 rows in set. Elapsed: 0.001 sec. ``` + ```sql -DESCRIBE TABLE undropMe -FORMAT Vertical +DESCRIBE TABLE tab +FORMAT Vertical; ``` + ```response Row 1: ────── diff --git a/docs/en/sql-reference/table-functions/generate.md b/docs/en/sql-reference/table-functions/generate.md index 3b9b077af49..a78015e9830 100644 --- a/docs/en/sql-reference/table-functions/generate.md +++ b/docs/en/sql-reference/table-functions/generate.md @@ -53,7 +53,7 @@ SELECT * FROM random; └──────────────────────────────┴──────────────┴────────────────────────────────────────────────────────────────────┘ ``` -In combination with [generateRandomStructure](../../sql-reference/functions/other-functions.md#generateRandomStructure): +In combination with [generateRandomStructure](../../sql-reference/functions/other-functions.md#generaterandomstructure): ```sql SELECT * FROM generateRandom(generateRandomStructure(4, 101), 101) LIMIT 3; diff --git a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md index 59a9c7f8cf1..6463f6bd95d 100644 --- a/docs/ru/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/ru/sql-reference/aggregate-functions/parametric-functions.md @@ -476,7 +476,7 @@ FROM - `r1` - количество уникальных посетителей за 2020-01-01 (`cond1`). - `r2` - количество уникальных посетителей в период между 2020-01-01 и 2020-01-02 (`cond1` и `cond2`). -- `r3` - количество уникальных посетителей в период между 2020-01-01 и 2020-01-03 (`cond1` и `cond3`). +- `r3` - количество уникальных посетителей в период за 2020-01-01 и 2020-01-03 (`cond1` и `cond3`). ## uniqUpTo(N)(x) {#uniquptonx} diff --git a/docs/ru/sql-reference/functions/date-time-functions.md b/docs/ru/sql-reference/functions/date-time-functions.md index cbbb456aa80..56ae4359bf1 100644 --- a/docs/ru/sql-reference/functions/date-time-functions.md +++ b/docs/ru/sql-reference/functions/date-time-functions.md @@ -627,7 +627,7 @@ SELECT toDate('2016-12-27') AS date, toYearWeek(date) AS yearWeek0, toYearWeek(d ## age -Вычисляет компонент `unit` разницы между `startdate` и `enddate`. Разница вычисляется с точностью в 1 микросекунду. +Вычисляет компонент `unit` разницы между `startdate` и `enddate`. Разница вычисляется с точностью в 1 наносекунду. Например, разница между `2021-12-29` и `2022-01-01` 3 дня для единицы `day`, 0 месяцев для единицы `month`, 0 лет для единицы `year`. **Синтаксис** @@ -641,6 +641,7 @@ age('unit', startdate, enddate, [timezone]) - `unit` — единица измерения времени, в которой будет выражено возвращаемое значение функции. [String](../../sql-reference/data-types/string.md). Возможные значения: + - `nanosecond` (возможные сокращения: `ns`) - `microsecond` (возможные сокращения: `us`, `u`) - `millisecond` (возможные сокращения: `ms`) - `second` (возможные сокращения: `ss`, `s`) @@ -716,6 +717,7 @@ date_diff('unit', startdate, enddate, [timezone]) - `unit` — единица измерения времени, в которой будет выражено возвращаемое значение функции. [String](../../sql-reference/data-types/string.md). Возможные значения: + - `nanosecond` (возможные сокращения: `ns`) - `microsecond` (возможные сокращения: `us`, `u`) - `millisecond` (возможные сокращения: `ms`) - `second` (возможные сокращения: `ss`, `s`) diff --git a/docs/zh/engines/database-engines/materialize-mysql.md b/docs/zh/engines/database-engines/materialize-mysql.md deleted file mode 100644 index 5d1394f9456..00000000000 --- a/docs/zh/engines/database-engines/materialize-mysql.md +++ /dev/null @@ -1,196 +0,0 @@ ---- -slug: /zh/engines/database-engines/materialize-mysql -sidebar_position: 29 -sidebar_label: "[experimental] MaterializedMySQL" ---- - -# [experimental] MaterializedMySQL {#materialized-mysql} - -**这是一个实验性的特性，不应该在生产中使用。** - -创建ClickHouse数据库，包含MySQL中所有的表，以及这些表中的所有数据。 - -ClickHouse服务器作为MySQL副本工作。它读取binlog并执行DDL和DML查询。 - -这个功能是实验性的。 - -## 创建数据库 {#creating-a-database} - -``` sql -CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] -ENGINE = MaterializeMySQL('host:port', ['database' | database], 'user', 'password') [SETTINGS ...] -``` - -**引擎参数** - -- `host:port` — MySQL服务地址 -- `database` — MySQL数据库名称 -- `user` — MySQL用户名 -- `password` — MySQL用户密码 - -**引擎配置** - -- `max_rows_in_buffer` — 允许数据缓存到内存中的最大行数(对于单个表和无法查询的缓存数据)。当超过行数时，数据将被物化。默认值: `65505`。 -- `max_bytes_in_buffer` — 允许在内存中缓存数据的最大字节数(对于单个表和无法查询的缓存数据)。当超过行数时，数据将被物化。默认值: `1048576`. -- `max_rows_in_buffers` — 允许数据缓存到内存中的最大行数(对于数据库和无法查询的缓存数据)。当超过行数时，数据将被物化。默认值: `65505`. -- `max_bytes_in_buffers` — 允许在内存中缓存数据的最大字节数(对于数据库和无法查询的缓存数据)。当超过行数时，数据将被物化。默认值: `1048576`. -- `max_flush_data_time` — 允许数据在内存中缓存的最大毫秒数(对于数据库和无法查询的缓存数据)。当超过这个时间时，数据将被物化。默认值: `1000`. -- `max_wait_time_when_mysql_unavailable` — 当MySQL不可用时重试间隔(毫秒)。负值禁止重试。默认值: `1000`. -- `allows_query_when_mysql_lost` — 当mysql丢失时，允许查询物化表。默认值: `0` (`false`). -``` -CREATE DATABASE mysql ENGINE = MaterializeMySQL('localhost:3306', 'db', 'user', '***') - SETTINGS - allows_query_when_mysql_lost=true, - max_wait_time_when_mysql_unavailable=10000; -``` - -**MySQL服务器端配置** - -为了`MaterializeMySQL`正确的工作，有一些强制性的`MySQL`侧配置设置应该设置: - -- `default_authentication_plugin = mysql_native_password`，因为`MaterializeMySQL`只能使用此方法授权。 -- `gtid_mode = on`，因为要提供正确的`MaterializeMySQL`复制，基于GTID的日志记录是必须的。注意，在打开这个模式`On`时，你还应该指定`enforce_gtid_consistency = on`。 - -## 虚拟列 {#virtual-columns} - -当使用`MaterializeMySQL`数据库引擎时，[ReplacingMergeTree](../../engines/table-engines/mergetree-family/replacingmergetree.md)表与虚拟的`_sign`和`_version`列一起使用。 - -- `_version` — 同步版本。 类型[UInt64](../../sql-reference/data-types/int-uint.md). -- `_sign` — 删除标记。类型 [Int8](../../sql-reference/data-types/int-uint.md). Possible values: - - `1` — 行不会删除, - - `-1` — 行被删除。 - -## 支持的数据类型 {#data_types-support} - -| MySQL | ClickHouse | -|-------------------------|--------------------------------------------------------------| -| TINY | [Int8](../../sql-reference/data-types/int-uint.md) | -| SHORT | [Int16](../../sql-reference/data-types/int-uint.md) | -| INT24 | [Int32](../../sql-reference/data-types/int-uint.md) | -| LONG | [UInt32](../../sql-reference/data-types/int-uint.md) | -| LONGLONG | [UInt64](../../sql-reference/data-types/int-uint.md) | -| FLOAT | [Float32](../../sql-reference/data-types/float.md) | -| DOUBLE | [Float64](../../sql-reference/data-types/float.md) | -| DECIMAL, NEWDECIMAL | [Decimal](../../sql-reference/data-types/decimal.md) | -| DATE, NEWDATE | [Date](../../sql-reference/data-types/date.md) | -| DATETIME, TIMESTAMP | [DateTime](../../sql-reference/data-types/datetime.md) | -| DATETIME2, TIMESTAMP2 | [DateTime64](../../sql-reference/data-types/datetime64.md) | -| ENUM | [Enum](../../sql-reference/data-types/enum.md) | -| STRING | [String](../../sql-reference/data-types/string.md) | -| VARCHAR, VAR_STRING | [String](../../sql-reference/data-types/string.md) | -| BLOB | [String](../../sql-reference/data-types/string.md) | -| BINARY | [FixedString](../../sql-reference/data-types/fixedstring.md) | - -不支持其他类型。如果MySQL表包含此类类型的列，ClickHouse抛出异常"Unhandled data type"并停止复制。 - -[Nullable](../../sql-reference/data-types/nullable.md)已经支持 - -## 使用方式 {#specifics-and-recommendations} - -### 兼容性限制 - -除了数据类型的限制外，与`MySQL`数据库相比，还存在一些限制，在实现复制之前应先解决这些限制： - -- `MySQL`中的每个表都应该包含`PRIMARY KEY` - -- 对于包含`ENUM`字段值超出范围（在`ENUM`签名中指定）的行的表，复制将不起作用。 - -### DDL查询 {#ddl-queries} - -MySQL DDL查询转换为相应的ClickHouse DDL查询([ALTER](../../sql-reference/statements/alter/index.md), [CREATE](../../sql-reference/statements/create.md), [DROP](../../sql-reference/statements/drop.md), [RENAME](../../sql-reference/statements/rename.md))。如果ClickHouse无法解析某个DDL查询，则该查询将被忽略。 - -### Data Replication {#data-replication} - -`MaterializeMySQL`不支持直接`INSERT`, `DELETE`和`UPDATE`查询. 但是，它们是在数据复制方面支持的： - -- MySQL的`INSERT`查询转换为`INSERT`并携带`_sign=1`. - -- MySQL的`DELETE`查询转换为`INSERT`并携带`_sign=-1`. - -- MySQL的`UPDATE`查询转换为`INSERT`并携带`_sign=-1`, `INSERT`和`_sign=1`. - -### 查询MaterializeMySQL表 {#select} - -`SELECT`查询`MaterializeMySQL`表有一些细节: - -- 如果`_version`在`SELECT`中没有指定，则使用[FINAL](../../sql-reference/statements/select/from.md#select-from-final)修饰符。所以只有带有`MAX(_version)`的行才会被选中。 - -- 如果`_sign`在`SELECT`中没有指定，则默认使用`WHERE _sign=1`。因此，删除的行不会包含在结果集中。 - -- 结果包括列中的列注释，因为它们存在于SQL数据库表中。 - -### Index Conversion {#index-conversion} - -MySQL的`PRIMARY KEY`和`INDEX`子句在ClickHouse表中转换为`ORDER BY`元组。 - -ClickHouse只有一个物理顺序，由`ORDER BY`子句决定。要创建一个新的物理顺序，使用[materialized views](../../sql-reference/statements/create/view.md#materialized)。 - -**Notes** - -- 带有`_sign=-1`的行不会从表中物理删除。 -- `MaterializeMySQL`引擎不支持级联`UPDATE/DELETE`查询。 -- 复制很容易被破坏。 -- 禁止对数据库和表进行手工操作。 -- `MaterializeMySQL`受[optimize_on_insert](../../operations/settings/settings.md#optimize-on-insert)设置的影响。当MySQL服务器中的表发生变化时，数据会合并到`MaterializeMySQL`数据库中相应的表中。 - -## 使用示例 {#examples-of-use} - -MySQL操作: - -``` sql -mysql> CREATE DATABASE db; -mysql> CREATE TABLE db.test (a INT PRIMARY KEY, b INT); -mysql> INSERT INTO db.test VALUES (1, 11), (2, 22); -mysql> DELETE FROM db.test WHERE a=1; -mysql> ALTER TABLE db.test ADD COLUMN c VARCHAR(16); -mysql> UPDATE db.test SET c='Wow!', b=222; -mysql> SELECT * FROM test; -``` - -```text -+---+------+------+ -| a | b | c | -+---+------+------+ -| 2 | 222 | Wow! | -+---+------+------+ -``` - -ClickHouse中的数据库，与MySQL服务器交换数据: - -创建的数据库和表: - -``` sql -CREATE DATABASE mysql ENGINE = MaterializeMySQL('localhost:3306', 'db', 'user', '***'); -SHOW TABLES FROM mysql; -``` - -``` text -┌─name─┐ -│ test │ -└──────┘ -``` - -然后插入数据: - -``` sql -SELECT * FROM mysql.test; -``` - -``` text -┌─a─┬──b─┐ -│ 1 │ 11 │ -│ 2 │ 22 │ -└───┴────┘ -``` - -删除数据后，添加列并更新: - -``` sql -SELECT * FROM mysql.test; -``` - -``` text -┌─a─┬───b─┬─c────┐ -│ 2 │ 222 │ Wow! │ -└───┴─────┴──────┘ -``` diff --git a/docs/zh/sql-reference/aggregate-functions/parametric-functions.md b/docs/zh/sql-reference/aggregate-functions/parametric-functions.md index 1c7de515c58..cb1dcc35f5c 100644 --- a/docs/zh/sql-reference/aggregate-functions/parametric-functions.md +++ b/docs/zh/sql-reference/aggregate-functions/parametric-functions.md @@ -472,7 +472,7 @@ FROM - `r1`-2020-01-01期间访问该网站的独立访问者数量（ `cond1` 条件）。 - `r2`-在2020-01-01和2020-01-02之间的特定时间段内访问该网站的唯一访问者的数量 (`cond1` 和 `cond2` 条件）。 -- `r3`-在2020-01-01和2020-01-03之间的特定时间段内访问该网站的唯一访问者的数量 (`cond1` 和 `cond3` 条件）。 +- `r3`-在2020-01-01和2020-01-03 网站的独立访客数量 (`cond1` 和 `cond3` 条件）。 ## uniqUpTo(N)(x) {#uniquptonx} diff --git a/docs/zh/sql-reference/functions/date-time-functions.md b/docs/zh/sql-reference/functions/date-time-functions.md index e4b70322477..d6493ffe605 100644 --- a/docs/zh/sql-reference/functions/date-time-functions.md +++ b/docs/zh/sql-reference/functions/date-time-functions.md @@ -643,6 +643,7 @@ date_diff('unit', startdate, enddate, [timezone]) - `unit` — `value`对应的时间单位。类型为[String](../../sql-reference/data-types/string.md)。 可能的值： + - `nanosecond` - `microsecond` - `millisecond` - `second` diff --git a/docs/zh/sql-reference/functions/string-search-functions.md b/docs/zh/sql-reference/functions/string-search-functions.md index e4167127424..972fd84e2a1 100644 --- a/docs/zh/sql-reference/functions/string-search-functions.md +++ b/docs/zh/sql-reference/functions/string-search-functions.md @@ -1,128 +1,702 @@ --- slug: /zh/sql-reference/functions/string-search-functions --- -# 字符串搜索函数 {#zi-fu-chuan-sou-suo-han-shu} -下列所有函数在默认的情况下区分大小写。对于不区分大小写的搜索，存在单独的变体。 +# 字符串搜索函数 -## 位置（大海捞针），定位（大海捞针) {#positionhaystack-needle-locatehaystack-needle} +本节中的所有函数默认情况下都区分大小写进行搜索。不区分大小写的搜索通常由单独的函数变体提供。 +请注意，不区分大小写的搜索，遵循英语的小写-大写规则。 +例如。英语中大写的`i`是`I`，而在土耳其语中则是`İ`, 对于英语以外的语言，结果可能会不符合预期。 -在字符串`haystack`中搜索子串`needle`。 -返回子串的位置（以字节为单位），从1开始，如果未找到子串，则返回0。 +本节中的函数还假设搜索字符串和被搜索字符串是单字节编码文本(例如ASCII)。如果违反此假设，不会抛出异常且结果为undefined。 +UTF-8 编码字符串的搜索通常由单独的函数变体提供。同样，如果使用 UTF-8 函数变体但输入字符串不是 UTF-8 编码文本，不会抛出异常且结果为 undefined。 +需要注意，函数不会执行自动 Unicode 规范化，您可以使用[normalizeUTF8*()](https://clickhouse.com/docs/zh/sql-reference/functions/string-functions/) 函数来执行此操作。 +在[字符串函数](string-functions.md) 和 [字符串替换函数](string-replace-functions.md) 会分别说明. -对于不区分大小写的搜索，请使用函数`positionCaseInsensitive`。 +## position -## positionUTF8(大海捞针) {#positionutf8haystack-needle} +返回字符串`haystack`中子字符串`needle`的位置（以字节为单位，从 1 开始）。 -与`position`相同，但位置以Unicode字符返回。此函数工作在UTF-8编码的文本字符集中。如非此编码的字符集，则返回一些非预期结果（他不会抛出异常）。 +**语法** -对于不区分大小写的搜索，请使用函数`positionCaseInsensitiveUTF8`。 +``` sql +position(haystack, needle[, start_pos]) +``` -## 多搜索分配（干草堆，\[针1，针2, …, needlen\]) {#multisearchallpositionshaystack-needle1-needle2-needlen} +别名： +- `position(needle IN haystack)` -与`position`相同，但函数返回一个数组，其中包含所有匹配needle我的位置。 +**参数** -对于不区分大小写的搜索或/和UTF-8格式，使用函数`multiSearchAllPositionsCaseInsensitive，multiSearchAllPositionsUTF8，multiSearchAllPositionsCaseInsensitiveUTF8`。 +- `haystack` — 被检索查询字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — 进行查询的子字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `start_pos` – 在字符串`haystack` 中开始检索的位置(从1开始)，类型为[UInt](../../sql-reference/data-types/int-uint.md)，可选。 -## multiSearchFirstPosition(大海捞针,\[针1，针2, …, needlen\]) {#multisearchfirstpositionhaystack-needle1-needle2-needlen} +**返回值** -与`position`相同，但返回在`haystack`中与needles字符串匹配的最左偏移。 +- 若子字符串存在，返回位置(以字节为单位，从 1 开始）。 +- 如果不存在子字符串，返回 0。 -对于不区分大小写的搜索或/和UTF-8格式，使用函数`multiSearchFirstPositionCaseInsensitive，multiSearchFirstPositionUTF8，multiSearchFirstPositionCaseInsensitiveUTF8`。 +如果子字符串 `needle` 为空，则： +- 如果未指定 `start_pos`，返回 `1` +- 如果 `start_pos = 0`，则返回 `1` +- 如果 `start_pos >= 1` 且 `start_pos <= length(haystack) + 1`，则返回 `start_pos` +- 否则返回 `0` -## multiSearchFirstIndex(大海捞针,\[针1，针2, …, needlen\]) {#multisearchfirstindexhaystack-needle1-needle2-needlen} +以上规则同样在这些函数中生效: [locate](#locate), [positionCaseInsensitive](#positionCaseInsensitive), [positionUTF8](#positionUTF8), [positionCaseInsensitiveUTF8](#positionCaseInsensitiveUTF8) -返回在字符串`haystack`中最先查找到的needle我的索引`i`（从1开始），没有找到任何匹配项则返回0。 +数据类型: `Integer`. -对于不区分大小写的搜索或/和UTF-8格式，使用函数`multiSearchFirstIndexCaseInsensitive，multiSearchFirstIndexUTF8，multiSearchFirstIndexCaseInsensitiveUTF8`。 +**示例** -## 多搜索（大海捞针，\[针1，针2, …, needlen\]) {#multisearchanyhaystack-needle1-needle2-needlen} +``` sql +SELECT position('Hello, world!', '!'); +``` -如果`haystack`中至少存在一个needle我匹配则返回1，否则返回0。 +结果： + +``` text +┌─position('Hello, world!', '!')─┐ +│ 13 │ +└────────────────────────────────┘ +``` + +示例，使用参数 `start_pos` : + +``` sql +SELECT + position('Hello, world!', 'o', 1), + position('Hello, world!', 'o', 7) +``` + +结果： + +``` text +┌─position('Hello, world!', 'o', 1)─┬─position('Hello, world!', 'o', 7)─┐ +│ 5 │ 9 │ +└───────────────────────────────────┴───────────────────────────────────┘ +``` + +示例，`needle IN haystack`: + +```sql +SELECT 6 = position('/' IN s) FROM (SELECT 'Hello/World' AS s); +``` + +结果： + +```text +┌─equals(6, position(s, '/'))─┐ +│ 1 │ +└─────────────────────────────┘ +``` + +示例，子字符串 `needle` 为空: + +``` sql +SELECT + position('abc', ''), + position('abc', '', 0), + position('abc', '', 1), + position('abc', '', 2), + position('abc', '', 3), + position('abc', '', 4), + position('abc', '', 5) +``` +结果： +``` text +┌─position('abc', '')─┬─position('abc', '', 0)─┬─position('abc', '', 1)─┬─position('abc', '', 2)─┬─position('abc', '', 3)─┬─position('abc', '', 4)─┬─position('abc', '', 5)─┐ +│ 1 │ 1 │ 1 │ 2 │ 3 │ 4 │ 0 │ +└─────────────────────┴────────────────────────┴────────────────────────┴────────────────────────┴────────────────────────┴────────────────────────┴────────────────────────┘ +``` + +## locate + +类似于 [position](#position) 但交换了 `haystack` 和 `locate` 参数。 + +此函数的行为取决于 ClickHouse 版本： +- 在 v24.3 以下的版本中，`locate` 是函数`position`的别名，参数为 `(haystack, needle[, start_pos])`。 +- 在 v24.3 及以上的版本中,, `locate` 是独立的函数 (以更好地兼容 MySQL) ,参数为 `(needle, haystack[, start_pos])`。 之前的行为 + 可以在设置中恢复 [function_locate_has_mysql_compatible_argument_order = false](../../operations/settings/settings.md#function-locate-has-mysql-compatible-argument-order); + +**语法** + +``` sql +locate(needle, haystack[, start_pos]) +``` + +## positionCaseInsensitive + +类似于 [position](#position) 但是不区分大小写。 + +## positionUTF8 + +类似于 [position](#position) 但是假定 `haystack` 和 `needle` 是 UTF-8 编码的字符串。 + +**示例** + +函数 `positionUTF8` 可以正确的将字符 `ö` 计为单个 Unicode 代码点(`ö`由两个点表示): + +``` sql +SELECT positionUTF8('Motörhead', 'r'); +``` + +结果： + +``` text +┌─position('Motörhead', 'r')─┐ +│ 5 │ +└────────────────────────────┘ +``` + +## positionCaseInsensitiveUTF8 + +类似于 [positionUTF8](#positionutf8) 但是不区分大小写。 + +## multiSearchAllPositions + +类似于 [position](#position) 但是返回多个在字符串 `haystack` 中 `needle` 子字符串的位置的数组（以字节为单位，从 1 开始）。 -对于不区分大小写的搜索或/和UTF-8格式，使用函数`multiSearchAnyCaseInsensitive，multiSearchAnyUTF8，multiSearchAnyCaseInsensitiveUTF8`。 :::note -在所有`multiSearch*`函数中，由于实现规范，needles的数量应小于28。 +所有以 `multiSearch*()` 开头的函数仅支持最多 28 个`needle`. ::: -## 匹配（大海捞针，模式) {#matchhaystack-pattern} +**语法** -检查字符串是否与`pattern`正则表达式匹配。`pattern`可以是一个任意的`re2`正则表达式。 `re2`正则表达式的[语法](https://github.com/google/re2/wiki/Syntax)比Perl正则表达式的语法存在更多限制。 +``` sql +multiSearchAllPositions(haystack, [needle1, needle2, ..., needleN]) +``` -如果不匹配返回0，否则返回1。 +**参数** -请注意，反斜杠符号（`\`）用于在正则表达式中转义。由于字符串中采用相同的符号来进行转义。因此，为了在正则表达式中转义符号，必须在字符串文字中写入两个反斜杠（\\）。 +- `haystack` — 被检索查询字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — 子字符串数组, 类型为[Array](../../sql-reference/data-types/array.md) -正则表达式与字符串一起使用，就像它是一组字节一样。正则表达式中不能包含空字节。 -对于在字符串中搜索子字符串的模式，最好使用LIKE或«position»，因为它们更加高效。 +**返回值** -## multiMatchAny（大海捞针，\[模式1，模式2, …, patternn\]) {#multimatchanyhaystack-pattern1-pattern2-patternn} +- 位置数组，数组中的每个元素对应于 `needle` 数组中的一个元素。如果在 `haystack` 中找到子字符串，则返回的数组中的元素为子字符串的位置（以字节为单位，从 1 开始）；如果未找到子字符串，则返回的数组中的元素为 0。 -与`match`相同，但如果所有正则表达式都不匹配，则返回0；如果任何模式匹配，则返回1。它使用[超扫描](https://github.com/intel/hyperscan)库。对于在字符串中搜索子字符串的模式，最好使用«multisearchany»，因为它更高效。 +**示例** + +``` sql +SELECT multiSearchAllPositions('Hello, World!', ['hello', '!', 'world']); +``` + +结果： + +``` text +┌─multiSearchAllPositions('Hello, World!', ['hello', '!', 'world'])─┐ +│ [0,13,0] │ +└───────────────────────────────────────────────────────────────────┘ +``` + +## multiSearchAllPositionsUTF8 + +类似于 [multiSearchAllPositions](#multiSearchAllPositions) ,但假定 `haystack` 和 `needle`-s 是 UTF-8 编码的字符串。 + +## multiSearchFirstPosition + +类似于 `position` , 在字符串`haystack`中匹配多个`needle`子字符串，从左开始任一匹配的子串，返回其位置。 + +函数 `multiSearchFirstPositionCaseInsensitive`, `multiSearchFirstPositionUTF8` 和 `multiSearchFirstPositionCaseInsensitiveUTF8` 提供此函数的不区分大小写 以及/或 UTF-8 变体。 + +**语法** + +```sql +multiSearchFirstPosition(haystack, [needle1, needle2, …, needleN]) +``` + +## multiSearchFirstIndex + +在字符串`haystack`中匹配最左侧的 needlei 子字符串，返回其索引 `i` (从1开始)，如无法匹配则返回0。 + +函数 `multiSearchFirstIndexCaseInsensitive`, `multiSearchFirstIndexUTF8` 和 `multiSearchFirstIndexCaseInsensitiveUTF8` 提供此函数的不区分大小写以及/或 UTF-8 变体。 + +**语法** + +```sql +multiSearchFirstIndex(haystack, \[needle1, needle2, …, needlen\]) +``` + +## multiSearchAny {#multisearchany} + +至少已有一个子字符串`needle`匹配 `haystack` 时返回1，否则返回 0 。 + +函数 `multiSearchAnyCaseInsensitive`, `multiSearchAnyUTF8` 和 `multiSearchAnyCaseInsensitiveUTF8` 提供此函数的不区分大小写以及/或 UTF-8 变体。 + + +**语法** + +```sql +multiSearchAny(haystack, [needle1, needle2, …, needleN]) +``` + +## match {#match} + +返回字符串 `haystack` 是否匹配正则表达式 `pattern` （[re2正则语法参考](https://github.com/google/re2/wiki/Syntax) + +匹配基于 UTF-8，例如`.` 匹配 Unicode 代码点 `¥`，它使用两个字节以 UTF-8 表示。T正则表达式不得包含空字节。如果 `haystack` 或`pattern`不是有效的 UTF-8，则此行为为undefined。 +与 re2 的默认行为不同，`.` 会匹配换行符。要禁用此功能，请在模式前面添加`(?-s)`。 + +如果仅希望搜索子字符串，可以使用函数 [like](#like)或 [position](#position) 来替代，这些函数的性能比此函数更高。 + +**语法** + +```sql +match(haystack, pattern) +``` + +别名： `haystack REGEXP pattern operator` + +## multiMatchAny + +类似于 `match`，如果至少有一个表达式 `patterni` 匹配字符串 `haystack`，则返回1，否则返回0。 :::note -任何`haystack`字符串的长度必须小于232\字节，否则抛出异常。这种限制是因为hyperscan API而产生的。 +`multi[Fuzzy]Match*()` 函数家族使用了(Vectorscan)[https://github.com/VectorCamp/vectorscan]库. 因此，只有当 ClickHouse 编译时支持矢量扫描时，它们才会启用。 + +要关闭所有使用矢量扫描(hyperscan)的功能，请使用设置 `SET allow_hyperscan = 0;`。 + +由于Vectorscan的限制，`haystack` 字符串的长度必须小于232字节。 + +Hyperscan 通常容易受到正则表达式拒绝服务 (ReDoS) 攻击。有关更多信息，请参见 +[https://www.usenix.org/conference/usenixsecurity22/presentation/turonova](https://www.usenix.org/conference/usenixsecurity22/presentation/turonova) +[https://doi.org/10.1007/s10664-021-10033-1](https://doi.org/10.1007/s10664-021-10033-1) +[https://doi.org/10.1145/3236024.3236027](https://doi.org/10.1145/3236024.3236027) +建议用户谨慎检查提供的表达式。 + ::: -## multiMatchAnyIndex（大海捞针，\[模式1，模式2, …, patternn\]) {#multimatchanyindexhaystack-pattern1-pattern2-patternn} +如果仅希望搜索子字符串，可以使用函数 [multiSearchAny](#multisearchany) 来替代，这些函数的性能比此函数更高。 -与`multiMatchAny`相同，但返回与haystack匹配的任何内容的索引位置。 +**语法** -## multiFuzzyMatchAny(干草堆,距离,\[模式1，模式2, …, patternn\]) {#multifuzzymatchanyhaystack-distance-pattern1-pattern2-patternn} +```sql +multiMatchAny(haystack, \[pattern1, pattern2, …, patternn\]) +``` -与`multiMatchAny`相同，但如果在haystack能够查找到任何模式匹配能够在指定的[编辑距离](https://en.wikipedia.org/wiki/Edit_distance)内进行匹配，则返回1。此功能也处于实验模式，可能非常慢。有关更多信息，请参阅[hyperscan文档](https://intel.github.io/hyperscan/dev-reference/compilation.html#approximate-matching)。 +## multiMatchAnyIndex -## multiFuzzyMatchAnyIndex(大海捞针,距离,\[模式1，模式2, …, patternn\]) {#multifuzzymatchanyindexhaystack-distance-pattern1-pattern2-patternn} +类似于 `multiMatchAny` ，返回任何子串匹配 `haystack` 的索引。 -与`multiFuzzyMatchAny`相同，但返回匹配项的匹配能容的索引位置。 +**语法** + +```sql +multiMatchAnyIndex(haystack, \[pattern1, pattern2, …, patternn\]) +``` + +## multiMatchAllIndices + +类似于 `multiMatchAny`，返回一个数组，包含所有匹配 `haystack` 的子字符串的索引。 + +**语法** + +```sql +multiMatchAllIndices(haystack, \[pattern1, pattern2, …, patternn\]) +``` + +## multiFuzzyMatchAny + +类似于 `multiMatchAny` ，如果任一 `pattern` 匹配 `haystack`，则返回1 within a constant [edit distance](https://en.wikipedia.org/wiki/Edit_distance). 该功能依赖于实验特征 [hyperscan](https://intel.github.io/hyperscan/dev-reference/compilation.html#approximate-matching) 库，并且对于某些边缘场景可能会很慢。性能取决于编辑距离`distance`的值和使用的`partten`，但与非模糊搜索相比，它的开销总是更高的。 :::note -`multiFuzzyMatch*`函数不支持UTF-8正则表达式，由于hyperscan限制，这些表达式被按字节解析。 +由于 hyperscan 的限制，`multiFuzzyMatch*()` 函数族不支持 UTF-8 正则表达式（hyperscan以一串字节来处理）。 ::: +**语法** + +```sql +multiFuzzyMatchAny(haystack, distance, \[pattern1, pattern2, …, patternn\]) +``` + +## multiFuzzyMatchAnyIndex + +类似于 `multiFuzzyMatchAny` 返回在编辑距离内与`haystack`匹配的任何索引 + +**语法** + +```sql +multiFuzzyMatchAnyIndex(haystack, distance, \[pattern1, pattern2, …, patternn\]) +``` + +## multiFuzzyMatchAllIndices + +类似于 `multiFuzzyMatchAny` 返回在编辑距离内与`haystack`匹配的所有索引的数组。 + +**语法** + +```sql +multiFuzzyMatchAllIndices(haystack, distance, \[pattern1, pattern2, …, patternn\]) +``` + +## extract + +使用正则表达式提取字符串。如果字符串 `haystack` 不匹配正则表达式 `pattern` ，则返回空字符串。 + +对于没有子模式的正则表达式，该函数使用与整个正则表达式匹配的片段。否则，它使用与第一个子模式匹配的片段。 + +**语法** + +```sql +extract(haystack, pattern) +``` + +## extractAll + +使用正则表达式提取字符串的所有片段。如果字符串 `haystack` 不匹配正则表达式 `pattern` ，则返回空字符串。 + +返回所有匹配项组成的字符串数组。 + +子模式的行为与函数`extract`中的行为相同。 + +**语法** + +```sql +extractAll(haystack, pattern) +``` + +## extractAllGroupsHorizontal + +使用`pattern`正则表达式匹配`haystack`字符串的所有组。 + +返回一个元素为数组的数组，其中第一个数组包含与第一组匹配的所有片段，第二个数组包含与第二组匹配的所有片段，依此类推。 + +这个函数相比 [extractAllGroupsVertical](#extractallgroups-vertical)更慢。 + +**语法** + +``` sql +extractAllGroupsHorizontal(haystack, pattern) +``` + +**参数** + +- `haystack` — 输入的字符串，数据类型为[String](../../sql-reference/data-types/string.md). +- `pattern` — 正则表达式（[re2正则语法参考](https://github.com/google/re2/wiki/Syntax) ，必须包含 group，每个 group 用括号括起来。 如果 `pattern` 不包含 group 则会抛出异常。 数据类型为[String](../../sql-reference/data-types/string.md). + +**返回值** + +- 数据类型: [Array](../../sql-reference/data-types/array.md). + +如果`haystack`不匹配`pattern`正则表达式，则返回一个空数组的数组。 + +**示例** + +``` sql +SELECT extractAllGroupsHorizontal('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)'); +``` + +结果： + +``` text +┌─extractAllGroupsHorizontal('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)')─┐ +│ [['abc','def','ghi'],['111','222','333']] │ +└──────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +## extractAllGroupsVertical + +使用正则表达式 `pattern`匹配字符串`haystack`中的所有group。返回一个数组，其中每个数组包含每个group的匹配片段。片段按照在`haystack`中出现的顺序进行分组。 + +**语法** + +``` sql +extractAllGroupsVertical(haystack, pattern) +``` + +**参数** + +- `haystack` — 输入的字符串，数据类型为[String](../../sql-reference/data-types/string.md). +- `pattern` — 正则表达式（[re2正则语法参考](https://github.com/google/re2/wiki/Syntax) ，必须包含group，每个group用括号括起来。 如果 `pattern` 不包含group则会抛出异常。 数据类型为[String](../../sql-reference/data-types/string.md). + +**返回值** + +- 数据类型: [Array](../../sql-reference/data-types/array.md). + +如果`haystack`不匹配`pattern`正则表达式，则返回一个空数组。 + +**示例** + +``` sql +SELECT extractAllGroupsVertical('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)'); +``` + +结果： + +``` text +┌─extractAllGroupsVertical('abc=111, def=222, ghi=333', '("[^"]+"|\\w+)=("[^"]+"|\\w+)')─┐ +│ [['abc','111'],['def','222'],['ghi','333']] │ +└────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +## like {#like} + +返回字符串 `haystack` 是否匹配 LIKE 表达式 `pattern`。 + +一个 LIKE 表达式可以包含普通字符和以下元字符： + +- `%` 表示任意数量的任意字符（包括零个字符）。 +- `_` 表示单个任意字符。 +- `\` 用于转义文字 `%`, `_` 和 `\`。 + +匹配基于 UTF-8，例如 `_` 匹配 Unicode 代码点 `¥`，它使用两个字节以 UTF-8 表示。 + +如果 `haystack` 或 `LIKE` 表达式不是有效的 UTF-8，则行为是未定义的。 + +不会自动执行 Unicode 规范化，您可以使用[normalizeUTF8*()](https://clickhouse.com/docs/zh/sql-reference/functions/string-functions/) 函数来执行此操作。 + +如果需要匹配字符 `%`, `_` 和 `/`（这些是 LIKE 元字符），请在其前面加上反斜杠：`\%`, `\_` 和 `\\`。 +如果在非 `%`, `_` 或 `\` 字符前使用反斜杠，则反斜杠将失去其特殊含义（即被解释为字面值）。 +请注意，ClickHouse 要求字符串中使用反斜杠 [也需要被转义](../syntax.md#string), 因此您实际上需要编写 `\\%`、`\\_` 和 `\\\\`。 + + +对于形式为 `%needle%` 的 LIKE 表达式，函数的性能与 `position` 函数相同。 +所有其他 LIKE 表达式都会被内部转换为正则表达式，并以与函数 `match` 相似的性能执行。 + +**语法** + +```sql +like(haystack, pattern) +``` + +别名： `haystack LIKE pattern` (operator) + +## notLike {#notlike} + +类似于 `like` 但是返回相反的结果。 + +别名： `haystack NOT LIKE pattern` (operator) + +## ilike + +类似于 `like` 但是不区分大小写。 + +别名： `haystack ILIKE pattern` (operator) + +## notILike + +类似于 `ilike` 但是返回相反的结果。 + +别名： `haystack NOT ILIKE pattern` (operator) + +## ngramDistance + +计算字符串 `haystack` 和子字符串 `needle` 的 4-gram 距离。 为此，它计算两个 4-gram 多重集之间的对称差异，并通过它们的基数之和对其进行标准化。返回 0 到 1 之间的 Float32 浮点数。返回值越小，代表字符串越相似. 如果参数 `needle` or `haystack` 是常数且大小超过 32Kb，则抛出异常。如果参数 `haystack` 或 `needle` 是非常数且大小超过 32Kb ，则返回值恒为 1。 + +函数 `ngramDistanceCaseInsensitive, ngramDistanceUTF8, ngramDistanceCaseInsensitiveUTF8` 提供此函数的不区分大小写以及/或 UTF-8 变体。 + +**语法** + +```sql +ngramDistance(haystack, needle) +``` + +## ngramSearch + +类似于`ngramDistance`，但计算`needle`字符串和 `haystack` 字符串之间的非对称差异，即来自 `needle` 的 n-gram 数量减去由`needle`数量归一化的 n-gram 的公共数量 n-gram。返回 0 到 1 之间的 Float32 浮点数。结果越大，`needle` 越有可能在 `haystack` 中。该函数对于模糊字符串搜索很有用。另请参阅函数 `soundex``。 + +函数 `ngramSearchCaseInsensitive, ngramSearchUTF8, ngramSearchCaseInsensitiveUTF8` 提供此函数的不区分大小写以及/或 UTF-8 变体。 + :::note -如要关闭所有hyperscan函数的使用，请设置`SET allow_hyperscan = 0;`。 +UTF-8 变体使用了 3-gram 距离。这些并不是完全公平的 n-gram 距离。我们使用 2 字节的哈希函数来哈希 n-gram，然后计算这些哈希表之间的(非)对称差异——可能会发生冲突。在使用 UTF-8 大小写不敏感格式时，我们并不使用公平的 `tolower` 函数——我们将每个码点字节的第 5 位（从零开始）和第零字节的第一个比特位位置为零（如果该串的大小超过一个字节）——这对拉丁字母和大部分西里尔字母都有效。 ::: -## 提取（大海捞针，图案) {#extracthaystack-pattern} +**语法** -使用正则表达式截取字符串。如果’haystack’与’pattern’不匹配，则返回空字符串。如果正则表达式中不包含子模式，它将获取与整个正则表达式匹配的子串。否则，它将获取与第一个子模式匹配的子串。 +```sql +ngramSearch(haystack, needle) +``` -## extractAll（大海捞针，图案) {#extractallhaystack-pattern} +## countSubstrings -使用正则表达式提取字符串的所有片段。如果’haystack’与’pattern’正则表达式不匹配，则返回一个空字符串。否则返回所有与正则表达式匹配的字符串数组。通常，行为与’extract’函数相同（它采用第一个子模式，如果没有子模式，则采用整个表达式）。 +返回字符串 `haystack` 中子字符串 `needle` 出现的次数。 -## 像（干草堆，模式），干草堆像模式运算符 {#likehaystack-pattern-haystack-like-pattern-operator} +函数 `countSubstringsCaseInsensitive` 和 `countSubstringsCaseInsensitiveUTF8` 提供此函数的不区分大小写以及 UTF-8 变体。 -检查字符串是否与简单正则表达式匹配。 -正则表达式可以包含的元符号有`％`和`_`。 +**语法** -`%` 表示任何字节数（包括零字符）。 +``` sql +countSubstrings(haystack, needle[, start_pos]) +``` -`_` 表示任何一个字节。 +**参数** -可以使用反斜杠（`\`）来对元符号进行转义。请参阅«match»函数说明中有关转义的说明。 +- `haystack` — 被搜索的字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — 用于搜索的模式子字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `start_pos` – 在字符串`haystack` 中开始检索的位置(从 1 开始)，类型为[UInt](../../sql-reference/data-types/int-uint.md)，可选。 -对于像`％needle％`这样的正则表达式，改函数与`position`函数一样快。 -对于其他正则表达式，函数与’match’函数相同。 +**返回值** -## 不喜欢（干草堆，模式），干草堆不喜欢模式运算符 {#notlikehaystack-pattern-haystack-not-like-pattern-operator} +- 子字符串出现的次数。 -与’like’函数返回相反的结果。 +数据类型: [UInt64](../../sql-reference/data-types/int-uint.md). -## 大海捞针) {#ngramdistancehaystack-needle} +**示例** -基于4-gram计算`haystack`和`needle`之间的距离：计算两个4-gram集合之间的对称差异，并用它们的基数和对其进行归一化。返回0到1之间的任何浮点数 – 越接近0则表示越多的字符串彼此相似。如果常量的`needle`或`haystack`超过32KB，函数将抛出异常。如果非常量的`haystack`或`needle`字符串超过32Kb，则距离始终为1。 +``` sql +SELECT countSubstrings('aaaa', 'aa'); +``` -对于不区分大小写的搜索或/和UTF-8格式，使用函数`ngramDistanceCaseInsensitive，ngramDistanceUTF8，ngramDistanceCaseInsensitiveUTF8`。 +结果： -## ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人) {#ngramsearchhaystack-needle} +``` text +┌─countSubstrings('aaaa', 'aa')─┐ +│ 2 │ +└───────────────────────────────┘ +``` -与`ngramDistance`相同，但计算`needle`和`haystack`之间的非对称差异——`needle`的n-gram减去`needle`归一化n-gram。可用于模糊字符串搜索。 +示例，使用参数 `start_pos` : -对于不区分大小写的搜索或/和UTF-8格式，使用函数`ngramSearchCaseInsensitive，ngramSearchUTF8，ngramSearchCaseInsensitiveUTF8`。 +```sql +SELECT countSubstrings('abc___abc', 'abc', 4); +``` -:::note -对于UTF-8，我们使用3-gram。所有这些都不是完全公平的n-gram距离。我们使用2字节哈希来散列n-gram，然后计算这些哈希表之间的（非）对称差异 - 可能会发生冲突。对于UTF-8不区分大小写的格式，我们不使用公平的`tolower`函数 - 我们将每个Unicode字符字节的第5位（从零开始）和字节的第一位归零 - 这适用于拉丁语，主要用于所有西里尔字母。 -::: +结果： + +``` text +┌─countSubstrings('abc___abc', 'abc', 4)─┐ +│ 1 │ +└────────────────────────────────────────┘ +``` + +## countMatches + +返回正则表达式 `pattern` 在 `haystack` 中成功匹配的次数。 + +**语法** + +``` sql +countMatches(haystack, pattern) +``` + +**参数** + +- `haystack` — 输入的字符串，数据类型为[String](../../sql-reference/data-types/string.md). +- `pattern` — 正则表达式（[re2正则语法参考](https://github.com/google/re2/wiki/Syntax)） 数据类型为[String](../../sql-reference/data-types/string.md). + +**返回值** + +- 匹配次数。 + +数据类型: [UInt64](../../sql-reference/data-types/int-uint.md). + +**示例** + +``` sql +SELECT countMatches('foobar.com', 'o+'); +``` + +结果： + +``` text +┌─countMatches('foobar.com', 'o+')─┐ +│ 2 │ +└──────────────────────────────────┘ +``` + +``` sql +SELECT countMatches('aaaa', 'aa'); +``` + +结果： + +``` text +┌─countMatches('aaaa', 'aa')────┐ +│ 2 │ +└───────────────────────────────┘ +``` + +## countMatchesCaseInsensitive + +类似于 `countMatches(haystack, pattern)` 但是不区分大小写。 + +## regexpExtract + +提取匹配正则表达式模式的字符串`haystack`中的第一个字符串，并对应于正则表达式组索引。 + +**语法** + +``` sql +regexpExtract(haystack, pattern[, index]) +``` + +别名： `REGEXP_EXTRACT(haystack, pattern[, index])`. + +**参数** + +- `haystack` — 被匹配字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `pattern` — 正则表达式，必须是常量。类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `index` – 一个大于等于 0 的整数，默认为 1 ，它代表要提取哪个正则表达式组。 [UInt or Int](../../sql-reference/data-types/int-uint.md) 可选。 + +**返回值** + +`pattern`可以包含多个正则组, `index` 代表要提取哪个正则表达式组。如果 `index` 为 0，则返回整个匹配的字符串。 + +数据类型: `String`. + +**示例** + +``` sql +SELECT + regexpExtract('100-200', '(\\d+)-(\\d+)', 1), + regexpExtract('100-200', '(\\d+)-(\\d+)', 2), + regexpExtract('100-200', '(\\d+)-(\\d+)', 0), + regexpExtract('100-200', '(\\d+)-(\\d+)'); +``` + +结果： + +``` text +┌─regexpExtract('100-200', '(\\d+)-(\\d+)', 1)─┬─regexpExtract('100-200', '(\\d+)-(\\d+)', 2)─┬─regexpExtract('100-200', '(\\d+)-(\\d+)', 0)─┬─regexpExtract('100-200', '(\\d+)-(\\d+)')─┐ +│ 100 │ 200 │ 100-200 │ 100 │ +└──────────────────────────────────────────────┴──────────────────────────────────────────────┴──────────────────────────────────────────────┴───────────────────────────────────────────┘ +``` + +## hasSubsequence + +如果`needle`是`haystack`的子序列，返回1，否则返回0。 +子序列是从给定字符串中删除零个或多个元素而不改变剩余元素的顺序得到的序列。 + +**语法** + +``` sql +hasSubsequence(haystack, needle) +``` + +**参数** + +- `haystack` — 被搜索的字符串，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). +- `needle` — 搜索子序列，类型为[String](../../sql-reference/syntax.md#syntax-string-literal). + +**返回值** + +- 1, 如果`needle`是`haystack`的子序列 +- 0, 如果`needle`不是`haystack`的子序列 + +数据类型: `UInt8`. + +**示例** + +``` sql +SELECT hasSubsequence('garbage', 'arg') ; +``` + +结果： + +``` text +┌─hasSubsequence('garbage', 'arg')─┐ +│ 1 │ +└──────────────────────────────────┘ +``` + +## hasSubsequenceCaseInsensitive +类似于 [hasSubsequence](#hasSubsequence) 但是不区分大小写。 + +## hasSubsequenceUTF8 + +类似于 [hasSubsequence](#hasSubsequence) 但是假定 `haystack` 和 `needle` 是 UTF-8 编码的字符串。 + +## hasSubsequenceCaseInsensitiveUTF8 + +类似于 [hasSubsequenceUTF8](#hasSubsequenceUTF8) 但是不区分大小写。 diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index d4bf2f686c8..192f9e61891 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -17,12 +17,13 @@ #include -#include -#include -#include -#include #include #include +#include +#include +#include +#include +#include #include #include diff --git a/rust/CMakeLists.txt b/rust/CMakeLists.txt index 6715a54221a..cebfd36a24a 100644 --- a/rust/CMakeLists.txt +++ b/rust/CMakeLists.txt @@ -99,6 +99,19 @@ function(add_rust_subdirectory src) message(STATUS "Copy ${src} to ${dst}") file(COPY "${src}" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}" PATTERN target EXCLUDE) + + # Check is Rust available or not. + # + # `cargo update --dry-run` will not update anything, but will check the internet connectivity. + execute_process(COMMAND ${Rust_CARGO_CACHED} update --dry-run + WORKING_DIRECTORY "${dst}" + RESULT_VARIABLE CARGO_UPDATE_RESULT + OUTPUT_VARIABLE CARGO_UPDATE_STDOUT + ERROR_VARIABLE CARGO_UPDATE_STDERR) + if (CARGO_UPDATE_RESULT) + message(FATAL_ERROR "Rust (${Rust_CARGO_CACHED}) support is not available (likely there is no internet connectivity):\n${CARGO_UPDATE_STDERR}\nYou can disable Rust support with -DENABLE_RUST=OFF") + endif() + add_subdirectory("${dst}" "${dst}") # cmake -E copy* do now know how to exclude files diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index 36a68bc0a34..a87e9361e8e 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -1,6 +1,8 @@ #include -#include #include +#include +#include + #include #include #include diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index 6095f8ce6d3..83b50ce96c3 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -205,7 +205,7 @@ enum class AccessType M(SYSTEM_FLUSH, "", GROUP, SYSTEM) \ M(SYSTEM_THREAD_FUZZER, "SYSTEM START THREAD FUZZER, SYSTEM STOP THREAD FUZZER, START THREAD FUZZER, STOP THREAD FUZZER", GLOBAL, SYSTEM) \ M(SYSTEM_UNFREEZE, "SYSTEM UNFREEZE", GLOBAL, SYSTEM) \ - M(SYSTEM_FAILPOINT, "SYSTEM ENABLE FAILPOINT, SYSTEM DISABLE FAILPOINT", GLOBAL, SYSTEM) \ + M(SYSTEM_FAILPOINT, "SYSTEM ENABLE FAILPOINT, SYSTEM DISABLE FAILPOINT, SYSTEM WAIT FAILPOINT", GLOBAL, SYSTEM) \ M(SYSTEM_LISTEN, "SYSTEM START LISTEN, SYSTEM STOP LISTEN", GLOBAL, SYSTEM) \ M(SYSTEM_JEMALLOC, "SYSTEM JEMALLOC PURGE, SYSTEM JEMALLOC ENABLE PROFILE, SYSTEM JEMALLOC DISABLE PROFILE, SYSTEM JEMALLOC FLUSH PROFILE", GLOBAL, SYSTEM) \ M(SYSTEM, "", GROUP, ALL) /* allows to execute SYSTEM {SHUTDOWN|RELOAD CONFIG|...} */ \ diff --git a/src/AggregateFunctions/AggregateFunctionFactory.cpp b/src/AggregateFunctions/AggregateFunctionFactory.cpp index 18edb7c8ce0..6555ae63128 100644 --- a/src/AggregateFunctions/AggregateFunctionFactory.cpp +++ b/src/AggregateFunctions/AggregateFunctionFactory.cpp @@ -1,11 +1,11 @@ #include #include - #include #include #include #include #include +#include static constexpr size_t MAX_AGGREGATE_FUNCTION_NAME_LENGTH = 1000; diff --git a/src/AggregateFunctions/AggregateFunctionUniq.h b/src/AggregateFunctions/AggregateFunctionUniq.h index 891f2ac4284..cef23f766c7 100644 --- a/src/AggregateFunctions/AggregateFunctionUniq.h +++ b/src/AggregateFunctions/AggregateFunctionUniq.h @@ -457,9 +457,9 @@ public: detail::Adder::add(this->data(place), columns, num_args, row_begin, row_end, flags, null_map); } - bool isParallelizeMergePrepareNeeded() const override { return is_parallelize_merge_prepare_needed;} + bool isParallelizeMergePrepareNeeded() const override { return is_parallelize_merge_prepare_needed; } - void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool) const override + void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic & is_cancelled) const override { if constexpr (is_parallelize_merge_prepare_needed) { @@ -469,7 +469,7 @@ public: for (size_t i = 0; i < data_vec.size(); ++i) data_vec[i] = &this->data(places[i]).set; - DataSet::parallelizeMergePrepare(data_vec, thread_pool); + DataSet::parallelizeMergePrepare(data_vec, thread_pool, is_cancelled); } else { @@ -485,10 +485,10 @@ public: bool isAbleToParallelizeMerge() const override { return is_able_to_parallelize_merge; } bool canOptimizeEqualKeysRanges() const override { return !is_able_to_parallelize_merge; } - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, Arena *) const override + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic & is_cancelled, Arena *) const override { if constexpr (is_able_to_parallelize_merge) - this->data(place).set.merge(this->data(rhs).set, &thread_pool); + this->data(place).set.merge(this->data(rhs).set, &thread_pool, &is_cancelled); else this->data(place).set.merge(this->data(rhs).set); } @@ -579,10 +579,10 @@ public: bool isAbleToParallelizeMerge() const override { return is_able_to_parallelize_merge; } bool canOptimizeEqualKeysRanges() const override { return !is_able_to_parallelize_merge; } - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, Arena *) const override + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic & is_cancelled, Arena *) const override { if constexpr (is_able_to_parallelize_merge) - this->data(place).set.merge(this->data(rhs).set, &thread_pool); + this->data(place).set.merge(this->data(rhs).set, &thread_pool, &is_cancelled); else this->data(place).set.merge(this->data(rhs).set); } diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionArray.h b/src/AggregateFunctions/Combinators/AggregateFunctionArray.h index 6b918926d0d..1940985f8e3 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionArray.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionArray.h @@ -144,9 +144,14 @@ public: bool isAbleToParallelizeMerge() const override { return nested_func->isAbleToParallelizeMerge(); } bool canOptimizeEqualKeysRanges() const override { return nested_func->canOptimizeEqualKeysRanges(); } - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, Arena * arena) const override + void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic & is_cancelled) const override { - nested_func->merge(place, rhs, thread_pool, arena); + nested_func->parallelizeMergePrepare(places, thread_pool, is_cancelled); + } + + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic & is_cancelled, Arena * arena) const override + { + nested_func->merge(place, rhs, thread_pool, is_cancelled, arena); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional version) const override diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionIf.h b/src/AggregateFunctions/Combinators/AggregateFunctionIf.h index df23398a10d..a893fc91780 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionIf.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionIf.h @@ -167,9 +167,14 @@ public: bool isAbleToParallelizeMerge() const override { return nested_func->isAbleToParallelizeMerge(); } bool canOptimizeEqualKeysRanges() const override { return nested_func->canOptimizeEqualKeysRanges(); } - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, Arena * arena) const override + void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic & is_cancelled) const override { - nested_func->merge(place, rhs, thread_pool, arena); + nested_func->parallelizeMergePrepare(places, thread_pool, is_cancelled); + } + + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic & is_cancelled, Arena * arena) const override + { + nested_func->merge(place, rhs, thread_pool, is_cancelled, arena); } void mergeBatch( diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h b/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h index 53c24bd60c1..4a39ec0ab87 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h @@ -113,9 +113,14 @@ public: bool isAbleToParallelizeMerge() const override { return nested_func->isAbleToParallelizeMerge(); } bool canOptimizeEqualKeysRanges() const override { return nested_func->canOptimizeEqualKeysRanges(); } - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, Arena * arena) const override + void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic & is_cancelled) const override { - nested_func->merge(place, rhs, thread_pool, arena); + nested_func->parallelizeMergePrepare(places, thread_pool, is_cancelled); + } + + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic & is_cancelled, Arena * arena) const override + { + nested_func->merge(place, rhs, thread_pool, is_cancelled, arena); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional version) const override diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionNull.h b/src/AggregateFunctions/Combinators/AggregateFunctionNull.h index 72ab3cf5acb..306e293cae7 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionNull.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionNull.h @@ -154,9 +154,18 @@ public: bool isAbleToParallelizeMerge() const override { return nested_function->isAbleToParallelizeMerge(); } bool canOptimizeEqualKeysRanges() const override { return nested_function->canOptimizeEqualKeysRanges(); } - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, Arena * arena) const override + void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic & is_cancelled) const override { - nested_function->merge(nestedPlace(place), nestedPlace(rhs), thread_pool, arena); + AggregateDataPtrs nested_places(places.begin(), places.end()); + for (auto & nested_place : nested_places) + nested_place = nestedPlace(nested_place); + + nested_function->parallelizeMergePrepare(nested_places, thread_pool, is_cancelled); + } + + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic & is_cancelled, Arena * arena) const override + { + nested_function->merge(nestedPlace(place), nestedPlace(rhs), thread_pool, is_cancelled, arena); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional version) const override diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionState.h b/src/AggregateFunctions/Combinators/AggregateFunctionState.h index b0ab6d49604..5f2eb647c92 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionState.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionState.h @@ -94,9 +94,14 @@ public: bool isAbleToParallelizeMerge() const override { return nested_func->isAbleToParallelizeMerge(); } bool canOptimizeEqualKeysRanges() const override { return nested_func->canOptimizeEqualKeysRanges(); } - void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, Arena * arena) const override + void parallelizeMergePrepare(AggregateDataPtrs & places, ThreadPool & thread_pool, std::atomic & is_cancelled) const override { - nested_func->merge(place, rhs, thread_pool, arena); + nested_func->parallelizeMergePrepare(places, thread_pool, is_cancelled); + } + + void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, ThreadPool & thread_pool, std::atomic & is_cancelled, Arena * arena) const override + { + nested_func->merge(place, rhs, thread_pool, is_cancelled, arena); } void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional version) const override diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index 499185320e6..97e0e89aee9 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -151,7 +151,7 @@ public: virtual bool isParallelizeMergePrepareNeeded() const { return false; } - virtual void parallelizeMergePrepare(AggregateDataPtrs & /*places*/, ThreadPool & /*thread_pool*/) const + virtual void parallelizeMergePrepare(AggregateDataPtrs & /*places*/, ThreadPool & /*thread_pool*/, std::atomic & /*is_cancelled*/) const { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "parallelizeMergePrepare() with thread pool parameter isn't implemented for {} ", getName()); } @@ -168,7 +168,7 @@ public: /// Should be used only if isAbleToParallelizeMerge() returned true. virtual void - merge(AggregateDataPtr __restrict /*place*/, ConstAggregateDataPtr /*rhs*/, ThreadPool & /*thread_pool*/, Arena * /*arena*/) const + merge(AggregateDataPtr __restrict /*place*/, ConstAggregateDataPtr /*rhs*/, ThreadPool & /*thread_pool*/, std::atomic & /*is_cancelled*/, Arena * /*arena*/) const { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "merge() with thread pool parameter isn't implemented for {} ", getName()); } diff --git a/src/AggregateFunctions/UniqExactSet.h b/src/AggregateFunctions/UniqExactSet.h index 131c59b9ed6..2ae8c3a8386 100644 --- a/src/AggregateFunctions/UniqExactSet.h +++ b/src/AggregateFunctions/UniqExactSet.h @@ -37,7 +37,7 @@ public: /// In merge, if one of the lhs and rhs is twolevelset and the other is singlelevelset, then the singlelevelset will need to convertToTwoLevel(). /// It's not in parallel and will cost extra large time if the thread_num is large. /// This method will convert all the SingleLevelSet to TwoLevelSet in parallel if the hashsets are not all singlelevel or not all twolevel. - static void parallelizeMergePrepare(const std::vector & data_vec, ThreadPool & thread_pool) + static void parallelizeMergePrepare(const std::vector & data_vec, ThreadPool & thread_pool, std::atomic & is_cancelled) { UInt64 single_level_set_num = 0; UInt64 all_single_hash_size = 0; @@ -63,7 +63,7 @@ public: try { auto data_vec_atomic_index = std::make_shared(0); - auto thread_func = [data_vec, data_vec_atomic_index, thread_group = CurrentThread::getGroup()]() + auto thread_func = [data_vec, data_vec_atomic_index, &is_cancelled, thread_group = CurrentThread::getGroup()]() { SCOPE_EXIT_SAFE( if (thread_group) @@ -76,6 +76,9 @@ public: while (true) { + if (is_cancelled.load(std::memory_order_seq_cst)) + return; + const auto i = data_vec_atomic_index->fetch_add(1); if (i >= data_vec.size()) return; @@ -96,7 +99,7 @@ public: } } - auto merge(const UniqExactSet & other, ThreadPool * thread_pool = nullptr) + auto merge(const UniqExactSet & other, ThreadPool * thread_pool = nullptr, std::atomic * is_cancelled = nullptr) { if (isSingleLevel() && other.isTwoLevel()) convertToTwoLevel(); @@ -113,7 +116,9 @@ public: if (!thread_pool) { for (size_t i = 0; i < rhs.NUM_BUCKETS; ++i) + { lhs.impls[i].merge(rhs.impls[i]); + } } else { @@ -121,7 +126,7 @@ public: { auto next_bucket_to_merge = std::make_shared(0); - auto thread_func = [&lhs, &rhs, next_bucket_to_merge, thread_group = CurrentThread::getGroup()]() + auto thread_func = [&lhs, &rhs, next_bucket_to_merge, is_cancelled, thread_group = CurrentThread::getGroup()]() { SCOPE_EXIT_SAFE( if (thread_group) @@ -133,6 +138,9 @@ public: while (true) { + if (is_cancelled->load(std::memory_order_seq_cst)) + return; + const auto bucket = next_bucket_to_merge->fetch_add(1); if (bucket >= rhs.NUM_BUCKETS) return; diff --git a/src/Analyzer/ConstantNode.h b/src/Analyzer/ConstantNode.h index b065853e315..98a8eb78277 100644 --- a/src/Analyzer/ConstantNode.h +++ b/src/Analyzer/ConstantNode.h @@ -4,6 +4,7 @@ #include #include +#include namespace DB { @@ -86,6 +87,11 @@ public: mask_id = id; } + void convertToNullable() override + { + constant_value = std::make_shared(constant_value->getValue(), makeNullableSafe(constant_value->getType())); + } + void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override; protected: diff --git a/src/Analyzer/HashUtils.h b/src/Analyzer/HashUtils.h index 3727ea1ea14..eb6aac88fe9 100644 --- a/src/Analyzer/HashUtils.h +++ b/src/Analyzer/HashUtils.h @@ -11,35 +11,37 @@ namespace DB * Example of usage: * std::unordered_map map; */ -template +template struct QueryTreeNodeWithHash { QueryTreeNodeWithHash(QueryTreeNodePtrType node_) /// NOLINT : node(std::move(node_)) - , hash(node->getTreeHash()) + , hash(node->getTreeHash({.compare_aliases = compare_aliases})) {} QueryTreeNodePtrType node = nullptr; CityHash_v1_0_2::uint128 hash; }; -template -inline bool operator==(const QueryTreeNodeWithHash & lhs, const QueryTreeNodeWithHash & rhs) +template +inline bool operator==(const QueryTreeNodeWithHash & lhs, const QueryTreeNodeWithHash & rhs) { - return lhs.hash == rhs.hash && lhs.node->isEqual(*rhs.node); + return lhs.hash == rhs.hash && lhs.node->isEqual(*rhs.node, {.compare_aliases = compare_aliases}); } -template -inline bool operator!=(const QueryTreeNodeWithHash & lhs, const QueryTreeNodeWithHash & rhs) +template +inline bool operator!=(const QueryTreeNodeWithHash & lhs, const QueryTreeNodeWithHash & rhs) { return !(lhs == rhs); } using QueryTreeNodePtrWithHash = QueryTreeNodeWithHash; +using QueryTreeNodePtrWithHashWithoutAlias = QueryTreeNodeWithHash; using QueryTreeNodeRawPtrWithHash = QueryTreeNodeWithHash; using QueryTreeNodeConstRawPtrWithHash = QueryTreeNodeWithHash; using QueryTreeNodePtrWithHashSet = std::unordered_set; +using QueryTreeNodePtrWithHashWithoutAliasSet = std::unordered_set; using QueryTreeNodeConstRawPtrWithHashSet = std::unordered_set; template @@ -50,10 +52,10 @@ using QueryTreeNodeConstRawPtrWithHashMap = std::unordered_map -struct std::hash> +template +struct std::hash> { - size_t operator()(const DB::QueryTreeNodeWithHash & node_with_hash) const + size_t operator()(const DB::QueryTreeNodeWithHash & node_with_hash) const { return node_with_hash.hash.low64; } diff --git a/src/Analyzer/IQueryTreeNode.cpp b/src/Analyzer/IQueryTreeNode.cpp index d61cb0ffab1..7815b93c3ac 100644 --- a/src/Analyzer/IQueryTreeNode.cpp +++ b/src/Analyzer/IQueryTreeNode.cpp @@ -164,7 +164,7 @@ bool IQueryTreeNode::isEqual(const IQueryTreeNode & rhs, CompareOptions compare_ return true; } -IQueryTreeNode::Hash IQueryTreeNode::getTreeHash() const +IQueryTreeNode::Hash IQueryTreeNode::getTreeHash(CompareOptions compare_options) const { /** Compute tree hash with this node as root. * @@ -201,7 +201,7 @@ IQueryTreeNode::Hash IQueryTreeNode::getTreeHash() const } hash_state.update(static_cast(node_to_process->getNodeType())); - if (!node_to_process->alias.empty()) + if (compare_options.compare_aliases && !node_to_process->alias.empty()) { hash_state.update(node_to_process->alias.size()); hash_state.update(node_to_process->alias); diff --git a/src/Analyzer/IQueryTreeNode.h b/src/Analyzer/IQueryTreeNode.h index b07aa2d31b0..92e34616c4d 100644 --- a/src/Analyzer/IQueryTreeNode.h +++ b/src/Analyzer/IQueryTreeNode.h @@ -114,7 +114,7 @@ public: * Alias of query tree node is part of query tree hash. * Original AST is not part of query tree hash. */ - Hash getTreeHash() const; + Hash getTreeHash(CompareOptions compare_options = { .compare_aliases = true }) const; /// Get a deep copy of the query tree QueryTreeNodePtr clone() const; diff --git a/src/Analyzer/Passes/ConvertInToEqualPass.cpp b/src/Analyzer/Passes/ConvertInToEqualPass.cpp deleted file mode 100644 index 66a37fea5bd..00000000000 --- a/src/Analyzer/Passes/ConvertInToEqualPass.cpp +++ /dev/null @@ -1,73 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -namespace DB -{ - -class ConvertInToEqualPassVisitor : public InDepthQueryTreeVisitorWithContext -{ -public: - using Base = InDepthQueryTreeVisitorWithContext; - using Base::Base; - - void enterImpl(QueryTreeNodePtr & node) - { - static const std::unordered_map MAPPING = { - {"in", "equals"}, - {"notIn", "notEquals"} - }; - auto * func_node = node->as(); - if (!func_node - || !MAPPING.contains(func_node->getFunctionName()) - || func_node->getArguments().getNodes().size() != 2) - return ; - auto args = func_node->getArguments().getNodes(); - auto * column_node = args[0]->as(); - auto * constant_node = args[1]->as(); - if (!column_node || !constant_node) - return ; - // IN multiple values is not supported - if (constant_node->getValue().getType() == Field::Types::Which::Tuple - || constant_node->getValue().getType() == Field::Types::Which::Array) - return ; - // x IN null not equivalent to x = null - if (constant_node->getValue().isNull()) - return ; - auto result_func_name = MAPPING.at(func_node->getFunctionName()); - auto equal = std::make_shared(result_func_name); - QueryTreeNodes arguments{column_node->clone(), constant_node->clone()}; - equal->getArguments().getNodes() = std::move(arguments); - FunctionOverloadResolverPtr resolver; - bool decimal_check_overflow = getContext()->getSettingsRef().decimal_check_overflow; - if (result_func_name == "equals") - { - resolver = createInternalFunctionEqualOverloadResolver(decimal_check_overflow); - } - else - { - resolver = createInternalFunctionNotEqualOverloadResolver(decimal_check_overflow); - } - try - { - equal->resolveAsFunction(resolver); - } - catch (...) - { - // When function resolver fails, we should not replace the function node - return; - } - node = equal; - } -}; - -void ConvertInToEqualPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context) -{ - ConvertInToEqualPassVisitor visitor(std::move(context)); - visitor.visit(query_tree_node); -} -} diff --git a/src/Analyzer/Passes/ConvertInToEqualPass.h b/src/Analyzer/Passes/ConvertInToEqualPass.h deleted file mode 100644 index bd4f8607c88..00000000000 --- a/src/Analyzer/Passes/ConvertInToEqualPass.h +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once - -#include - -namespace DB -{ -/** Optimize `in` to `equals` if possible. - * 1. convert in single value to equal - * Example: SELECT * from test where x IN (1); - * Result: SELECT * from test where x = 1; - * - * 2. convert not in single value to notEqual - * Example: SELECT * from test where x NOT IN (1); - * Result: SELECT * from test where x != 1; - * - * If value is null or tuple, do not convert. - */ -class ConvertInToEqualPass final : public IQueryTreePass -{ -public: - String getName() override { return "ConvertInToEqualPass"; } - - String getDescription() override { return "Convert in to equal"; } - - void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override; -}; -} diff --git a/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp b/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp index cf0fb824b4d..085519c7220 100644 --- a/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp +++ b/src/Analyzer/Passes/GroupingFunctionsResolvePass.cpp @@ -146,7 +146,7 @@ void resolveGroupingFunctions(QueryTreeNodePtr & query_node, ContextPtr context) if (query_node_typed.hasGroupBy()) { /// It is expected by execution layer that if there are only 1 grouping set it will be removed - if (query_node_typed.isGroupByWithGroupingSets() && query_node_typed.getGroupBy().getNodes().size() == 1) + if (query_node_typed.isGroupByWithGroupingSets() && query_node_typed.getGroupBy().getNodes().size() == 1 && !context->getSettingsRef().group_by_use_nulls) { auto grouping_set_list_node = query_node_typed.getGroupBy().getNodes().front(); auto & grouping_set_list_node_typed = grouping_set_list_node->as(); diff --git a/src/Analyzer/Passes/IfChainToMultiIfPass.cpp b/src/Analyzer/Passes/IfChainToMultiIfPass.cpp index 70b717f3108..beb2247607e 100644 --- a/src/Analyzer/Passes/IfChainToMultiIfPass.cpp +++ b/src/Analyzer/Passes/IfChainToMultiIfPass.cpp @@ -65,6 +65,12 @@ public: auto multi_if_function = std::make_shared("multiIf"); multi_if_function->getArguments().getNodes() = std::move(multi_if_arguments); multi_if_function->resolveAsFunction(multi_if_function_ptr->build(multi_if_function->getArgumentColumns())); + + /// Ignore if returned type changed. + /// Example : SELECT now64(if(Null, NULL, if(Null, nan, toFloat64(number))), Null) FROM numbers(2) + if (!multi_if_function->getResultType()->equals(*function_node->getResultType())) + return; + node = std::move(multi_if_function); } diff --git a/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp b/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp index d881af3a51b..fd8c3e6ee6c 100644 --- a/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp +++ b/src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp @@ -31,6 +31,12 @@ public: if (!getSettings().optimize_group_by_function_keys) return; + /// When group_by_use_nulls = 1 removing keys from GROUP BY can lead + /// to unexpected types in some functions. + /// See example in https://github.com/ClickHouse/ClickHouse/pull/61567#issuecomment-2018007887 + if (getSettings().group_by_use_nulls) + return; + auto * query = node->as(); if (!query) return; @@ -73,12 +79,14 @@ private: candidates.push_back({ *it, is_deterministic }); /// Using DFS we traverse function tree and try to find if it uses other keys as function arguments. + bool found_at_least_one_usage = false; while (!candidates.empty()) { auto [candidate, parents_are_only_deterministic] = candidates.back(); candidates.pop_back(); bool found = group_by_keys.contains(candidate); + found_at_least_one_usage |= found; switch (candidate->getNodeType()) { @@ -111,7 +119,7 @@ private: } } - return true; + return found_at_least_one_usage; } static void optimizeGroupingSet(QueryTreeNodes & grouping_set) diff --git a/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp b/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp index ad649834fb0..61893202525 100644 --- a/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp +++ b/src/Analyzer/Passes/OptimizeGroupByInjectiveFunctionsPass.cpp @@ -43,6 +43,13 @@ public: if (!getSettings().optimize_injective_functions_in_group_by) return; + /// Don't optimize injective functions when group_by_use_nulls=true, + /// because in this case we make initial group by keys Nullable + /// and eliminating some functions can cause issues with arguments Nullability + /// during their execution. See examples in https://github.com/ClickHouse/ClickHouse/pull/61567#issuecomment-2008181143 + if (getSettings().group_by_use_nulls) + return; + auto * query = node->as(); if (!query) return; diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index be783f207c2..f5474ddb662 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -776,7 +776,13 @@ struct IdentifierResolveScope /// Table expression node to data std::unordered_map table_expression_node_to_data; - QueryTreeNodePtrWithHashSet nullable_group_by_keys; + QueryTreeNodePtrWithHashWithoutAliasSet nullable_group_by_keys; + /// Here we count the number of nullable GROUP BY keys we met resolving expression. + /// E.g. for a query `SELECT tuple(tuple(number)) FROM numbers(10) GROUP BY (number, tuple(number)) with cube` + /// both `number` and `tuple(number)` would be in nullable_group_by_keys. + /// But when we resolve `tuple(tuple(number))` we should figure out that `tuple(number)` is already a key, + /// and we should not convert `number` to nullable. + size_t found_nullable_group_by_key_in_scope = 0; /** It's possible that after a JOIN, a column in the projection has a type different from the column in the source table. * (For example, after join_use_nulls or USING column casted to supertype) @@ -2059,13 +2065,9 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden subquery_context->setSetting("use_structure_from_insertion_table_in_table_functions", false); auto options = SelectQueryOptions(QueryProcessingStage::Complete, scope.subquery_depth, true /*is_subquery*/); + options.only_analyze = only_analyze; auto interpreter = std::make_unique(node->toAST(), subquery_context, subquery_context->getViewSource(), options); - auto io = interpreter->execute(); - PullingAsyncPipelineExecutor executor(io.pipeline); - io.pipeline.setProgressCallback(context->getProgressCallback()); - io.pipeline.setProcessListElement(context->getProcessListElement()); - if (only_analyze) { /// If query is only analyzed, then constants are not correct. @@ -2082,6 +2084,11 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden } else { + auto io = interpreter->execute(); + PullingAsyncPipelineExecutor executor(io.pipeline); + io.pipeline.setProgressCallback(context->getProgressCallback()); + io.pipeline.setProcessListElement(context->getProcessListElement()); + Block block; while (block.rows() == 0 && executor.pull(block)) @@ -2193,7 +2200,7 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden auto & nearest_query_scope_query_node = nearest_query_scope->scope_node->as(); auto & mutable_context = nearest_query_scope_query_node.getMutableContext(); - auto scalar_query_hash_string = DB::toString(node_with_hash.hash); + auto scalar_query_hash_string = DB::toString(node_with_hash.hash) + (only_analyze ? "_analyze" : ""); if (mutable_context->hasQueryContext()) mutable_context->getQueryContext()->addScalar(scalar_query_hash_string, scalar_block); @@ -6148,6 +6155,12 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(QueryTreeNodePtr & node, Id return resolved_expression_it->second; } + bool is_nullable_group_by_key = scope.nullable_group_by_keys.contains(node) && !scope.expressions_in_resolve_process_stack.hasAggregateFunction(); + if (is_nullable_group_by_key) + ++scope.found_nullable_group_by_key_in_scope; + + SCOPE_EXIT(scope.found_nullable_group_by_key_in_scope -= is_nullable_group_by_key); + String node_alias = node->getAlias(); ProjectionNames result_projection_names; @@ -6439,7 +6452,7 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(QueryTreeNodePtr & node, Id validateTreeSize(node, scope.context->getSettingsRef().max_expanded_ast_elements, node_to_tree_size); - if (scope.nullable_group_by_keys.contains(node) && !scope.expressions_in_resolve_process_stack.hasAggregateFunction()) + if (is_nullable_group_by_key && scope.found_nullable_group_by_key_in_scope == 1) { node = node->clone(); node->convertToNullable(); @@ -6666,45 +6679,48 @@ void QueryAnalyzer::resolveGroupByNode(QueryNode & query_node_typed, IdentifierR if (query_node_typed.isGroupByWithGroupingSets()) { + QueryTreeNodes nullable_group_by_keys; for (auto & grouping_sets_keys_list_node : query_node_typed.getGroupBy().getNodes()) { if (settings.enable_positional_arguments) replaceNodesWithPositionalArguments(grouping_sets_keys_list_node, query_node_typed.getProjection().getNodes(), scope); - resolveExpressionNodeList(grouping_sets_keys_list_node, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); - // Remove redundant calls to `tuple` function. It simplifies checking if expression is an aggregation key. // It's required to support queries like: SELECT number FROM numbers(3) GROUP BY (number, number % 2) auto & group_by_list = grouping_sets_keys_list_node->as().getNodes(); expandTuplesInList(group_by_list); + + if (scope.group_by_use_nulls) + for (const auto & group_by_elem : group_by_list) + nullable_group_by_keys.push_back(group_by_elem->clone()); + + resolveExpressionNodeList(grouping_sets_keys_list_node, scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); } - if (scope.group_by_use_nulls) - { - for (const auto & grouping_set : query_node_typed.getGroupBy().getNodes()) - { - for (const auto & group_by_elem : grouping_set->as()->getNodes()) - scope.nullable_group_by_keys.insert(group_by_elem); - } - } + for (auto & nullable_group_by_key : nullable_group_by_keys) + scope.nullable_group_by_keys.insert(std::move(nullable_group_by_key)); } else { if (settings.enable_positional_arguments) replaceNodesWithPositionalArguments(query_node_typed.getGroupByNode(), query_node_typed.getProjection().getNodes(), scope); - resolveExpressionNodeList(query_node_typed.getGroupByNode(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); - // Remove redundant calls to `tuple` function. It simplifies checking if expression is an aggregation key. // It's required to support queries like: SELECT number FROM numbers(3) GROUP BY (number, number % 2) auto & group_by_list = query_node_typed.getGroupBy().getNodes(); expandTuplesInList(group_by_list); + QueryTreeNodes nullable_group_by_keys; if (scope.group_by_use_nulls) { for (const auto & group_by_elem : query_node_typed.getGroupBy().getNodes()) - scope.nullable_group_by_keys.insert(group_by_elem); + nullable_group_by_keys.push_back(group_by_elem->clone()); } + + resolveExpressionNodeList(query_node_typed.getGroupByNode(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); + + for (auto & nullable_group_by_key : nullable_group_by_keys) + scope.nullable_group_by_keys.insert(std::move(nullable_group_by_key)); } } @@ -7360,7 +7376,7 @@ void QueryAnalyzer::resolveTableFunction(QueryTreeNodePtr & table_function_node, ColumnDescription column = insert_columns.get(*insert_column_name_it); /// Change ephemeral columns to default columns. column.default_desc.kind = ColumnDefaultKind::Default; - structure_hint.add(insert_columns.get(*insert_column_name_it)); + structure_hint.add(std::move(column)); } } diff --git a/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.cpp b/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.cpp index 3c93bf9e1bf..917256bf4b1 100644 --- a/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.cpp +++ b/src/Analyzer/Passes/RewriteSumFunctionWithSumAndCountPass.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include namespace DB @@ -83,7 +84,7 @@ public: rhs->getArguments().getNodes().push_back(rhs_count); resolveOrdinaryFunctionNode(*rhs, rhs->getFunctionName()); - const auto new_node = std::make_shared(Poco::toLower(func_plus_minus_node->getFunctionName())); + auto new_node = std::make_shared(Poco::toLower(func_plus_minus_node->getFunctionName())); if (column_id == 0) new_node->getArguments().getNodes() = {lhs, rhs}; else if (column_id == 1) @@ -93,7 +94,12 @@ public: if (!new_node) return; - node = new_node; + QueryTreeNodePtr res = std::move(new_node); + + if (!res->getResultType()->equals(*function_node->getResultType())) + res = createCastFunction(res, function_node->getResultType(), getContext()); + + node = std::move(res); } diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp index 14eb179680c..9c07884a464 100644 --- a/src/Analyzer/QueryTreePassManager.cpp +++ b/src/Analyzer/QueryTreePassManager.cpp @@ -28,7 +28,6 @@ #include #include #include -#include #include #include #include @@ -264,7 +263,6 @@ void addQueryTreePasses(QueryTreePassManager & manager, bool only_analyze) manager.addPass(std::make_unique()); manager.addPass(std::make_unique()); manager.addPass(std::make_unique()); - manager.addPass(std::make_unique()); /// should before AggregateFunctionsArithmericOperationsPass manager.addPass(std::make_unique()); diff --git a/src/Analyzer/SetUtils.h b/src/Analyzer/SetUtils.h index c35b45dce59..aef906a6576 100644 --- a/src/Analyzer/SetUtils.h +++ b/src/Analyzer/SetUtils.h @@ -1,14 +1,15 @@ #pragma once -#include +#include -#include - -#include +#include namespace DB { +class IDataType; +using DataTypePtr = std::shared_ptr; + class Set; using SetPtr = std::shared_ptr; diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 767a9b2b9f9..7a3192d1d9c 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -753,7 +753,7 @@ void ClientBase::setDefaultFormatsFromConfiguration() else default_output_format = "TSV"; } - else if (is_interactive || stdout_is_a_tty) + else if (is_interactive) { default_output_format = "PrettyCompact"; } diff --git a/src/Client/ConnectionEstablisher.cpp b/src/Client/ConnectionEstablisher.cpp index c43aa6d8087..303105751ad 100644 --- a/src/Client/ConnectionEstablisher.cpp +++ b/src/Client/ConnectionEstablisher.cpp @@ -70,6 +70,12 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: ProfileEvents::increment(ProfileEvents::DistributedConnectionUsable); result.is_usable = true; + if (table_status_it->second.is_readonly) + { + result.is_readonly = true; + LOG_TRACE(log, "Table {}.{} is readonly on server {}", table_to_check->database, table_to_check->table, result.entry->getDescription()); + } + const UInt64 max_allowed_delay = settings.max_replica_delay_for_distributed_queries; if (!max_allowed_delay) { diff --git a/src/Client/ConnectionPoolWithFailover.cpp b/src/Client/ConnectionPoolWithFailover.cpp index 492fd4ae9e2..fb895d17763 100644 --- a/src/Client/ConnectionPoolWithFailover.cpp +++ b/src/Client/ConnectionPoolWithFailover.cpp @@ -52,7 +52,7 @@ IConnectionPool::Entry ConnectionPoolWithFailover::get(const ConnectionTimeouts settings.distributed_replica_max_ignored_errors = 0; settings.fallback_to_stale_replicas_for_distributed_queries = true; - return get(timeouts, settings, true); + return get(timeouts, settings, /* force_connected= */ true); } IConnectionPool::Entry ConnectionPoolWithFailover::get(const ConnectionTimeouts & timeouts, @@ -65,7 +65,7 @@ IConnectionPool::Entry ConnectionPoolWithFailover::get(const ConnectionTimeouts TryGetEntryFunc try_get_entry = [&](const NestedPoolPtr & pool, std::string & fail_message) { - return tryGetEntry(pool, timeouts, fail_message, settings, {}); + return tryGetEntry(pool, timeouts, fail_message, settings); }; const size_t offset = settings.load_balancing_first_offset % nested_pools.size(); @@ -158,6 +158,21 @@ std::vector ConnectionPoolWithFailover::g return getManyImpl(settings, pool_mode, try_get_entry, skip_unavailable_endpoints, priority_func); } +std::vector ConnectionPoolWithFailover::getManyCheckedForInsert( + const ConnectionTimeouts & timeouts, + const Settings & settings, + PoolMode pool_mode, + const QualifiedTableName & table_to_check) +{ + TryGetEntryFunc try_get_entry = [&](const NestedPoolPtr & pool, std::string & fail_message) + { return tryGetEntry(pool, timeouts, fail_message, settings, &table_to_check, /*async_callback=*/ {}); }; + + return getManyImpl(settings, pool_mode, try_get_entry, + /*skip_unavailable_endpoints=*/ std::nullopt, + /*priority_func=*/ {}, + settings.distributed_insert_skip_read_only_replicas); +} + ConnectionPoolWithFailover::Base::GetPriorityFunc ConnectionPoolWithFailover::makeGetPriorityFunc(const Settings & settings) { const size_t offset = settings.load_balancing_first_offset % nested_pools.size(); @@ -171,7 +186,8 @@ std::vector ConnectionPoolWithFailover::g PoolMode pool_mode, const TryGetEntryFunc & try_get_entry, std::optional skip_unavailable_endpoints, - GetPriorityForLoadBalancing::Func priority_func) + GetPriorityForLoadBalancing::Func priority_func, + bool skip_read_only_replicas) { if (nested_pools.empty()) throw DB::Exception( @@ -191,11 +207,17 @@ std::vector ConnectionPoolWithFailover::g max_entries = nested_pools.size(); } else if (pool_mode == PoolMode::GET_ONE) + { max_entries = 1; + } else if (pool_mode == PoolMode::GET_MANY) + { max_entries = settings.max_parallel_replicas; + } else + { throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unknown pool allocation mode"); + } if (!priority_func) priority_func = makeGetPriorityFunc(settings); @@ -203,7 +225,7 @@ std::vector ConnectionPoolWithFailover::g UInt64 max_ignored_errors = settings.distributed_replica_max_ignored_errors.value; bool fallback_to_stale_replicas = settings.fallback_to_stale_replicas_for_distributed_queries.value; - return Base::getMany(min_entries, max_entries, max_tries, max_ignored_errors, fallback_to_stale_replicas, try_get_entry, priority_func); + return Base::getMany(min_entries, max_entries, max_tries, max_ignored_errors, fallback_to_stale_replicas, skip_read_only_replicas, try_get_entry, priority_func); } ConnectionPoolWithFailover::TryResult diff --git a/src/Client/ConnectionPoolWithFailover.h b/src/Client/ConnectionPoolWithFailover.h index edfcbe6e4df..a363a50244e 100644 --- a/src/Client/ConnectionPoolWithFailover.h +++ b/src/Client/ConnectionPoolWithFailover.h @@ -77,6 +77,12 @@ public: AsyncCallback async_callback = {}, std::optional skip_unavailable_endpoints = std::nullopt, GetPriorityForLoadBalancing::Func priority_func = {}); + /// The same as getManyChecked(), but respects distributed_insert_skip_read_only_replicas setting. + std::vector getManyCheckedForInsert( + const ConnectionTimeouts & timeouts, + const Settings & settings, + PoolMode pool_mode, + const QualifiedTableName & table_to_check); struct NestedPoolStatus { @@ -107,7 +113,8 @@ private: PoolMode pool_mode, const TryGetEntryFunc & try_get_entry, std::optional skip_unavailable_endpoints = std::nullopt, - GetPriorityForLoadBalancing::Func priority_func = {}); + GetPriorityForLoadBalancing::Func priority_func = {}, + bool skip_read_only_replicas = false); /// Try to get a connection from the pool and check that it is good. /// If table_to_check is not null and the check is enabled in settings, check that replication delay diff --git a/src/Client/HedgedConnectionsFactory.cpp b/src/Client/HedgedConnectionsFactory.cpp index f5b074a0257..0fa2bc12924 100644 --- a/src/Client/HedgedConnectionsFactory.cpp +++ b/src/Client/HedgedConnectionsFactory.cpp @@ -82,7 +82,7 @@ std::vector HedgedConnectionsFactory::getManyConnections(PoolMode } case PoolMode::GET_MANY: { - max_entries = max_parallel_replicas; + max_entries = std::min(max_parallel_replicas, shuffled_pools.size()); break; } } diff --git a/src/Client/HedgedConnectionsFactory.h b/src/Client/HedgedConnectionsFactory.h index 2b7ec3f3fe5..c5bcbdf0689 100644 --- a/src/Client/HedgedConnectionsFactory.h +++ b/src/Client/HedgedConnectionsFactory.h @@ -158,7 +158,7 @@ private: /// checking the number of requested replicas that are still in process). size_t requested_connections_count = 0; - const size_t max_parallel_replicas = 0; + const size_t max_parallel_replicas = 1; const bool skip_unavailable_shards = false; }; diff --git a/src/Columns/ColumnObject.cpp b/src/Columns/ColumnObject.cpp index 25a62440629..0a4f90c2262 100644 --- a/src/Columns/ColumnObject.cpp +++ b/src/Columns/ColumnObject.cpp @@ -20,12 +20,12 @@ namespace DB namespace ErrorCodes { - extern const int ILLEGAL_COLUMN; + extern const int ARGUMENT_OUT_OF_BOUND; extern const int DUPLICATE_COLUMN; + extern const int EXPERIMENTAL_FEATURE_ERROR; + extern const int ILLEGAL_COLUMN; extern const int NUMBER_OF_DIMENSIONS_MISMATCHED; extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; - extern const int ARGUMENT_OUT_OF_BOUND; - extern const int EXPERIMENTAL_FEATURE_ERROR; } namespace @@ -334,7 +334,18 @@ void ColumnObject::Subcolumn::insert(Field field, FieldInfo info) if (type_changed || info.need_convert) field = convertFieldToTypeOrThrow(field, *least_common_type.get()); - data.back()->insert(field); + if (!data.back()->tryInsert(field)) + { + /** Normalization of the field above is pretty complicated (it uses several FieldVisitors), + * so in the case of a bug, we may get mismatched types. + * The `IColumn::insert` method does not check the type of the inserted field, and it can lead to a segmentation fault. + * Therefore, we use the safer `tryInsert` method to get an exception instead of a segmentation fault. + */ + throw Exception(ErrorCodes::EXPERIMENTAL_FEATURE_ERROR, + "Cannot insert field {} to column {}", + field.dump(), data.back()->dumpStructure()); + } + ++num_rows; } diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 015a114105d..4e3b9963107 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -460,6 +460,28 @@ Float32 ColumnVector::getFloat32(size_t n [[maybe_unused]]) const throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get the value of {} as Float32", TypeName); } +template +bool ColumnVector::tryInsert(const DB::Field & x) +{ + NearestFieldType value; + if (!x.tryGet>(value)) + { + if constexpr (std::is_same_v) + { + /// It's also possible to insert boolean values into UInt8 column. + bool boolean_value; + if (x.tryGet(boolean_value)) + { + data.push_back(static_cast(boolean_value)); + return true; + } + } + return false; + } + data.push_back(static_cast(value)); + return true; +} + template void ColumnVector::insertRangeFrom(const IColumn & src, size_t start, size_t length) { diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h index b03d476a8f1..39ee1d931bd 100644 --- a/src/Columns/ColumnVector.h +++ b/src/Columns/ColumnVector.h @@ -224,14 +224,8 @@ public: data.push_back(static_cast(x.get())); } - bool tryInsert(const DB::Field & x) override - { - NearestFieldType value; - if (!x.tryGet>(value)) - return false; - data.push_back(static_cast(value)); - return true; - } + bool tryInsert(const DB::Field & x) override; + void insertRangeFrom(const IColumn & src, size_t start, size_t length) override; ColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override; diff --git a/src/Common/AtomicLogger.h b/src/Common/AtomicLogger.h index 4bda55e070b..0ece9e8a09a 100644 --- a/src/Common/AtomicLogger.h +++ b/src/Common/AtomicLogger.h @@ -2,10 +2,13 @@ #include +#include #include -#include #include +#include +namespace DB +{ /** AtomicLogger allows to atomically change logger. * Standard library does not have atomic_shared_ptr, and we do not use std::atomic* operations, @@ -49,3 +52,5 @@ private: mutable DB::SharedMutex log_mutex; LoggerPtr logger; }; + +} diff --git a/src/Common/CurrentThreadHelpers.cpp b/src/Common/CurrentThreadHelpers.cpp new file mode 100644 index 00000000000..cbfb50bf3b1 --- /dev/null +++ b/src/Common/CurrentThreadHelpers.cpp @@ -0,0 +1,16 @@ +#include +#include + +namespace DB +{ + +bool currentThreadHasGroup() +{ + return DB::CurrentThread::getGroup() != nullptr; +} + +LogsLevel currentThreadLogsLevel() +{ + return DB::CurrentThread::get().getClientLogsLevel(); +} +} diff --git a/src/Common/CurrentThreadHelpers.h b/src/Common/CurrentThreadHelpers.h new file mode 100644 index 00000000000..01a180e74d2 --- /dev/null +++ b/src/Common/CurrentThreadHelpers.h @@ -0,0 +1,9 @@ +#pragma once + +#include + +namespace DB +{ +bool currentThreadHasGroup(); +LogsLevel currentThreadLogsLevel(); +} diff --git a/src/Common/DateLUT.cpp b/src/Common/DateLUT.cpp index 2b261a28469..3a20fb1a125 100644 --- a/src/Common/DateLUT.cpp +++ b/src/Common/DateLUT.cpp @@ -1,13 +1,15 @@ #include "DateLUT.h" +#include +#include +#include + #include #include #include -#include #include #include -#include namespace @@ -140,6 +142,38 @@ std::string determineDefaultTimeZone() } +const DateLUTImpl & DateLUT::instance() +{ + const auto & date_lut = getInstance(); + + if (DB::CurrentThread::isInitialized()) + { + std::string timezone_from_context; + const DB::ContextPtr query_context = DB::CurrentThread::get().getQueryContext(); + + if (query_context) + { + timezone_from_context = extractTimezoneFromContext(query_context); + + if (!timezone_from_context.empty()) + return date_lut.getImplementation(timezone_from_context); + } + + /// On the server side, timezone is passed in query_context, + /// but on CH-client side we have no query context, + /// and each time we modify client's global context + const DB::ContextPtr global_context = DB::CurrentThread::get().getGlobalContext(); + if (global_context) + { + timezone_from_context = extractTimezoneFromContext(global_context); + + if (!timezone_from_context.empty()) + return date_lut.getImplementation(timezone_from_context); + } + } + return serverTimezoneInstance(); +} + DateLUT::DateLUT() { /// Initialize the pointer to the default DateLUTImpl. diff --git a/src/Common/DateLUT.h b/src/Common/DateLUT.h index 2045d4895e7..d0b85ea9895 100644 --- a/src/Common/DateLUT.h +++ b/src/Common/DateLUT.h @@ -1,17 +1,23 @@ #pragma once -#include "DateLUTImpl.h" - #include +#include #include -#include "Common/CurrentThread.h" #include #include #include #include +namespace DB +{ +class Context; +using ContextPtr = std::shared_ptr; +} + +class DateLUTImpl; + /// This class provides lazy initialization and lookup of singleton DateLUTImpl objects for a given timezone. class DateLUT : private boost::noncopyable @@ -20,38 +26,7 @@ public: /// Return DateLUTImpl instance for session timezone. /// session_timezone is a session-level setting. /// If setting is not set, returns the server timezone. - static ALWAYS_INLINE const DateLUTImpl & instance() - { - const auto & date_lut = getInstance(); - - if (DB::CurrentThread::isInitialized()) - { - std::string timezone_from_context; - const DB::ContextPtr query_context = DB::CurrentThread::get().getQueryContext(); - - if (query_context) - { - timezone_from_context = extractTimezoneFromContext(query_context); - - if (!timezone_from_context.empty()) - return date_lut.getImplementation(timezone_from_context); - } - - /// On the server side, timezone is passed in query_context, - /// but on CH-client side we have no query context, - /// and each time we modify client's global context - const DB::ContextPtr global_context = DB::CurrentThread::get().getGlobalContext(); - if (global_context) - { - timezone_from_context = extractTimezoneFromContext(global_context); - - if (!timezone_from_context.empty()) - return date_lut.getImplementation(timezone_from_context); - } - - } - return serverTimezoneInstance(); - } + static const DateLUTImpl & instance(); static ALWAYS_INLINE const DateLUTImpl & instance(const std::string & time_zone) { diff --git a/src/Common/DateLUTImpl.cpp b/src/Common/DateLUTImpl.cpp index bb677b3a62d..341e571e4eb 100644 --- a/src/Common/DateLUTImpl.cpp +++ b/src/Common/DateLUTImpl.cpp @@ -1,8 +1,5 @@ -#include "DateLUTImpl.h" - -#include -#include -#include +#include +#include #include #include @@ -11,6 +8,10 @@ #include #include +#include +#include +#include + namespace DB { @@ -214,6 +215,29 @@ DateLUTImpl::DateLUTImpl(const std::string & time_zone_) } } +unsigned int DateLUTImpl::toMillisecond(const DB::DateTime64 & datetime, Int64 scale_multiplier) const +{ + constexpr Int64 millisecond_multiplier = 1'000; + constexpr Int64 microsecond_multiplier = 1'000 * millisecond_multiplier; + constexpr Int64 divider = microsecond_multiplier / millisecond_multiplier; + + auto components = DB::DecimalUtils::splitWithScaleMultiplier(datetime, scale_multiplier); + + if (datetime.value < 0 && components.fractional) + { + components.fractional = scale_multiplier + (components.whole ? Int64(-1) : Int64(1)) * components.fractional; + --components.whole; + } + Int64 fractional = components.fractional; + if (scale_multiplier > microsecond_multiplier) + fractional = fractional / (scale_multiplier / microsecond_multiplier); + else if (scale_multiplier < microsecond_multiplier) + fractional = fractional * (microsecond_multiplier / scale_multiplier); + + UInt16 millisecond = static_cast(fractional / divider); + return millisecond; +} + /// Prefer to load timezones from blobs linked to the binary. /// The blobs are provided by "tzdata" library. diff --git a/src/Common/DateLUTImpl.h b/src/Common/DateLUTImpl.h index 082127e717c..01cbae3d447 100644 --- a/src/Common/DateLUTImpl.h +++ b/src/Common/DateLUTImpl.h @@ -3,7 +3,6 @@ #include #include #include -#include #include #include @@ -50,6 +49,11 @@ enum class WeekDayMode WeekStartsSunday1 = 3 }; +namespace DB +{ +class DateTime64; +} + /** Lookup table to conversion of time to date, and to month / year / day of week / day of month and so on. * First time was implemented for OLAPServer, that needed to do billions of such transformations. */ @@ -593,29 +597,7 @@ public: return time % 60; } - template - unsigned toMillisecond(const DateOrTime & datetime, Int64 scale_multiplier) const - { - constexpr Int64 millisecond_multiplier = 1'000; - constexpr Int64 microsecond_multiplier = 1'000 * millisecond_multiplier; - constexpr Int64 divider = microsecond_multiplier / millisecond_multiplier; - - auto components = DB::DecimalUtils::splitWithScaleMultiplier(datetime, scale_multiplier); - - if (datetime.value < 0 && components.fractional) - { - components.fractional = scale_multiplier + (components.whole ? Int64(-1) : Int64(1)) * components.fractional; - --components.whole; - } - Int64 fractional = components.fractional; - if (scale_multiplier > microsecond_multiplier) - fractional = fractional / (scale_multiplier / microsecond_multiplier); - else if (scale_multiplier < microsecond_multiplier) - fractional = fractional * (microsecond_multiplier / scale_multiplier); - - UInt16 millisecond = static_cast(fractional / divider); - return millisecond; - } + unsigned toMillisecond(const DB::DateTime64 & datetime, Int64 scale_multiplier) const; unsigned toMinute(Time t) const { diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index 7e73e2c0783..7d38fdafddb 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -1,26 +1,27 @@ -#include "Exception.h" - -#include -#include -#include -#include -#include #include #include #include #include #include #include -#include +#include #include +#include #include #include #include +#include #include #include #include -#include +#include +#include +#include +#include +#include + +#include namespace fs = std::filesystem; diff --git a/src/Common/Exception.h b/src/Common/Exception.h index 0c8a7177a99..97af8d1ffc3 100644 --- a/src/Common/Exception.h +++ b/src/Common/Exception.h @@ -1,22 +1,20 @@ #pragma once -#include -#include -#include -#include - -#include - #include #include #include #include -#include #include #include #include +#include +#include +#include +#include + #include +#include namespace Poco { class Logger; } @@ -24,6 +22,8 @@ namespace Poco { class Logger; } namespace DB { +class AtomicLogger; + [[noreturn]] void abortOnFailedAssertion(const String & description); /// This flag can be set for testing purposes - to check that no exceptions are thrown. diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 9e551c8f2cd..2434c6004ad 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -50,7 +50,9 @@ static struct InitFiu REGULAR(check_table_query_delay_for_part) \ REGULAR(dummy_failpoint) \ REGULAR(prefetched_reader_pool_failpoint) \ - PAUSEABLE_ONCE(dummy_pausable_failpoint_once) \ + PAUSEABLE_ONCE(replicated_merge_tree_insert_retry_pause) \ + PAUSEABLE_ONCE(finish_set_quorum_failed_parts) \ + PAUSEABLE_ONCE(finish_clean_quorum_failed_parts) \ PAUSEABLE(dummy_pausable_failpoint) \ ONCE(execute_query_calling_empty_set_result_func_on_exception) diff --git a/src/Common/HTTPConnectionPool.cpp b/src/Common/HTTPConnectionPool.cpp index f729b8ea8d0..cd2505df7f3 100644 --- a/src/Common/HTTPConnectionPool.cpp +++ b/src/Common/HTTPConnectionPool.cpp @@ -295,8 +295,13 @@ private: String getTarget() const { if (!Session::getProxyConfig().host.empty()) - return fmt::format("{} over proxy {}", Session::getHost(), Session::getProxyConfig().host); - return Session::getHost(); + return fmt::format("{}:{} over proxy {}", + Session::getHost(), + Session::getPort(), + Session::getProxyConfig().host); + return fmt::format("{}:{}", + Session::getHost(), + Session::getPort()); } void flushRequest() override @@ -472,7 +477,8 @@ public: String getTarget() const { if (!proxy_configuration.isEmpty()) - return fmt::format("{} over proxy {}", host, proxy_configuration.host); + return fmt::format("{} over proxy {}", + host, proxy_configuration.host); return host; } diff --git a/src/Common/Jemalloc.cpp b/src/Common/Jemalloc.cpp index 3eb8691a1e1..6514639e700 100644 --- a/src/Common/Jemalloc.cpp +++ b/src/Common/Jemalloc.cpp @@ -2,6 +2,7 @@ #if USE_JEMALLOC +#include #include #include #include diff --git a/src/Common/LocalDate.h b/src/Common/LocalDate.h index 2331a40fd12..f1abc98c8e2 100644 --- a/src/Common/LocalDate.h +++ b/src/Common/LocalDate.h @@ -1,9 +1,10 @@ #pragma once #include -#include #include +#include #include +#include /** Stores a calendar date in broken-down form (year, month, day-in-month). diff --git a/src/Common/Logger.h b/src/Common/Logger.h index 0425da8c847..b4688eb0a9c 100644 --- a/src/Common/Logger.h +++ b/src/Common/Logger.h @@ -1,15 +1,20 @@ #pragma once -#include - #include -#include +#include + #include #include -using LoggerPtr = Poco::LoggerPtr; +namespace Poco +{ +class Channel; +class Logger; +using LoggerPtr = std::shared_ptr; +} +using LoggerPtr = std::shared_ptr; using LoggerRawPtr = Poco::Logger *; /** RAII wrappers around Poco/Logger.h. diff --git a/src/Common/LoggingFormatStringHelpers.cpp b/src/Common/LoggingFormatStringHelpers.cpp index 7cbef779f28..3e90526f76d 100644 --- a/src/Common/LoggingFormatStringHelpers.cpp +++ b/src/Common/LoggingFormatStringHelpers.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include diff --git a/src/Common/PODArray.cpp b/src/Common/PODArray.cpp index dd1fed08cb5..a8195f15241 100644 --- a/src/Common/PODArray.cpp +++ b/src/Common/PODArray.cpp @@ -22,7 +22,7 @@ void protectMemoryRegion(void * addr, size_t len, int prot) } #endif -size_t byte_size(size_t num_elements, size_t element_size) +ALWAYS_INLINE size_t byte_size(size_t num_elements, size_t element_size) { size_t amount; if (__builtin_mul_overflow(num_elements, element_size, &amount)) @@ -30,7 +30,7 @@ size_t byte_size(size_t num_elements, size_t element_size) return amount; } -size_t minimum_memory_for_elements(size_t num_elements, size_t element_size, size_t pad_left, size_t pad_right) +ALWAYS_INLINE size_t minimum_memory_for_elements(size_t num_elements, size_t element_size, size_t pad_left, size_t pad_right) { size_t amount; if (__builtin_add_overflow(byte_size(num_elements, element_size), pad_left + pad_right, &amount)) diff --git a/src/Common/PoolWithFailoverBase.h b/src/Common/PoolWithFailoverBase.h index cf270c9dad0..8365e818840 100644 --- a/src/Common/PoolWithFailoverBase.h +++ b/src/Common/PoolWithFailoverBase.h @@ -30,6 +30,7 @@ namespace ProfileEvents { extern const Event DistributedConnectionFailTry; extern const Event DistributedConnectionFailAtAll; + extern const Event DistributedConnectionSkipReadOnlyReplica; } /// This class provides a pool with fault tolerance. It is used for pooling of connections to replicated DB. @@ -73,13 +74,7 @@ public: { TryResult() = default; - void reset() - { - entry = Entry(); - is_usable = false; - is_up_to_date = false; - delay = 0; - } + void reset() { *this = {}; } Entry entry; /// use isNull() to check if connection is established bool is_usable = false; /// if connection is established, then can be false only with table check @@ -87,6 +82,7 @@ public: bool is_up_to_date = false; /// If true, the entry is a connection to up-to-date replica /// Depends on max_replica_delay_for_distributed_queries setting UInt32 delay = 0; /// Helps choosing the "least stale" option when all replicas are stale. + bool is_readonly = false; /// Table is in read-only mode, INSERT can ignore such replicas. }; struct PoolState; @@ -117,6 +113,7 @@ public: size_t min_entries, size_t max_entries, size_t max_tries, size_t max_ignored_errors, bool fallback_to_stale_replicas, + bool skip_read_only_replicas, const TryGetEntryFunc & try_get_entry, const GetPriorityFunc & get_priority); @@ -205,8 +202,12 @@ PoolWithFailoverBase::get(size_t max_ignored_errors, bool fallback_ const TryGetEntryFunc & try_get_entry, const GetPriorityFunc & get_priority) { std::vector results = getMany( - 1 /* min entries */, 1 /* max entries */, 1 /* max tries */, - max_ignored_errors, fallback_to_stale_replicas, + /* min_entries= */ 1, + /* max_entries= */ 1, + /* max_tries= */ 1, + max_ignored_errors, + fallback_to_stale_replicas, + /* skip_read_only_replicas= */ false, try_get_entry, get_priority); if (results.empty() || results[0].entry.isNull()) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, @@ -220,6 +221,7 @@ PoolWithFailoverBase::getMany( size_t min_entries, size_t max_entries, size_t max_tries, size_t max_ignored_errors, bool fallback_to_stale_replicas, + bool skip_read_only_replicas, const TryGetEntryFunc & try_get_entry, const GetPriorityFunc & get_priority) { @@ -271,9 +273,14 @@ PoolWithFailoverBase::getMany( ++entries_count; if (result.is_usable) { - ++usable_count; - if (result.is_up_to_date) - ++up_to_date_count; + if (skip_read_only_replicas && result.is_readonly) + ProfileEvents::increment(ProfileEvents::DistributedConnectionSkipReadOnlyReplica); + else + { + ++usable_count; + if (result.is_up_to_date) + ++up_to_date_count; + } } } else @@ -296,7 +303,7 @@ PoolWithFailoverBase::getMany( throw DB::NetException(DB::ErrorCodes::ALL_CONNECTION_TRIES_FAILED, "All connection tries failed. Log: \n\n{}\n", fail_messages); - std::erase_if(try_results, [](const TryResult & r) { return r.entry.isNull() || !r.is_usable; }); + std::erase_if(try_results, [&](const TryResult & r) { return r.entry.isNull() || !r.is_usable || (skip_read_only_replicas && r.is_readonly); }); /// Sort so that preferred items are near the beginning. std::stable_sort( diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index e91b5adec87..33ccb4e9f02 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -156,6 +156,7 @@ M(DistributedConnectionFailTry, "Total count when distributed connection fails with retry.") \ M(DistributedConnectionMissingTable, "Number of times we rejected a replica from a distributed query, because it did not contain a table needed for the query.") \ M(DistributedConnectionStaleReplica, "Number of times we rejected a replica from a distributed query, because some table needed for a query had replication lag higher than the configured threshold.") \ + M(DistributedConnectionSkipReadOnlyReplica, "Number of replicas skipped during INSERT into Distributed table due to replicas being read-only") \ M(DistributedConnectionFailAtAll, "Total count when distributed connection fails after all retries finished.") \ \ M(HedgedRequestsChangeReplica, "Total count when timeout for changing replica expired in hedged requests.") \ @@ -355,8 +356,6 @@ The server successfully detected this situation and will download merged part fr M(PerfLocalMemoryReferences, "Local NUMA node memory reads") \ M(PerfLocalMemoryMisses, "Local NUMA node memory read misses") \ \ - M(CreatedHTTPConnections, "Total amount of created HTTP connections (counter increase every time connection is created).") \ - \ M(CannotWriteToWriteBufferDiscard, "Number of stack traces dropped by query profiler or signal handler because pipe is full or cannot write to pipe.") \ M(QueryProfilerSignalOverruns, "Number of times we drop processing of a query profiler signal due to overrun plus the number of signals that OS has not delivered due to overrun.") \ M(QueryProfilerConcurrencyOverruns, "Number of times we drop processing of a query profiler signal due to too many concurrent query profilers in other threads, which may indicate overload.") \ @@ -436,8 +435,6 @@ The server successfully detected this situation and will download merged part fr M(ReadBufferFromS3ResetSessions, "Number of HTTP sessions that were reset in ReadBufferFromS3.") \ M(ReadBufferFromS3PreservedSessions, "Number of HTTP sessions that were preserved in ReadBufferFromS3.") \ \ - M(ReadWriteBufferFromHTTPPreservedSessions, "Number of HTTP sessions that were preserved in ReadWriteBufferFromHTTP.") \ - \ M(WriteBufferFromS3Microseconds, "Time spent on writing to S3.") \ M(WriteBufferFromS3Bytes, "Bytes written to S3.") \ M(WriteBufferFromS3RequestsErrors, "Number of exceptions while writing to S3.") \ @@ -459,7 +456,8 @@ The server successfully detected this situation and will download merged part fr M(FilesystemCacheLoadMetadataMicroseconds, "Time spent loading filesystem cache metadata") \ M(FilesystemCacheEvictedBytes, "Number of bytes evicted from filesystem cache") \ M(FilesystemCacheEvictedFileSegments, "Number of file segments evicted from filesystem cache") \ - M(FilesystemCacheEvictionSkippedFileSegments, "Number of file segments skipped for eviction because of being unreleasable") \ + M(FilesystemCacheEvictionSkippedFileSegments, "Number of file segments skipped for eviction because of being in unreleasable state") \ + M(FilesystemCacheEvictionSkippedEvictingFileSegments, "Number of file segments skipped for eviction because of being in evicting state") \ M(FilesystemCacheEvictionTries, "Number of filesystem cache eviction attempts") \ M(FilesystemCacheLockKeyMicroseconds, "Lock cache key time") \ M(FilesystemCacheLockMetadataMicroseconds, "Lock filesystem cache metadata time") \ @@ -726,6 +724,9 @@ The server successfully detected this situation and will download merged part fr M(AddressesDiscovered, "Total count of new addresses in dns resolve results for http connections") \ M(AddressesExpired, "Total count of expired addresses which is no longer presented in dns resolve results for http connections") \ M(AddressesMarkedAsFailed, "Total count of addresses which has been marked as faulty due to connection errors for http connections") \ + \ + M(ReadWriteBufferFromHTTPRequestsSent, "Number of HTTP requests sent by ReadWriteBufferFromHTTP") \ + M(ReadWriteBufferFromHTTPBytes, "Total size of payload bytes received and sent by ReadWriteBufferFromHTTP. Doesn't include HTTP headers.") \ #ifdef APPLY_FOR_EXTERNAL_EVENTS diff --git a/src/Common/ProfileEventsScope.cpp b/src/Common/ProfileEventsScope.cpp index 92f75f4f5b0..1ba5f7f165c 100644 --- a/src/Common/ProfileEventsScope.cpp +++ b/src/Common/ProfileEventsScope.cpp @@ -1,3 +1,4 @@ +#include #include namespace DB diff --git a/src/Common/ProfileEventsScope.h b/src/Common/ProfileEventsScope.h index 0444531d02b..8eabfcb55c0 100644 --- a/src/Common/ProfileEventsScope.h +++ b/src/Common/ProfileEventsScope.h @@ -1,7 +1,8 @@ #pragma once #include -#include + +#include namespace DB { diff --git a/src/Common/QueryProfiler.cpp b/src/Common/QueryProfiler.cpp index 34ffbf6c498..61d4d7d609c 100644 --- a/src/Common/QueryProfiler.cpp +++ b/src/Common/QueryProfiler.cpp @@ -1,15 +1,16 @@ #include "QueryProfiler.h" #include -#include +#include +#include +#include #include #include +#include #include -#include +#include #include -#include -#include -#include +#include #include diff --git a/src/Common/ThreadFuzzer.cpp b/src/Common/ThreadFuzzer.cpp index 9f9ec4fa356..d1e252a8184 100644 --- a/src/Common/ThreadFuzzer.cpp +++ b/src/Common/ThreadFuzzer.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 03d1b5a93d4..2185d32e47a 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -1,7 +1,8 @@ #include -#include -#include +#include +#include +#include #include #include #include @@ -10,17 +11,17 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include #include -#include -#include -#include +#include #include "Coordination/KeeperConstants.h" #include "config.h" diff --git a/src/Common/logger_useful.h b/src/Common/logger_useful.h index 1ce4f545e6f..9d6ebaddcc6 100644 --- a/src/Common/logger_useful.h +++ b/src/Common/logger_useful.h @@ -5,13 +5,11 @@ #include #include #include -#include -#include -#include -#include #include - -namespace Poco { class Logger; } +#include +#include +#include +#include #define LogToStr(x, y) std::make_unique(x, y) @@ -22,7 +20,7 @@ using LogSeriesLimiterPtr = std::shared_ptr; namespace impl { [[maybe_unused]] inline LoggerPtr getLoggerHelper(const LoggerPtr & logger) { return logger; } - [[maybe_unused]] inline LoggerPtr getLoggerHelper(const AtomicLogger & logger) { return logger.load(); } + [[maybe_unused]] inline LoggerPtr getLoggerHelper(const DB::AtomicLogger & logger) { return logger.load(); } [[maybe_unused]] inline const ::Poco::Logger * getLoggerHelper(const ::Poco::Logger * logger) { return logger; } [[maybe_unused]] inline std::unique_ptr getLoggerHelper(std::unique_ptr && logger) { return logger; } [[maybe_unused]] inline std::unique_ptr getLoggerHelper(std::unique_ptr && logger) { return logger; } @@ -66,8 +64,7 @@ namespace impl #define LOG_IMPL(logger, priority, PRIORITY, ...) do \ { \ auto _logger = ::impl::getLoggerHelper(logger); \ - const bool _is_clients_log = (DB::CurrentThread::getGroup() != nullptr) && \ - (DB::CurrentThread::get().getClientLogsLevel() >= (priority)); \ + const bool _is_clients_log = DB::currentThreadHasGroup() && DB::currentThreadLogsLevel() >= (priority); \ if (!_is_clients_log && !_logger->is((PRIORITY))) \ break; \ \ diff --git a/src/Common/mysqlxx/mysqlxx/Pool.h b/src/Common/mysqlxx/mysqlxx/Pool.h index c85295c4dd0..6e509d8bdd6 100644 --- a/src/Common/mysqlxx/mysqlxx/Pool.h +++ b/src/Common/mysqlxx/mysqlxx/Pool.h @@ -1,5 +1,7 @@ #pragma once +#include + #include #include #include diff --git a/src/Common/tests/gtest_resolve_pool.cpp b/src/Common/tests/gtest_resolve_pool.cpp index 25e867fdebc..eef4635e7b1 100644 --- a/src/Common/tests/gtest_resolve_pool.cpp +++ b/src/Common/tests/gtest_resolve_pool.cpp @@ -1,6 +1,7 @@ #include -#include #include +#include +#include #include #include diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index d531b1266ef..58d396aad88 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -2219,7 +2219,7 @@ uint64_t Changelog::getStartIndex() const LogEntryPtr Changelog::getLastEntry() const { /// This entry treaded in special way by NuRaft - static LogEntryPtr fake_entry = nuraft::cs_new(0, nuraft::buffer::alloc(sizeof(uint64_t))); + static LogEntryPtr fake_entry = nuraft::cs_new(0, nuraft::buffer::alloc(0)); auto entry = entry_storage.getEntry(max_log_id); if (entry == nullptr) diff --git a/src/Coordination/InMemoryLogStore.cpp b/src/Coordination/InMemoryLogStore.cpp index ee93c02b4b0..32aaf8e0d4a 100644 --- a/src/Coordination/InMemoryLogStore.cpp +++ b/src/Coordination/InMemoryLogStore.cpp @@ -16,7 +16,7 @@ ptr makeClone(const ptr & entry) InMemoryLogStore::InMemoryLogStore() : start_idx(1) { - nuraft::ptr buf = nuraft::buffer::alloc(sizeof(uint64_t)); + nuraft::ptr buf = nuraft::buffer::alloc(0); logs[0] = nuraft::cs_new(0, buf); } diff --git a/src/Coordination/KeeperConstants.cpp b/src/Coordination/KeeperConstants.cpp index aea2391cf13..8251dca3d1e 100644 --- a/src/Coordination/KeeperConstants.cpp +++ b/src/Coordination/KeeperConstants.cpp @@ -111,7 +111,6 @@ M(PerfLocalMemoryReferences) \ M(PerfLocalMemoryMisses) \ \ - M(CreatedHTTPConnections) \ M(CannotWriteToWriteBufferDiscard) \ \ M(S3ReadMicroseconds) \ @@ -180,8 +179,6 @@ M(ReadBufferFromS3RequestsErrors) \ M(ReadBufferFromS3ResetSessions) \ M(ReadBufferFromS3PreservedSessions) \ -\ - M(ReadWriteBufferFromHTTPPreservedSessions) \ \ M(WriteBufferFromS3Microseconds) \ M(WriteBufferFromS3Bytes) \ diff --git a/src/Core/BackgroundSchedulePool.h b/src/Core/BackgroundSchedulePool.h index eca93353283..a1450be2466 100644 --- a/src/Core/BackgroundSchedulePool.h +++ b/src/Core/BackgroundSchedulePool.h @@ -1,21 +1,20 @@ #pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include #include -#include +#include namespace DB diff --git a/src/Core/ExternalTable.cpp b/src/Core/ExternalTable.cpp index e043a2f9492..f8bbd16d038 100644 --- a/src/Core/ExternalTable.cpp +++ b/src/Core/ExternalTable.cpp @@ -81,8 +81,10 @@ void BaseExternalTable::parseStructureFromStructureField(const std::string & arg for (auto & child : columns_list_raw->children) { auto * column = child->as(); + /// We use `formatWithPossiblyHidingSensitiveData` instead of `getColumnNameWithoutAlias` because `column->type` is an ASTFunction. + /// `getColumnNameWithoutAlias` will return name of the function with `(arguments)` even if arguments is empty. if (column) - structure.emplace_back(column->name, column->type->getColumnNameWithoutAlias()); + structure.emplace_back(column->name, column->type->formatWithPossiblyHidingSensitiveData(0, true, true)); else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Error while parsing table structure: expected column definition, got {}", child->formatForErrorMessage()); } @@ -99,7 +101,7 @@ void BaseExternalTable::parseStructureFromTypesField(const std::string & argumen throw Exception(ErrorCodes::BAD_ARGUMENTS, "Error while parsing table structure: {}", error); for (size_t i = 0; i < type_list_raw->children.size(); ++i) - structure.emplace_back("_" + toString(i + 1), type_list_raw->children[i]->getColumnNameWithoutAlias()); + structure.emplace_back("_" + toString(i + 1), type_list_raw->children[i]->formatWithPossiblyHidingSensitiveData(0, true, true)); } void BaseExternalTable::initSampleBlock() diff --git a/src/Core/ProtocolDefines.h b/src/Core/ProtocolDefines.h index 058c6fdc903..159a4c28b6d 100644 --- a/src/Core/ProtocolDefines.h +++ b/src/Core/ProtocolDefines.h @@ -76,6 +76,9 @@ static constexpr auto DBMS_MIN_REVISION_WITH_SPARSE_SERIALIZATION = 54465; static constexpr auto DBMS_MIN_REVISION_WITH_SSH_AUTHENTICATION = 54466; +/// Send read-only flag for Replicated tables as well +static constexpr auto DBMS_MIN_REVISION_WITH_TABLE_READ_ONLY_CHECK = 54467; + /// Version of ClickHouse TCP protocol. /// /// Should be incremented manually on protocol changes. @@ -83,6 +86,6 @@ static constexpr auto DBMS_MIN_REVISION_WITH_SSH_AUTHENTICATION = 54466; /// NOTE: DBMS_TCP_PROTOCOL_VERSION has nothing common with VERSION_REVISION, /// later is just a number for server version (one number instead of commit SHA) /// for simplicity (sometimes it may be more convenient in some use cases). -static constexpr auto DBMS_TCP_PROTOCOL_VERSION = 54466; +static constexpr auto DBMS_TCP_PROTOCOL_VERSION = 54467; } diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 170fd4e9ca0..d5ea9534e6c 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -136,6 +136,7 @@ class IColumn; M(Bool, stream_like_engine_allow_direct_select, false, "Allow direct SELECT query for Kafka, RabbitMQ, FileLog, Redis Streams, and NATS engines. In case there are attached materialized views, SELECT query is not allowed even if this setting is enabled.", 0) \ M(String, stream_like_engine_insert_queue, "", "When stream like engine reads from multiple queues, user will need to select one queue to insert into when writing. Used by Redis Streams and NATS.", 0) \ \ + M(Bool, distributed_insert_skip_read_only_replicas, false, "If true, INSERT into Distributed will skip read-only replicas.", 0) \ M(Bool, distributed_foreground_insert, false, "If setting is enabled, insert query into distributed waits until data are sent to all nodes in a cluster. \n\nEnables or disables synchronous data insertion into a `Distributed` table.\n\nBy default, when inserting data into a Distributed table, the ClickHouse server sends data to cluster nodes in the background. When `distributed_foreground_insert` = 1, the data is processed synchronously, and the `INSERT` operation succeeds only after all the data is saved on all shards (at least one replica for each shard if `internal_replication` is true).", 0) ALIAS(insert_distributed_sync) \ M(UInt64, distributed_background_insert_timeout, 0, "Timeout for insert query into distributed. Setting is used only with insert_distributed_sync enabled. Zero value means no timeout.", 0) ALIAS(insert_distributed_timeout) \ M(Milliseconds, distributed_background_insert_sleep_time_ms, 100, "Sleep time for background INSERTs into Distributed, in case of any errors delay grows exponentially.", 0) ALIAS(distributed_directory_monitor_sleep_time_ms) \ @@ -187,7 +188,7 @@ class IColumn; \ M(Bool, group_by_use_nulls, false, "Treat columns mentioned in ROLLUP, CUBE or GROUPING SETS as Nullable", 0) \ \ - M(UInt64, max_parallel_replicas, 1, "The maximum number of replicas of each shard used when the query is executed. For consistency (to get different parts of the same partition), this option only works for the specified sampling key. The lag of the replicas is not controlled.", 0) \ + M(NonZeroUInt64, max_parallel_replicas, 1, "The maximum number of replicas of each shard used when the query is executed. For consistency (to get different parts of the same partition), this option only works for the specified sampling key. The lag of the replicas is not controlled. Should be always greater than 0", 0) \ M(UInt64, parallel_replicas_count, 0, "This is internal setting that should not be used directly and represents an implementation detail of the 'parallel replicas' mode. This setting will be automatically set up by the initiator server for distributed queries to the number of parallel replicas participating in query processing.", 0) \ M(UInt64, parallel_replica_offset, 0, "This is internal setting that should not be used directly and represents an implementation detail of the 'parallel replicas' mode. This setting will be automatically set up by the initiator server for distributed queries to the index of the replica participating in query processing among parallel replicas.", 0) \ M(String, parallel_replicas_custom_key, "", "Custom key assigning work to replicas when parallel replicas are used.", 0) \ @@ -380,7 +381,7 @@ class IColumn; M(Float, opentelemetry_start_trace_probability, 0., "Probability to start an OpenTelemetry trace for an incoming query.", 0) \ M(Bool, opentelemetry_trace_processors, false, "Collect OpenTelemetry spans for processors.", 0) \ M(Bool, prefer_column_name_to_alias, false, "Prefer using column names instead of aliases if possible.", 0) \ - M(Bool, allow_experimental_analyzer, false, "Allow experimental analyzer", 0) \ + M(Bool, allow_experimental_analyzer, true, "Allow experimental analyzer.", 0) \ M(Bool, analyzer_compatibility_join_using_top_level_identifier, false, "Force to resolve identifier in JOIN USING from projection (for example, in `SELECT a + 1 AS b FROM t1 JOIN t2 USING (b)` join will be performed by `t1.a + 1 = t2.b`, rather then `t1.b = t2.b`).", 0) \ M(Bool, prefer_global_in_and_join, false, "If enabled, all IN/JOIN operators will be rewritten as GLOBAL IN/JOIN. It's useful when the to-be-joined tables are only available on the initiator and we need to always scatter their data on-the-fly during distributed processing with the GLOBAL keyword. It's also useful to reduce the need to access the external sources joining external tables.", 0) \ M(Bool, enable_vertical_final, true, "If enable, remove duplicated rows during FINAL by marking rows as deleted and filtering them later instead of merging rows", 0) \ @@ -589,6 +590,7 @@ class IColumn; M(Bool, optimize_respect_aliases, true, "If it is set to true, it will respect aliases in WHERE/GROUP BY/ORDER BY, that will help with partition pruning/secondary indexes/optimize_aggregation_in_order/optimize_read_in_order/optimize_trivial_count", 0) \ M(UInt64, mutations_sync, 0, "Wait for synchronous execution of ALTER TABLE UPDATE/DELETE queries (mutations). 0 - execute asynchronously. 1 - wait current server. 2 - wait all replicas if they exist.", 0) \ M(Bool, enable_lightweight_delete, true, "Enable lightweight DELETE mutations for mergetree tables.", 0) ALIAS(allow_experimental_lightweight_delete) \ + M(UInt64, lightweight_deletes_sync, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes", 0) \ M(Bool, apply_deleted_mask, true, "Enables filtering out rows deleted with lightweight DELETE. If disabled, a query will be able to read those rows. This is useful for debugging and \"undelete\" scenarios", 0) \ M(Bool, optimize_normalize_count_variants, true, "Rewrite aggregate functions that semantically equals to count() as count().", 0) \ M(Bool, optimize_injective_functions_inside_uniq, true, "Delete injective functions of one argument inside uniq*() functions.", 0) \ @@ -1048,6 +1050,7 @@ class IColumn; M(Bool, input_format_json_named_tuples_as_objects, true, "Deserialize named tuple columns as JSON objects", 0) \ M(Bool, input_format_json_ignore_unknown_keys_in_named_tuple, true, "Ignore unknown keys in json object for named tuples", 0) \ M(Bool, input_format_json_defaults_for_missing_elements_in_named_tuple, true, "Insert default value in named tuple element if it's missing in json object", 0) \ + M(Bool, input_format_json_throw_on_bad_escape_sequence, true, "Throw an exception if JSON string contains bad escape sequence in JSON input formats. If disabled, bad escape sequences will remain as is in the data", 0) \ M(Bool, input_format_try_infer_integers, true, "Try to infer integers instead of floats while schema inference in text formats", 0) \ M(Bool, input_format_try_infer_dates, true, "Try to infer dates from string fields while schema inference in text formats", 0) \ M(Bool, input_format_try_infer_datetimes, true, "Try to infer datetimes from string fields while schema inference in text formats", 0) \ diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 6b31e9cd249..f43ca154d56 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -85,6 +85,9 @@ namespace SettingsChangesHistory /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) static std::map settings_changes_history = { + {"24.4", {{"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"}, + {"lightweight_deletes_sync", 2, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes"}, + }}, {"24.3", {{"s3_connect_timeout_ms", 1000, 1000, "Introduce new dedicated setting for s3 connection timeout"}, {"allow_experimental_shared_merge_tree", false, true, "The setting is obsolete"}, {"use_page_cache_for_disks_without_file_cache", false, false, "Added userspace page cache"}, @@ -101,10 +104,12 @@ static std::map sett {"filesystem_cache_reserve_space_wait_lock_timeout_milliseconds", 1000, 1000, "Wait time to lock cache for sapce reservation in filesystem cache"}, {"max_parser_backtracks", 0, 1000000, "Limiting the complexity of parsing"}, {"analyzer_compatibility_join_using_top_level_identifier", false, false, "Force to resolve identifier in JOIN USING from projection"}, + {"distributed_insert_skip_read_only_replicas", false, false, "If true, INSERT into Distributed will skip read-only replicas"}, {"keeper_max_retries", 10, 10, "Max retries for general keeper operations"}, {"keeper_retry_initial_backoff_ms", 100, 100, "Initial backoff timeout for general keeper operations"}, {"keeper_retry_max_backoff_ms", 5000, 5000, "Max backoff timeout for general keeper operations"}, {"s3queue_allow_experimental_sharded_mode", false, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten"}, + {"allow_experimental_analyzer", false, true, "Enable analyzer and planner by default."}, {"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", 0.0, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability."}, {"allow_get_client_http_header", false, false, "Introduced a new function."}, {"output_format_pretty_row_numbers", false, true, "It is better for usability."}, diff --git a/src/Core/SettingsFields.cpp b/src/Core/SettingsFields.cpp index 001d3e09dc9..caa8b3fdffd 100644 --- a/src/Core/SettingsFields.cpp +++ b/src/Core/SettingsFields.cpp @@ -575,4 +575,40 @@ void SettingFieldCustom::readBinary(ReadBuffer & in) parseFromString(str); } +SettingFieldNonZeroUInt64::SettingFieldNonZeroUInt64(UInt64 x) : SettingFieldUInt64(x) +{ + checkValueNonZero(); +} + +SettingFieldNonZeroUInt64::SettingFieldNonZeroUInt64(const DB::Field & f) : SettingFieldUInt64(f) +{ + checkValueNonZero(); +} + +SettingFieldNonZeroUInt64 & SettingFieldNonZeroUInt64::operator=(UInt64 x) +{ + SettingFieldUInt64::operator=(x); + checkValueNonZero(); + return *this; +} + +SettingFieldNonZeroUInt64 & SettingFieldNonZeroUInt64::operator=(const DB::Field & f) +{ + SettingFieldUInt64::operator=(f); + checkValueNonZero(); + return *this; +} + +void SettingFieldNonZeroUInt64::parseFromString(const String & str) +{ + SettingFieldUInt64::parseFromString(str); + checkValueNonZero(); +} + +void SettingFieldNonZeroUInt64::checkValueNonZero() const +{ + if (value == 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "A setting's value has to be greater than 0"); +} + } diff --git a/src/Core/SettingsFields.h b/src/Core/SettingsFields.h index cef30bb1916..b39aa52c15d 100644 --- a/src/Core/SettingsFields.h +++ b/src/Core/SettingsFields.h @@ -516,4 +516,19 @@ struct SettingFieldCustom void readBinary(ReadBuffer & in); }; +struct SettingFieldNonZeroUInt64 : public SettingFieldUInt64 +{ +public: + explicit SettingFieldNonZeroUInt64(UInt64 x = 1); + explicit SettingFieldNonZeroUInt64(const Field & f); + + SettingFieldNonZeroUInt64 & operator=(UInt64 x); + SettingFieldNonZeroUInt64 & operator=(const Field & f); + + void parseFromString(const String & str); + +private: + void checkValueNonZero() const; +}; + } diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp index 7fc210a691a..cc22db3969c 100644 --- a/src/Daemon/BaseDaemon.cpp +++ b/src/Daemon/BaseDaemon.cpp @@ -1,9 +1,11 @@ #pragma clang diagnostic ignored "-Wreserved-identifier" +#include +#include +#include +#include #include #include -#include -#include #include #include diff --git a/src/DataTypes/DataTypeDate32.cpp b/src/DataTypes/DataTypeDate32.cpp index 83b1260eb6d..762552bcb4c 100644 --- a/src/DataTypes/DataTypeDate32.cpp +++ b/src/DataTypes/DataTypeDate32.cpp @@ -1,6 +1,8 @@ #include #include #include +#include +#include namespace DB { @@ -14,6 +16,11 @@ SerializationPtr DataTypeDate32::doGetDefaultSerialization() const return std::make_shared(); } +Field DataTypeDate32::getDefault() const +{ + return -static_cast(DateLUT::instance().getDayNumOffsetEpoch()); +} + void registerDataTypeDate32(DataTypeFactory & factory) { factory.registerSimpleDataType( diff --git a/src/DataTypes/DataTypeDate32.h b/src/DataTypes/DataTypeDate32.h index 02e818f10df..65633e7a228 100644 --- a/src/DataTypes/DataTypeDate32.h +++ b/src/DataTypes/DataTypeDate32.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include namespace DB @@ -15,10 +14,7 @@ public: TypeIndex getColumnType() const override { return TypeIndex::Int32; } const char * getFamilyName() const override { return family_name; } - Field getDefault() const override - { - return -static_cast(DateLUT::instance().getDayNumOffsetEpoch()); - } + Field getDefault() const override; bool canBeUsedAsVersion() const override { return true; } bool canBeInsideNullable() const override { return true; } diff --git a/src/DataTypes/ObjectUtils.cpp b/src/DataTypes/ObjectUtils.cpp index 92c87d061bb..99cf092e6cd 100644 --- a/src/DataTypes/ObjectUtils.cpp +++ b/src/DataTypes/ObjectUtils.cpp @@ -1054,6 +1054,14 @@ Field FieldVisitorFoldDimension::operator()(const Array & x) const return res; } +Field FieldVisitorFoldDimension::operator()(const Null & x) const +{ + if (num_dimensions_to_fold == 0) + return x; + + return Array(); +} + void setAllObjectsToDummyTupleType(NamesAndTypesList & columns) { for (auto & column : columns) diff --git a/src/DataTypes/ObjectUtils.h b/src/DataTypes/ObjectUtils.h index 35edea529af..3e3b1b96740 100644 --- a/src/DataTypes/ObjectUtils.h +++ b/src/DataTypes/ObjectUtils.h @@ -149,7 +149,7 @@ public: Field operator()(const Array & x) const; - Field operator()(const Null & x) const { return x; } + Field operator()(const Null & x) const; template Field operator()(const T & x) const diff --git a/src/DataTypes/Serializations/SerializationAggregateFunction.cpp b/src/DataTypes/Serializations/SerializationAggregateFunction.cpp index c9af5d1f838..6f09ed31e22 100644 --- a/src/DataTypes/Serializations/SerializationAggregateFunction.cpp +++ b/src/DataTypes/Serializations/SerializationAggregateFunction.cpp @@ -186,10 +186,10 @@ void SerializationAggregateFunction::serializeTextJSON(const IColumn & column, s } -void SerializationAggregateFunction::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void SerializationAggregateFunction::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { String s; - readJSONString(s, istr); + readJSONString(s, istr, settings.json); deserializeFromString(function, column, s, version); } diff --git a/src/DataTypes/Serializations/SerializationCustomSimpleText.cpp b/src/DataTypes/Serializations/SerializationCustomSimpleText.cpp index abe443cab1b..938fd050173 100644 --- a/src/DataTypes/Serializations/SerializationCustomSimpleText.cpp +++ b/src/DataTypes/Serializations/SerializationCustomSimpleText.cpp @@ -133,14 +133,14 @@ void SerializationCustomSimpleText::serializeTextJSON(const IColumn & column, si void SerializationCustomSimpleText::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { String str; - readJSONString(str, istr); + readJSONString(str, istr, settings.json); deserializeFromString(*this, column, str, settings); } bool SerializationCustomSimpleText::tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { String str; - if (!tryReadJSONStringInto(str, istr)) + if (!tryReadJSONStringInto(str, istr, settings.json)) return false; return tryDeserializeFromString(*this, column, str, settings); } diff --git a/src/DataTypes/Serializations/SerializationEnum.cpp b/src/DataTypes/Serializations/SerializationEnum.cpp index 14b1a33e2ce..d72442eec99 100644 --- a/src/DataTypes/Serializations/SerializationEnum.cpp +++ b/src/DataTypes/Serializations/SerializationEnum.cpp @@ -134,20 +134,20 @@ void SerializationEnum::serializeTextXML(const IColumn & column, size_t ro } template -void SerializationEnum::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void SerializationEnum::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { if (!istr.eof() && *istr.position() != '"') assert_cast(column).getData().push_back(readValue(istr)); else { std::string field_name; - readJSONString(field_name, istr); + readJSONString(field_name, istr, settings.json); assert_cast(column).getData().push_back(ref_enum_values.getValue(StringRef(field_name))); } } template -bool SerializationEnum::tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +bool SerializationEnum::tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { FieldType x; if (!istr.eof() && *istr.position() != '"') @@ -158,7 +158,7 @@ bool SerializationEnum::tryDeserializeTextJSON(IColumn & column, ReadBuffe else { std::string field_name; - readJSONString(field_name, istr); + readJSONString(field_name, istr, settings.json); if (!ref_enum_values.tryGetValue(x, StringRef(field_name))) return false; } diff --git a/src/DataTypes/Serializations/SerializationFixedString.cpp b/src/DataTypes/Serializations/SerializationFixedString.cpp index 23e959d80c9..481ae2a6165 100644 --- a/src/DataTypes/Serializations/SerializationFixedString.cpp +++ b/src/DataTypes/Serializations/SerializationFixedString.cpp @@ -230,14 +230,14 @@ void SerializationFixedString::serializeTextJSON(const IColumn & column, size_t } -void SerializationFixedString::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void SerializationFixedString::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - read(*this, column, [&istr](ColumnFixedString::Chars & data) { readJSONStringInto(data, istr); }); + read(*this, column, [&istr, &settings](ColumnFixedString::Chars & data) { readJSONStringInto(data, istr, settings.json); }); } -bool SerializationFixedString::tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +bool SerializationFixedString::tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - return tryRead(*this, column, [&istr](ColumnFixedString::Chars & data) { return tryReadJSONStringInto(data, istr); }); + return tryRead(*this, column, [&istr, &settings](ColumnFixedString::Chars & data) { return tryReadJSONStringInto(data, istr, settings.json); }); } void SerializationFixedString::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const diff --git a/src/DataTypes/Serializations/SerializationString.cpp b/src/DataTypes/Serializations/SerializationString.cpp index fd46206e9ad..8abaa3bd5ea 100644 --- a/src/DataTypes/Serializations/SerializationString.cpp +++ b/src/DataTypes/Serializations/SerializationString.cpp @@ -389,7 +389,7 @@ void SerializationString::deserializeTextJSON(IColumn & column, ReadBuffer & ist else if (settings.json.read_numbers_as_strings && !istr.eof() && *istr.position() != '"') { String field; - readJSONField(field, istr); + readJSONField(field, istr, settings.json); Float64 tmp; ReadBufferFromString buf(field); if (tryReadFloatText(tmp, buf) && buf.eof()) @@ -398,7 +398,7 @@ void SerializationString::deserializeTextJSON(IColumn & column, ReadBuffer & ist throw Exception(ErrorCodes::INCORRECT_DATA, "Cannot parse JSON String value here: {}", field); } else - read(column, [&](ColumnString::Chars & data) { readJSONStringInto(data, istr); }); + read(column, [&](ColumnString::Chars & data) { readJSONStringInto(data, istr, settings.json); }); } bool SerializationString::tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const @@ -432,7 +432,7 @@ bool SerializationString::tryDeserializeTextJSON(IColumn & column, ReadBuffer & if (settings.json.read_numbers_as_strings && !istr.eof() && *istr.position() != '"') { String field; - if (!tryReadJSONField(field, istr)) + if (!tryReadJSONField(field, istr, settings.json)) return false; Float64 tmp; @@ -446,7 +446,7 @@ bool SerializationString::tryDeserializeTextJSON(IColumn & column, ReadBuffer & return false; } - return read(column, [&](ColumnString::Chars & data) { return tryReadJSONStringInto(data, istr); }); + return read(column, [&](ColumnString::Chars & data) { return tryReadJSONStringInto(data, istr, settings.json); }); } diff --git a/src/DataTypes/Serializations/SerializationTuple.cpp b/src/DataTypes/Serializations/SerializationTuple.cpp index 399ad870d60..632a019d2d9 100644 --- a/src/DataTypes/Serializations/SerializationTuple.cpp +++ b/src/DataTypes/Serializations/SerializationTuple.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -382,8 +383,8 @@ ReturnType SerializationTuple::deserializeTextJSONImpl(IColumn & column, ReadBuf if (settings.json.ignore_unknown_keys_in_named_tuple) { if constexpr (throw_exception) - skipJSONField(istr, name); - else if (!trySkipJSONField(istr, name)) + skipJSONField(istr, name, settings.json); + else if (!trySkipJSONField(istr, name, settings.json)) return false; skipWhitespaceIfAny(istr); @@ -526,68 +527,26 @@ void SerializationTuple::serializeTextXML(const IColumn & column, size_t row_num void SerializationTuple::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const { - for (size_t i = 0; i < elems.size(); ++i) - { - if (i != 0) - writeChar(settings.csv.tuple_delimiter, ostr); - elems[i]->serializeTextCSV(extractElementColumn(column, i), row_num, ostr, settings); - } + WriteBufferFromOwnString wb; + serializeText(column, row_num, wb, settings); + writeCSV(wb.str(), ostr); } void SerializationTuple::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - addElementSafe(elems.size(), column, [&] - { - const size_t size = elems.size(); - for (size_t i = 0; i < size; ++i) - { - if (i != 0) - { - skipWhitespaceIfAny(istr); - assertChar(settings.csv.tuple_delimiter, istr); - skipWhitespaceIfAny(istr); - } - - auto & element_column = extractElementColumn(column, i); - if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(element_column)) - SerializationNullable::deserializeNullAsDefaultOrNestedTextCSV(element_column, istr, settings, elems[i]); - else - elems[i]->deserializeTextCSV(element_column, istr, settings); - } - return true; - }); + String s; + readCSV(s, istr, settings.csv); + ReadBufferFromString rb(s); + deserializeText(column, rb, settings, true); } bool SerializationTuple::tryDeserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { - return addElementSafe(elems.size(), column, [&] - { - const size_t size = elems.size(); - for (size_t i = 0; i < size; ++i) - { - if (i != 0) - { - skipWhitespaceIfAny(istr); - if (!checkChar(settings.csv.tuple_delimiter, istr)) - return false; - skipWhitespaceIfAny(istr); - } - - auto & element_column = extractElementColumn(column, i); - if (settings.null_as_default && !isColumnNullableOrLowCardinalityNullable(element_column)) - { - if (!SerializationNullable::tryDeserializeNullAsDefaultOrNestedTextCSV(element_column, istr, settings, elems[i])) - return false; - } - else - { - if (!elems[i]->tryDeserializeTextCSV(element_column, istr, settings)) - return false; - } - } - - return true; - }); + String s; + if (!tryReadCSV(s, istr, settings.csv)) + return false; + ReadBufferFromString rb(s); + return tryDeserializeText(column, rb, settings, true); } void SerializationTuple::enumerateStreams( diff --git a/src/DataTypes/Serializations/SerializationVariant.cpp b/src/DataTypes/Serializations/SerializationVariant.cpp index 5af94364167..8ca86c63bf6 100644 --- a/src/DataTypes/Serializations/SerializationVariant.cpp +++ b/src/DataTypes/Serializations/SerializationVariant.cpp @@ -800,7 +800,7 @@ void SerializationVariant::serializeTextJSON(const IColumn & column, size_t row_ bool SerializationVariant::tryDeserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { String field; - if (!tryReadJSONField(field, istr)) + if (!tryReadJSONField(field, istr, settings.json)) return false; return tryDeserializeTextJSONImpl(column, field, settings); } @@ -808,7 +808,7 @@ bool SerializationVariant::tryDeserializeTextJSON(IColumn & column, ReadBuffer & void SerializationVariant::deserializeTextJSON(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { String field; - readJSONField(field, istr); + readJSONField(field, istr, settings.json); if (!tryDeserializeTextJSONImpl(column, field, settings)) throw Exception(ErrorCodes::INCORRECT_DATA, "Cannot parse JSON value of type {} here: {}", variant_name, field); } diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 896baa561b4..d8acfb5fa01 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -1,11 +1,16 @@ #include +#include +#include +#include +#include +#include #include #include #include #include -#include #include +#include #include #include #include @@ -16,14 +21,11 @@ #include #include #include -#include -#include -#include #include #include -#include -#include -#include +#include +#include +#include namespace fs = std::filesystem; @@ -75,7 +77,7 @@ std::pair createTableFromAST( auto table_function = factory.get(table_function_ast, context); ColumnsDescription columns; if (ast_create_query.columns_list && ast_create_query.columns_list->columns) - columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, true, false); + columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, mode); StoragePtr storage = table_function->execute(table_function_ast, context, ast_create_query.getTable(), std::move(columns)); storage->renameInMemory(ast_create_query); return {ast_create_query.getTable(), storage}; @@ -107,7 +109,7 @@ std::pair createTableFromAST( } else { - columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, true, false); + columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, mode); constraints = InterpreterCreateQuery::getConstraintsDescription(ast_create_query.columns_list->constraints); } } @@ -613,7 +615,7 @@ void DatabaseOnDisk::iterateMetadataFiles(ContextPtr local_context, const Iterat }; /// Metadata files to load: name and flag for .tmp_drop files - std::set> metadata_files; + std::vector> metadata_files; fs::directory_iterator dir_end; for (fs::directory_iterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it) @@ -634,7 +636,7 @@ void DatabaseOnDisk::iterateMetadataFiles(ContextPtr local_context, const Iterat if (endsWith(file_name, ".sql.tmp_drop")) { /// There are files that we tried to delete previously - metadata_files.emplace(file_name, false); + metadata_files.emplace_back(file_name, false); } else if (endsWith(file_name, ".sql.tmp")) { @@ -645,23 +647,30 @@ void DatabaseOnDisk::iterateMetadataFiles(ContextPtr local_context, const Iterat else if (endsWith(file_name, ".sql")) { /// The required files have names like `table_name.sql` - metadata_files.emplace(file_name, true); + metadata_files.emplace_back(file_name, true); } else throw Exception(ErrorCodes::INCORRECT_FILE_NAME, "Incorrect file extension: {} in metadata directory {}", file_name, getMetadataPath()); } + std::sort(metadata_files.begin(), metadata_files.end()); + metadata_files.erase(std::unique(metadata_files.begin(), metadata_files.end()), metadata_files.end()); + /// Read and parse metadata in parallel ThreadPool pool(CurrentMetrics::DatabaseOnDiskThreads, CurrentMetrics::DatabaseOnDiskThreadsActive, CurrentMetrics::DatabaseOnDiskThreadsScheduled); - for (const auto & file : metadata_files) + const auto batch_size = metadata_files.size() / pool.getMaxThreads() + 1; + for (auto it = metadata_files.begin(); it < metadata_files.end(); std::advance(it, batch_size)) { - pool.scheduleOrThrowOnError([&]() - { - if (file.second) - process_metadata_file(file.first); - else - process_tmp_drop_metadata_file(file.first); - }); + std::span batch{it, std::min(std::next(it, batch_size), metadata_files.end())}; + pool.scheduleOrThrowOnError( + [batch, &process_metadata_file, &process_tmp_drop_metadata_file]() mutable + { + for (const auto & file : batch) + if (file.second) + process_metadata_file(file.first); + else + process_tmp_drop_metadata_file(file.first); + }); } pool.wait(); } diff --git a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp index b44bc136b1f..1c0d5fe3de1 100644 --- a/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabaseMaterializedPostgreSQL.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index f5d67d37b07..1a8d46668e0 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -153,7 +153,6 @@ public: return dir_path / entry->path().filename(); } - String name() const override { return entry->path().filename(); } private: diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp index 8595cc45218..c77709c27eb 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.cpp @@ -87,7 +87,7 @@ SeekableReadBufferPtr ReadBufferFromRemoteFSGather::createImplementationBuffer(c cache_key, settings.remote_fs_cache, FileCache::getCommonUser(), - [=, this]() { return read_buffer_creator(/* restricted_seek */true, object_path); }, + [=, this]() { return read_buffer_creator(/* restricted_seek */true, object); }, settings, query_id, object.bytes_size, @@ -102,14 +102,14 @@ SeekableReadBufferPtr ReadBufferFromRemoteFSGather::createImplementationBuffer(c /// former doesn't support seeks. if (with_page_cache && !buf) { - auto inner = read_buffer_creator(/* restricted_seek */false, object_path); + auto inner = read_buffer_creator(/* restricted_seek */false, object); auto cache_key = FileChunkAddress { .path = cache_path_prefix + object_path }; buf = std::make_unique( cache_key, settings.page_cache, std::move(inner), settings); } if (!buf) - buf = read_buffer_creator(/* restricted_seek */true, object_path); + buf = read_buffer_creator(/* restricted_seek */true, object); if (read_until_position > start_offset && read_until_position < start_offset + object.bytes_size) buf->setReadUntilPosition(read_until_position - start_offset); diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.h b/src/Disks/IO/ReadBufferFromRemoteFSGather.h index 8362b354e23..d5dfa18987a 100644 --- a/src/Disks/IO/ReadBufferFromRemoteFSGather.h +++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.h @@ -21,7 +21,7 @@ class ReadBufferFromRemoteFSGather final : public ReadBufferFromFileBase friend class ReadIndirectBufferFromRemoteFS; public: - using ReadBufferCreator = std::function(bool restricted_seek, const std::string & path)>; + using ReadBufferCreator = std::function(bool restricted_seek, const StoredObject & object)>; ReadBufferFromRemoteFSGather( ReadBufferCreator && read_buffer_creator_, diff --git a/src/Disks/IO/ReadBufferFromWebServer.cpp b/src/Disks/IO/ReadBufferFromWebServer.cpp index 03300cc0714..7c5de8a13de 100644 --- a/src/Disks/IO/ReadBufferFromWebServer.cpp +++ b/src/Disks/IO/ReadBufferFromWebServer.cpp @@ -21,10 +21,11 @@ namespace ErrorCodes ReadBufferFromWebServer::ReadBufferFromWebServer( const String & url_, ContextPtr context_, + size_t file_size_, const ReadSettings & settings_, bool use_external_buffer_, size_t read_until_position_) - : ReadBufferFromFileBase(settings_.remote_fs_buffer_size, nullptr, 0) + : ReadBufferFromFileBase(settings_.remote_fs_buffer_size, nullptr, 0, file_size_) , log(getLogger("ReadBufferFromWebServer")) , context(context_) , url(url_) @@ -36,7 +37,7 @@ ReadBufferFromWebServer::ReadBufferFromWebServer( } -std::unique_ptr ReadBufferFromWebServer::initialize() +std::unique_ptr ReadBufferFromWebServer::initialize() { Poco::URI uri(url); if (read_until_position) @@ -119,9 +120,8 @@ bool ReadBufferFromWebServer::nextImpl() auto result = impl->next(); - BufferBase::set(impl->buffer().begin(), impl->buffer().size(), impl->offset()); - - chassert(working_buffer.begin() == impl->buffer().begin()); + working_buffer = impl->buffer(); + pos = impl->position(); if (result) offset += working_buffer.size(); @@ -132,16 +132,29 @@ bool ReadBufferFromWebServer::nextImpl() off_t ReadBufferFromWebServer::seek(off_t offset_, int whence) { - if (impl) - throw Exception(ErrorCodes::CANNOT_SEEK_THROUGH_FILE, "Seek is allowed only before first read attempt from the buffer"); - if (whence != SEEK_SET) throw Exception(ErrorCodes::CANNOT_SEEK_THROUGH_FILE, "Only SEEK_SET mode is allowed"); if (offset_ < 0) throw Exception(ErrorCodes::SEEK_POSITION_OUT_OF_BOUND, "Seek position is out of bounds. Offset: {}", offset_); - offset = offset_; + if (impl) + { + if (use_external_buffer) + { + impl->set(internal_buffer.begin(), internal_buffer.size()); + } + + impl->seek(offset_, SEEK_SET); + + working_buffer = impl->buffer(); + pos = impl->position(); + offset = offset_ + available(); + } + else + { + offset = offset_; + } return offset; } diff --git a/src/Disks/IO/ReadBufferFromWebServer.h b/src/Disks/IO/ReadBufferFromWebServer.h index 68ad752bbdb..e4f436f2eb3 100644 --- a/src/Disks/IO/ReadBufferFromWebServer.h +++ b/src/Disks/IO/ReadBufferFromWebServer.h @@ -20,6 +20,7 @@ public: explicit ReadBufferFromWebServer( const String & url_, ContextPtr context_, + size_t file_size_, const ReadSettings & settings_ = {}, bool use_external_buffer_ = false, size_t read_until_position = 0); @@ -39,7 +40,7 @@ public: bool supportsRightBoundedReads() const override { return true; } private: - std::unique_ptr initialize(); + std::unique_ptr initialize(); LoggerPtr log; ContextPtr context; @@ -47,7 +48,7 @@ private: const String url; size_t buf_size; - std::unique_ptr impl; + std::unique_ptr impl; ReadSettings read_settings; diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp index b5c2bfab8b9..e0614613c3f 100644 --- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp +++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp @@ -206,11 +206,11 @@ std::unique_ptr AzureObjectStorage::readObjects( /// NOL auto read_buffer_creator = [this, settings_ptr, disk_read_settings] - (bool restricted_seek, const std::string & path) -> std::unique_ptr + (bool restricted_seek, const StoredObject & object_) -> std::unique_ptr { return std::make_unique( client.get(), - path, + object_.remote_path, disk_read_settings, settings_ptr->max_single_read_retries, settings_ptr->max_single_download_retries, diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp index f8545ecfe39..e717c88ed22 100644 --- a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp +++ b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp @@ -60,8 +60,9 @@ std::unique_ptr HDFSObjectStorage::readObjects( /// NOLI auto disk_read_settings = patchSettings(read_settings); auto read_buffer_creator = [this, disk_read_settings] - (bool /* restricted_seek */, const std::string & path) -> std::unique_ptr + (bool /* restricted_seek */, const StoredObject & object_) -> std::unique_ptr { + const auto & path = object_.remote_path; size_t begin_of_path = path.find('/', path.find("//") + 2); auto hdfs_path = path.substr(begin_of_path); auto hdfs_uri = path.substr(0, begin_of_path); diff --git a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp index f82c6156619..d44e17a0713 100644 --- a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp @@ -49,10 +49,10 @@ std::unique_ptr LocalObjectStorage::readObjects( /// NOL auto modified_settings = patchSettings(read_settings); auto global_context = Context::getGlobalContextInstance(); auto read_buffer_creator = - [=] (bool /* restricted_seek */, const std::string & file_path) + [=] (bool /* restricted_seek */, const StoredObject & object) -> std::unique_ptr { - return createReadBufferFromFileBase(file_path, modified_settings, read_hint, file_size); + return createReadBufferFromFileBase(object.remote_path, modified_settings, read_hint, file_size); }; switch (read_settings.remote_fs_method) diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp index bd67f6907b0..b343b73f7bd 100644 --- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp +++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp @@ -172,12 +172,12 @@ std::unique_ptr S3ObjectStorage::readObjects( /// NOLINT auto read_buffer_creator = [this, settings_ptr, disk_read_settings] - (bool restricted_seek, const std::string & path) -> std::unique_ptr + (bool restricted_seek, const StoredObject & object_) -> std::unique_ptr { return std::make_unique( client.get(), uri.bucket, - path, + object_.remote_path, uri.version_id, settings_ptr->request_settings, disk_read_settings, diff --git a/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp b/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp index 7e942a6cf6f..69f6137cd2d 100644 --- a/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp +++ b/src/Disks/ObjectStorages/Web/WebObjectStorage.cpp @@ -250,15 +250,17 @@ std::unique_ptr WebObjectStorage::readObject( /// NOLINT std::optional, std::optional) const { + size_t object_size = object.bytes_size; auto read_buffer_creator = - [this, read_settings] - (bool /* restricted_seek */, const std::string & path_) -> std::unique_ptr + [this, read_settings, object_size] + (bool /* restricted_seek */, const StoredObject & object_) -> std::unique_ptr { - return std::make_unique( - fs::path(url) / path_, - getContext(), - read_settings, - /* use_external_buffer */true); + return std::make_unique( + fs::path(url) / object_.remote_path, + getContext(), + object_size, + read_settings, + /* use_external_buffer */true); }; auto global_context = Context::getGlobalContextInstance(); diff --git a/src/Formats/EscapingRuleUtils.cpp b/src/Formats/EscapingRuleUtils.cpp index 577988871f3..3edade639df 100644 --- a/src/Formats/EscapingRuleUtils.cpp +++ b/src/Formats/EscapingRuleUtils.cpp @@ -85,7 +85,7 @@ void skipFieldByEscapingRule(ReadBuffer & buf, FormatSettings::EscapingRule esca readCSVStringInto(out, buf, format_settings.csv); break; case FormatSettings::EscapingRule::JSON: - skipJSONField(buf, StringRef(field_name, field_name_len)); + skipJSONField(buf, StringRef(field_name, field_name_len), format_settings.json); break; case FormatSettings::EscapingRule::Raw: readStringInto(out, buf); @@ -219,9 +219,9 @@ String readByEscapingRule(ReadBuffer & buf, FormatSettings::EscapingRule escapin break; case FormatSettings::EscapingRule::JSON: if constexpr (read_string) - readJSONString(result, buf); + readJSONString(result, buf, format_settings.json); else - readJSONField(result, buf); + readJSONField(result, buf, format_settings.json); break; case FormatSettings::EscapingRule::Raw: readString(result, buf); @@ -303,8 +303,8 @@ DataTypePtr tryInferDataTypeByEscapingRule(const String & field, const FormatSet /// Try to determine the type of value inside quotes auto type = tryInferDataTypeForSingleField(data, format_settings); - /// If we couldn't infer any type or it's tuple in quotes or it's a number and csv.try_infer_numbers_from_strings = 0, we determine it as a string. - if (!type || isTuple(type) || (isNumber(type) && !format_settings.csv.try_infer_numbers_from_strings)) + /// If we couldn't infer any type or it's a number and csv.try_infer_numbers_from_strings = 0, we determine it as a string. + if (!type || (isNumber(type) && !format_settings.csv.try_infer_numbers_from_strings)) return std::make_shared(); return type; diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index f60f0b676d6..8cbb1b9e563 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -142,6 +142,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se format_settings.json.allow_object_type = context->getSettingsRef().allow_experimental_object_type; format_settings.json.compact_allow_variable_number_of_columns = settings.input_format_json_compact_allow_variable_number_of_columns; format_settings.json.try_infer_objects_as_tuples = settings.input_format_json_try_infer_named_tuples_from_objects; + format_settings.json.throw_on_bad_escape_sequence = settings.input_format_json_throw_on_bad_escape_sequence; format_settings.null_as_default = settings.input_format_null_as_default; format_settings.decimal_trailing_zeros = settings.output_format_decimal_trailing_zeros; format_settings.parquet.row_group_rows = settings.output_format_parquet_row_group_size; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 928c7bb8da0..5b7995e0da2 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -193,7 +193,7 @@ struct FormatSettings bool allow_variable_number_of_columns = false; } custom{}; - struct + struct JSON { bool array_of_rows = false; bool quote_64bit_integers = true; @@ -221,7 +221,7 @@ struct FormatSettings bool compact_allow_variable_number_of_columns = false; bool try_infer_objects_as_tuples = false; bool infer_incomplete_types_as_strings = true; - + bool throw_on_bad_escape_sequence = true; } json{}; struct diff --git a/src/Formats/JSONUtils.cpp b/src/Formats/JSONUtils.cpp index 7d494c1e96f..acea5e7b748 100644 --- a/src/Formats/JSONUtils.cpp +++ b/src/Formats/JSONUtils.cpp @@ -215,7 +215,7 @@ namespace JSONUtils else first = false; - auto name = readFieldName(in); + auto name = readFieldName(in, settings.json); auto type = tryInferDataTypeForSingleJSONField(in, settings, inference_info); names_and_types.emplace_back(name, type); skipWhitespaceIfAny(in); @@ -277,7 +277,7 @@ namespace JSONUtils if (yield_strings) { String str; - readJSONString(str, in); + readJSONString(str, in, format_settings.json); ReadBufferFromString buf(str); @@ -567,34 +567,34 @@ namespace JSONUtils return true; } - String readFieldName(ReadBuffer & in) + String readFieldName(ReadBuffer & in, const FormatSettings::JSON & settings) { skipWhitespaceIfAny(in); String field; - readJSONString(field, in); + readJSONString(field, in, settings); skipColon(in); return field; } - bool tryReadFieldName(ReadBuffer & in, String & field) + bool tryReadFieldName(ReadBuffer & in, String & field, const FormatSettings::JSON & settings) { skipWhitespaceIfAny(in); - return tryReadJSONStringInto(field, in) && checkAndSkipColon(in); + return tryReadJSONStringInto(field, in, settings) && checkAndSkipColon(in); } - String readStringField(ReadBuffer & in) + String readStringField(ReadBuffer & in, const FormatSettings::JSON & settings) { skipWhitespaceIfAny(in); String value; - readJSONString(value, in); + readJSONString(value, in, settings); skipWhitespaceIfAny(in); return value; } - bool tryReadStringField(ReadBuffer & in, String & value) + bool tryReadStringField(ReadBuffer & in, String & value, const FormatSettings::JSON & settings) { skipWhitespaceIfAny(in); - if (!tryReadJSONStringInto(value, in)) + if (!tryReadJSONStringInto(value, in, settings)) return false; skipWhitespaceIfAny(in); return true; @@ -680,24 +680,24 @@ namespace JSONUtils return true; } - std::pair readStringFieldNameAndValue(ReadBuffer & in) + std::pair readStringFieldNameAndValue(ReadBuffer & in, const FormatSettings::JSON & settings) { - auto field_name = readFieldName(in); - auto field_value = readStringField(in); + auto field_name = readFieldName(in, settings); + auto field_value = readStringField(in, settings); return {field_name, field_value}; } - bool tryReadStringFieldNameAndValue(ReadBuffer & in, std::pair & field_and_value) + bool tryReadStringFieldNameAndValue(ReadBuffer & in, std::pair & field_and_value, const FormatSettings::JSON & settings) { - return tryReadFieldName(in, field_and_value.first) && tryReadStringField(in, field_and_value.second); + return tryReadFieldName(in, field_and_value.first, settings) && tryReadStringField(in, field_and_value.second, settings); } - NameAndTypePair readObjectWithNameAndType(ReadBuffer & in) + NameAndTypePair readObjectWithNameAndType(ReadBuffer & in, const FormatSettings::JSON & settings) { skipObjectStart(in); - auto [first_field_name, first_field_value] = readStringFieldNameAndValue(in); + auto [first_field_name, first_field_value] = readStringFieldNameAndValue(in, settings); skipComma(in); - auto [second_field_name, second_field_value] = readStringFieldNameAndValue(in); + auto [second_field_name, second_field_value] = readStringFieldNameAndValue(in, settings); NameAndTypePair name_and_type; if (first_field_name == "name" && second_field_name == "type") @@ -714,20 +714,20 @@ namespace JSONUtils return name_and_type; } - bool tryReadObjectWithNameAndType(ReadBuffer & in, NameAndTypePair & name_and_type) + bool tryReadObjectWithNameAndType(ReadBuffer & in, NameAndTypePair & name_and_type, const FormatSettings::JSON & settings) { if (!checkAndSkipObjectStart(in)) return false; std::pair first_field_and_value; - if (!tryReadStringFieldNameAndValue(in, first_field_and_value)) + if (!tryReadStringFieldNameAndValue(in, first_field_and_value, settings)) return false; if (!checkAndSkipComma(in)) return false; std::pair second_field_and_value; - if (!tryReadStringFieldNameAndValue(in, second_field_and_value)) + if (!tryReadStringFieldNameAndValue(in, second_field_and_value, settings)) return false; if (first_field_and_value.first == "name" && second_field_and_value.first == "type") @@ -752,9 +752,9 @@ namespace JSONUtils return checkAndSkipObjectEnd(in); } - NamesAndTypesList readMetadata(ReadBuffer & in) + NamesAndTypesList readMetadata(ReadBuffer & in, const FormatSettings::JSON & settings) { - auto field_name = readFieldName(in); + auto field_name = readFieldName(in, settings); if (field_name != "meta") throw Exception(ErrorCodes::INCORRECT_DATA, "Expected field \"meta\" with columns names and types, found field {}", field_name); skipArrayStart(in); @@ -767,15 +767,15 @@ namespace JSONUtils else first = false; - names_and_types.push_back(readObjectWithNameAndType(in)); + names_and_types.push_back(readObjectWithNameAndType(in, settings)); } return names_and_types; } - bool tryReadMetadata(ReadBuffer & in, NamesAndTypesList & names_and_types) + bool tryReadMetadata(ReadBuffer & in, NamesAndTypesList & names_and_types, const FormatSettings::JSON & settings) { String field_name; - if (!tryReadFieldName(in, field_name) || field_name != "meta") + if (!tryReadFieldName(in, field_name, settings) || field_name != "meta") return false; if (!checkAndSkipArrayStart(in)) @@ -795,7 +795,7 @@ namespace JSONUtils } NameAndTypePair name_and_type; - if (!tryReadObjectWithNameAndType(in, name_and_type)) + if (!tryReadObjectWithNameAndType(in, name_and_type, settings)) return false; names_and_types.push_back(name_and_type); } @@ -819,18 +819,18 @@ namespace JSONUtils } } - NamesAndTypesList readMetadataAndValidateHeader(ReadBuffer & in, const Block & header) + NamesAndTypesList readMetadataAndValidateHeader(ReadBuffer & in, const Block & header, const FormatSettings::JSON & settings) { - auto names_and_types = JSONUtils::readMetadata(in); + auto names_and_types = JSONUtils::readMetadata(in, settings); validateMetadataByHeader(names_and_types, header); return names_and_types; } - bool skipUntilFieldInObject(ReadBuffer & in, const String & desired_field_name) + bool skipUntilFieldInObject(ReadBuffer & in, const String & desired_field_name, const FormatSettings::JSON & settings) { while (!checkAndSkipObjectEnd(in)) { - auto field_name = JSONUtils::readFieldName(in); + auto field_name = JSONUtils::readFieldName(in, settings); if (field_name == desired_field_name) return true; } @@ -838,14 +838,14 @@ namespace JSONUtils return false; } - void skipTheRestOfObject(ReadBuffer & in) + void skipTheRestOfObject(ReadBuffer & in, const FormatSettings::JSON & settings) { while (!checkAndSkipObjectEnd(in)) { skipComma(in); - auto name = readFieldName(in); + auto name = readFieldName(in, settings); skipWhitespaceIfAny(in); - skipJSONField(in, name); + skipJSONField(in, name, settings); } } diff --git a/src/Formats/JSONUtils.h b/src/Formats/JSONUtils.h index a770ded9687..7ee111c1285 100644 --- a/src/Formats/JSONUtils.h +++ b/src/Formats/JSONUtils.h @@ -13,6 +13,7 @@ namespace DB { +class Block; struct JSONInferenceInfo; namespace JSONUtils @@ -114,7 +115,7 @@ namespace JSONUtils void skipComma(ReadBuffer & in); bool checkAndSkipComma(ReadBuffer & in); - String readFieldName(ReadBuffer & in); + String readFieldName(ReadBuffer & in, const FormatSettings::JSON & settings); void skipArrayStart(ReadBuffer & in); void skipArrayEnd(ReadBuffer & in); @@ -126,13 +127,13 @@ namespace JSONUtils bool checkAndSkipObjectStart(ReadBuffer & in); bool checkAndSkipObjectEnd(ReadBuffer & in); - NamesAndTypesList readMetadata(ReadBuffer & in); - bool tryReadMetadata(ReadBuffer & in, NamesAndTypesList & names_and_types); - NamesAndTypesList readMetadataAndValidateHeader(ReadBuffer & in, const Block & header); + NamesAndTypesList readMetadata(ReadBuffer & in, const FormatSettings::JSON & settings); + bool tryReadMetadata(ReadBuffer & in, NamesAndTypesList & names_and_types, const FormatSettings::JSON & settings); + NamesAndTypesList readMetadataAndValidateHeader(ReadBuffer & in, const Block & header, const FormatSettings::JSON & settings); void validateMetadataByHeader(const NamesAndTypesList & names_and_types_from_metadata, const Block & header); - bool skipUntilFieldInObject(ReadBuffer & in, const String & desired_field_name); - void skipTheRestOfObject(ReadBuffer & in); + bool skipUntilFieldInObject(ReadBuffer & in, const String & desired_field_name, const FormatSettings::JSON & settings); + void skipTheRestOfObject(ReadBuffer & in, const FormatSettings::JSON & settings); } } diff --git a/src/Formats/SchemaInferenceUtils.cpp b/src/Formats/SchemaInferenceUtils.cpp index 0bba7a1f424..02c0aa6dd77 100644 --- a/src/Formats/SchemaInferenceUtils.cpp +++ b/src/Formats/SchemaInferenceUtils.cpp @@ -1016,7 +1016,7 @@ namespace String field; bool ok = true; if constexpr (is_json) - ok = tryReadJSONStringInto(field, buf); + ok = tryReadJSONStringInto(field, buf, settings.json); else ok = tryReadQuotedString(field, buf); @@ -1071,7 +1071,7 @@ namespace first = false; String key; - if (!tryReadJSONStringInto(key, buf)) + if (!tryReadJSONStringInto(key, buf, settings.json)) return false; skipWhitespaceIfAny(buf); diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index d5eb12f3dee..733ae25274e 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -14,8 +14,6 @@ extract_into_parent_list(clickhouse_functions_sources dbms_sources multiMatchAny.cpp checkHyperscanRegexp.cpp array/has.cpp - equals.cpp - notEquals.cpp CastOverloadResolver.cpp ) extract_into_parent_list(clickhouse_functions_headers dbms_headers diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h index 558c309007c..1f683e64cff 100644 --- a/src/Functions/DateTimeTransforms.h +++ b/src/Functions/DateTimeTransforms.h @@ -22,8 +22,9 @@ namespace DB { -static constexpr auto microsecond_multiplier = 1000000; -static constexpr auto millisecond_multiplier = 1000; +static constexpr auto millisecond_multiplier = 1'000; +static constexpr auto microsecond_multiplier = 1'000'000; +static constexpr auto nanosecond_multiplier = 1'000'000'000; static constexpr FormatSettings::DateTimeOverflowBehavior default_date_time_overflow_behavior = FormatSettings::DateTimeOverflowBehavior::Ignore; @@ -1526,7 +1527,7 @@ struct ToMillisecondImpl static UInt16 execute(const DateTime64 & datetime64, Int64 scale_multiplier, const DateLUTImpl & time_zone) { - return time_zone.toMillisecond(datetime64, scale_multiplier); + return time_zone.toMillisecond(datetime64, scale_multiplier); } static UInt16 execute(UInt32, const DateLUTImpl &) @@ -1903,9 +1904,10 @@ struct ToRelativeSubsecondNumImpl { static constexpr auto name = "toRelativeSubsecondNumImpl"; - static Int64 execute(const DateTime64 & t, DateTime64::NativeType scale, const DateLUTImpl &) + static Int64 execute(const DateTime64 & t, const DateTime64::NativeType scale, const DateLUTImpl &) { - static_assert(scale_multiplier == 1000 || scale_multiplier == 1000000); + static_assert( + scale_multiplier == millisecond_multiplier || scale_multiplier == microsecond_multiplier || scale_multiplier == nanosecond_multiplier); if (scale == scale_multiplier) return t.value; if (scale > scale_multiplier) @@ -2031,13 +2033,14 @@ struct DateTimeComponentsWithFractionalPart : public DateLUTImpl::DateTimeCompon { UInt16 millisecond; UInt16 microsecond; + UInt16 nanosecond; }; struct ToDateTimeComponentsImpl { static constexpr auto name = "toDateTimeComponents"; - static DateTimeComponentsWithFractionalPart execute(const DateTime64 & t, DateTime64::NativeType scale_multiplier, const DateLUTImpl & time_zone) + static DateTimeComponentsWithFractionalPart execute(const DateTime64 & t, const DateTime64::NativeType scale_multiplier, const DateLUTImpl & time_zone) { auto components = DecimalUtils::splitWithScaleMultiplier(t, scale_multiplier); @@ -2046,28 +2049,33 @@ struct ToDateTimeComponentsImpl components.fractional = scale_multiplier + (components.whole ? Int64(-1) : Int64(1)) * components.fractional; --components.whole; } - Int64 fractional = components.fractional; - if (scale_multiplier > microsecond_multiplier) - fractional = fractional / (scale_multiplier / microsecond_multiplier); - else if (scale_multiplier < microsecond_multiplier) - fractional = fractional * (microsecond_multiplier / scale_multiplier); - constexpr Int64 divider = microsecond_multiplier/ millisecond_multiplier; - UInt16 millisecond = static_cast(fractional / divider); - UInt16 microsecond = static_cast(fractional % divider); - return DateTimeComponentsWithFractionalPart{time_zone.toDateTimeComponents(components.whole), millisecond, microsecond}; + // Normalize the dividers between microseconds and nanoseconds w.r.t. the scale. + Int64 microsecond_divider = (millisecond_multiplier * scale_multiplier) / microsecond_multiplier; + Int64 nanosecond_divider = scale_multiplier / microsecond_multiplier; + + // Protect against division by zero for smaller scale multipliers. + microsecond_divider = (microsecond_divider ? microsecond_divider : 1); + nanosecond_divider = (nanosecond_divider ? nanosecond_divider : 1); + + const Int64 & fractional = components.fractional; + UInt16 millisecond = static_cast(fractional / microsecond_divider); + UInt16 microsecond = static_cast((fractional % microsecond_divider) / nanosecond_divider); + UInt16 nanosecond = static_cast(fractional % nanosecond_divider); + + return DateTimeComponentsWithFractionalPart{time_zone.toDateTimeComponents(components.whole), millisecond, microsecond, nanosecond}; } static DateTimeComponentsWithFractionalPart execute(UInt32 t, const DateLUTImpl & time_zone) { - return DateTimeComponentsWithFractionalPart{time_zone.toDateTimeComponents(static_cast(t)), 0, 0}; + return DateTimeComponentsWithFractionalPart{time_zone.toDateTimeComponents(static_cast(t)), 0, 0, 0}; } static DateTimeComponentsWithFractionalPart execute(Int32 d, const DateLUTImpl & time_zone) { - return DateTimeComponentsWithFractionalPart{time_zone.toDateTimeComponents(ExtendedDayNum(d)), 0, 0}; + return DateTimeComponentsWithFractionalPart{time_zone.toDateTimeComponents(ExtendedDayNum(d)), 0, 0, 0}; } static DateTimeComponentsWithFractionalPart execute(UInt16 d, const DateLUTImpl & time_zone) { - return DateTimeComponentsWithFractionalPart{time_zone.toDateTimeComponents(DayNum(d)), 0, 0}; + return DateTimeComponentsWithFractionalPart{time_zone.toDateTimeComponents(DayNum(d)), 0, 0, 0}; } using FactorTransform = ZeroTransform; diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index 60c069f632c..844b09224c7 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -1,66 +1,67 @@ #include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include #include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include #include -#include -#include -#include +#include +#include #include +#include +#include +#include +#include +#include +#include #include +#include +#include +#include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include +#include +#include +#include +#include +#include +#include #include -#include -#include +#include +#include +#include +#include +#include #include +#include +#include +#include #include -#include #include -#include +#include +#include namespace DB diff --git a/src/Functions/FunctionsJSON.h b/src/Functions/FunctionsJSON.h index 8a193785f87..9e824fabc42 100644 --- a/src/Functions/FunctionsJSON.h +++ b/src/Functions/FunctionsJSON.h @@ -257,7 +257,7 @@ private: } case MoveType::Key: { - key = (*arguments[j + 1].column).getDataAt(row).toView(); + key = arguments[j + 1].column->getDataAt(row).toView(); if (!moveToElementByKey(res_element, key)) return false; break; @@ -334,6 +334,26 @@ private: }; +template +class JSONExtractImpl; + +template +class JSONExtractKeysAndValuesImpl; + +/** +* Functions JSONExtract and JSONExtractKeysAndValues force the return type - it is specified in the last argument. +* For example - `SELECT JSONExtract(materialize('{"a": 131231, "b": 1234}'), 'b', 'LowCardinality(FixedString(4))')` +* But by default ClickHouse decides on its own whether the return type will be LowCardinality based on the types of +* input arguments. +* And for these specific functions we cannot rely on this mechanism, so these functions have their own implementation - +* just convert all of the LowCardinality input columns to full ones, execute and wrap the resulting column in LowCardinality +* if needed. +*/ +template typename Impl> +constexpr bool functionForcesTheReturnType() +{ + return std::is_same_v, JSONExtractImpl> || std::is_same_v, JSONExtractKeysAndValuesImpl>; +} template typename Impl> class ExecutableFunctionJSON : public IExecutableFunction @@ -348,18 +368,50 @@ public: String getName() const override { return Name::name; } bool useDefaultImplementationForNulls() const override { return false; } bool useDefaultImplementationForConstants() const override { return true; } - bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } + bool useDefaultImplementationForLowCardinalityColumns() const override + { + return !functionForcesTheReturnType(); + } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { if (null_presence.has_null_constant) return result_type->createColumnConstWithDefaultValue(input_rows_count); - ColumnsWithTypeAndName temporary_columns = null_presence.has_nullable ? createBlockWithNestedColumns(arguments) : arguments; - ColumnPtr temporary_result = chooseAndRunJSONParser(temporary_columns, json_return_type, input_rows_count); - if (null_presence.has_nullable) - return wrapInNullable(temporary_result, arguments, result_type, input_rows_count); - return temporary_result; + if constexpr (functionForcesTheReturnType()) + { + ColumnsWithTypeAndName columns_without_low_cardinality = arguments; + + for (auto & column : columns_without_low_cardinality) + { + column.column = recursiveRemoveLowCardinality(column.column); + column.type = recursiveRemoveLowCardinality(column.type); + } + + ColumnsWithTypeAndName temporary_columns = null_presence.has_nullable ? createBlockWithNestedColumns(columns_without_low_cardinality) : columns_without_low_cardinality; + ColumnPtr temporary_result = chooseAndRunJSONParser(temporary_columns, json_return_type, input_rows_count); + + if (null_presence.has_nullable) + temporary_result = wrapInNullable(temporary_result, columns_without_low_cardinality, result_type, input_rows_count); + + if (result_type->lowCardinality()) + temporary_result = recursiveLowCardinalityTypeConversion(temporary_result, json_return_type, result_type); + + return temporary_result; + } + else + { + ColumnsWithTypeAndName temporary_columns = null_presence.has_nullable ? createBlockWithNestedColumns(arguments) : arguments; + ColumnPtr temporary_result = chooseAndRunJSONParser(temporary_columns, json_return_type, input_rows_count); + + if (null_presence.has_nullable) + temporary_result = wrapInNullable(temporary_result, arguments, result_type, input_rows_count); + + if (result_type->lowCardinality()) + temporary_result = recursiveLowCardinalityTypeConversion(temporary_result, json_return_type, result_type); + + return temporary_result; + } } private: @@ -430,7 +482,6 @@ private: DataTypePtr json_return_type; }; - /// We use IFunctionOverloadResolver instead of IFunction to handle non-default NULL processing. /// Both NULL and JSON NULL should generate NULL value. If any argument is NULL, return NULL. template typename Impl> @@ -451,6 +502,10 @@ public: bool isVariadic() const override { return true; } size_t getNumberOfArguments() const override { return 0; } bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForLowCardinalityColumns() const override + { + return !functionForcesTheReturnType(); + } FunctionBasePtr build(const ColumnsWithTypeAndName & arguments) const override { @@ -470,6 +525,9 @@ public: else return_type = json_return_type; + /// Top-level LowCardinality columns are processed outside JSON parser. + json_return_type = removeLowCardinality(json_return_type); + DataTypes argument_types; argument_types.reserve(arguments.size()); for (const auto & argument : arguments) @@ -479,7 +537,6 @@ public: } }; - struct NameJSONHas { static constexpr auto name{"JSONHas"}; }; struct NameIsValidJSON { static constexpr auto name{"isValidJSON"}; }; struct NameJSONLength { static constexpr auto name{"JSONLength"}; }; @@ -865,9 +922,11 @@ struct JSONExtractTree explicit LowCardinalityFixedStringNode(const size_t fixed_length_) : fixed_length(fixed_length_) { } bool insertResultToColumn(IColumn & dest, const Element & element) override { - // For types other than string, delegate the insertion to JSONExtractRawImpl. - if (!element.isString()) + // If element is an object we delegate the insertion to JSONExtractRawImpl + if (element.isObject()) return JSONExtractRawImpl::insertResultToLowCardinalityFixedStringColumn(dest, element, fixed_length); + else if (!element.isString()) + return false; auto str = element.getString(); if (str.size() > fixed_length) @@ -1482,6 +1541,9 @@ public: // We use insertResultToLowCardinalityFixedStringColumn in case we are inserting raw data in a Low Cardinality FixedString column static bool insertResultToLowCardinalityFixedStringColumn(IColumn & dest, const Element & element, size_t fixed_length) { + if (element.getObject().size() > fixed_length) + return false; + ColumnFixedString::Chars chars; WriteBufferFromVector buf(chars, AppendModeTag()); traverse(element, buf); diff --git a/src/Functions/FunctionsTimeWindow.h b/src/Functions/FunctionsTimeWindow.h index d52b76bec91..4532286830d 100644 --- a/src/Functions/FunctionsTimeWindow.h +++ b/src/Functions/FunctionsTimeWindow.h @@ -1,8 +1,9 @@ #pragma once -#include #include #include +#include +#include namespace DB diff --git a/src/Functions/dateDiff.cpp b/src/Functions/dateDiff.cpp index 8f318d1bc55..8e8865db7ed 100644 --- a/src/Functions/dateDiff.cpp +++ b/src/Functions/dateDiff.cpp @@ -177,10 +177,10 @@ public: DateTimeComponentsWithFractionalPart a_comp; DateTimeComponentsWithFractionalPart b_comp; Int64 adjust_value; - auto x_microseconds = TransformDateTime64>(transform_x.getScaleMultiplier()).execute(x, timezone_x); - auto y_microseconds = TransformDateTime64>(transform_y.getScaleMultiplier()).execute(y, timezone_y); + auto x_nanoseconds = TransformDateTime64>(transform_x.getScaleMultiplier()).execute(x, timezone_x); + auto y_nanoseconds = TransformDateTime64>(transform_y.getScaleMultiplier()).execute(y, timezone_y); - if (x_microseconds <= y_microseconds) + if (x_nanoseconds <= y_nanoseconds) { a_comp = TransformDateTime64(transform_x.getScaleMultiplier()).execute(x, timezone_x); b_comp = TransformDateTime64(transform_y.getScaleMultiplier()).execute(y, timezone_y); @@ -193,7 +193,6 @@ public: adjust_value = 1; } - if constexpr (std::is_same_v>>) { if ((a_comp.date.month > b_comp.date.month) @@ -202,7 +201,8 @@ public: || ((a_comp.time.hour == b_comp.time.hour) && ((a_comp.time.minute > b_comp.time.minute) || ((a_comp.time.minute == b_comp.time.minute) && ((a_comp.time.second > b_comp.time.second) || ((a_comp.time.second == b_comp.time.second) && ((a_comp.millisecond > b_comp.millisecond) - || ((a_comp.millisecond == b_comp.millisecond) && (a_comp.microsecond > b_comp.microsecond))))))))))))) + || ((a_comp.millisecond == b_comp.millisecond) && ((a_comp.microsecond > b_comp.microsecond) + || ((a_comp.microsecond == b_comp.microsecond) && (a_comp.nanosecond > b_comp.nanosecond))))))))))))))) res += adjust_value; } else if constexpr (std::is_same_v>>) @@ -215,7 +215,8 @@ public: || ((a_comp.time.hour == b_comp.time.hour) && ((a_comp.time.minute > b_comp.time.minute) || ((a_comp.time.minute == b_comp.time.minute) && ((a_comp.time.second > b_comp.time.second) || ((a_comp.time.second == b_comp.time.second) && ((a_comp.millisecond > b_comp.millisecond) - || ((a_comp.millisecond == b_comp.millisecond) && (a_comp.microsecond > b_comp.microsecond))))))))))))) + || ((a_comp.millisecond == b_comp.millisecond) && ((a_comp.microsecond > b_comp.microsecond) + || ((a_comp.microsecond == b_comp.microsecond) && (a_comp.nanosecond > b_comp.nanosecond))))))))))))))) res += adjust_value; } else if constexpr (std::is_same_v>>) @@ -225,7 +226,8 @@ public: || ((a_comp.time.hour == b_comp.time.hour) && ((a_comp.time.minute > b_comp.time.minute) || ((a_comp.time.minute == b_comp.time.minute) && ((a_comp.time.second > b_comp.time.second) || ((a_comp.time.second == b_comp.time.second) && ((a_comp.millisecond > b_comp.millisecond) - || ((a_comp.millisecond == b_comp.millisecond) && (a_comp.microsecond > b_comp.microsecond))))))))))) + || ((a_comp.millisecond == b_comp.millisecond) && ((a_comp.microsecond > b_comp.microsecond) + || ((a_comp.microsecond == b_comp.microsecond) && (a_comp.nanosecond > b_comp.nanosecond))))))))))))) res += adjust_value; } else if constexpr (std::is_same_v>>) @@ -237,7 +239,8 @@ public: || ((a_comp.time.hour == b_comp.time.hour) && ((a_comp.time.minute > b_comp.time.minute) || ((a_comp.time.minute == b_comp.time.minute) && ((a_comp.time.second > b_comp.time.second) || ((a_comp.time.second == b_comp.time.second) && ((a_comp.millisecond > b_comp.millisecond) - || ((a_comp.millisecond == b_comp.millisecond) && (a_comp.microsecond > b_comp.microsecond))))))))) + || ((a_comp.millisecond == b_comp.millisecond) && ((a_comp.microsecond > b_comp.microsecond) + || ((a_comp.microsecond == b_comp.microsecond) && (a_comp.nanosecond > b_comp.nanosecond))))))))))) res += adjust_value; } else if constexpr (std::is_same_v>>) @@ -246,7 +249,8 @@ public: || ((a_comp.time.hour == b_comp.time.hour) && ((a_comp.time.minute > b_comp.time.minute) || ((a_comp.time.minute == b_comp.time.minute) && ((a_comp.time.second > b_comp.time.second) || ((a_comp.time.second == b_comp.time.second) && ((a_comp.millisecond > b_comp.millisecond) - || ((a_comp.millisecond == b_comp.millisecond) && (a_comp.microsecond > b_comp.microsecond))))))))) + || ((a_comp.millisecond == b_comp.millisecond) && ((a_comp.microsecond > b_comp.microsecond) + || ((a_comp.microsecond == b_comp.microsecond) && (a_comp.nanosecond > b_comp.nanosecond))))))))))) res += adjust_value; } else if constexpr (std::is_same_v>>) @@ -254,25 +258,34 @@ public: if ((a_comp.time.minute > b_comp.time.minute) || ((a_comp.time.minute == b_comp.time.minute) && ((a_comp.time.second > b_comp.time.second) || ((a_comp.time.second == b_comp.time.second) && ((a_comp.millisecond > b_comp.millisecond) - || ((a_comp.millisecond == b_comp.millisecond) && (a_comp.microsecond > b_comp.microsecond))))))) + || ((a_comp.millisecond == b_comp.millisecond) && ((a_comp.microsecond > b_comp.microsecond) + || ((a_comp.microsecond == b_comp.microsecond) && (a_comp.nanosecond > b_comp.nanosecond))))))))) res += adjust_value; } else if constexpr (std::is_same_v>>) { if ((a_comp.time.second > b_comp.time.second) || ((a_comp.time.second == b_comp.time.second) && ((a_comp.millisecond > b_comp.millisecond) - || ((a_comp.millisecond == b_comp.millisecond) && (a_comp.microsecond > b_comp.microsecond))))) + || ((a_comp.millisecond == b_comp.millisecond) && ((a_comp.microsecond > b_comp.microsecond) + || ((a_comp.microsecond == b_comp.microsecond) && (a_comp.nanosecond > b_comp.nanosecond))))))) res += adjust_value; } else if constexpr (std::is_same_v>>) { if ((a_comp.millisecond > b_comp.millisecond) - || ((a_comp.millisecond == b_comp.millisecond) && (a_comp.microsecond > b_comp.microsecond))) + || ((a_comp.millisecond == b_comp.millisecond) && ((a_comp.microsecond > b_comp.microsecond) + || ((a_comp.microsecond == b_comp.microsecond) && (a_comp.nanosecond > b_comp.nanosecond))))) res += adjust_value; } - else if constexpr (std::is_same_v>>) + else if constexpr (std::is_same_v>>) { - if (a_comp.microsecond > b_comp.microsecond) + if ((a_comp.microsecond > b_comp.microsecond) + || ((a_comp.microsecond == b_comp.microsecond) && (a_comp.nanosecond > b_comp.nanosecond))) + res += adjust_value; + } + else if constexpr (std::is_same_v>>) + { + if (a_comp.nanosecond > b_comp.nanosecond) res += adjust_value; } return res; @@ -401,6 +414,8 @@ public: impl.template dispatchForColumns>(x, y, timezone_x, timezone_y, res->getData()); else if (unit == "microsecond" || unit == "microseconds" || unit == "us" || unit == "u") impl.template dispatchForColumns>(x, y, timezone_x, timezone_y, res->getData()); + else if (unit == "nanosecond" || unit == "nanoseconds" || unit == "ns") + impl.template dispatchForColumns>(x, y, timezone_x, timezone_y, res->getData()); else throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function {} does not support '{}' unit", getName(), unit); diff --git a/src/Functions/date_trunc.cpp b/src/Functions/date_trunc.cpp index 8493df17a2f..de5e71e09a8 100644 --- a/src/Functions/date_trunc.cpp +++ b/src/Functions/date_trunc.cpp @@ -56,6 +56,10 @@ public: if (!IntervalKind::tryParseString(datepart_param, datepart_kind)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "{} doesn't look like datepart name in {}", datepart_param, getName()); + if (datepart_kind == IntervalKind::Kind::Nanosecond || datepart_kind == IntervalKind::Kind::Microsecond + || datepart_kind == IntervalKind::Kind::Millisecond) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "{} doesn't support {}", getName(), datepart_param); + result_type_is_date = (datepart_kind == IntervalKind::Kind::Year) || (datepart_kind == IntervalKind::Kind::Quarter) || (datepart_kind == IntervalKind::Kind::Month) || (datepart_kind == IntervalKind::Kind::Week); diff --git a/src/Functions/equals.cpp b/src/Functions/equals.cpp index 512abaa6fc7..5c59daf0537 100644 --- a/src/Functions/equals.cpp +++ b/src/Functions/equals.cpp @@ -13,11 +13,6 @@ REGISTER_FUNCTION(Equals) factory.registerFunction(); } -FunctionOverloadResolverPtr createInternalFunctionEqualOverloadResolver(bool decimal_check_overflow) -{ - return std::make_unique(std::make_shared(decimal_check_overflow)); -} - template <> ColumnPtr FunctionComparison::executeTupleImpl( const ColumnsWithTypeAndName & x, const ColumnsWithTypeAndName & y, size_t tuple_size, size_t input_rows_count) const diff --git a/src/Functions/equals.h b/src/Functions/equals.h deleted file mode 100644 index 855cba4db3e..00000000000 --- a/src/Functions/equals.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once -#include - -namespace DB -{ - -class IFunctionOverloadResolver; -using FunctionOverloadResolverPtr = std::shared_ptr; - -FunctionOverloadResolverPtr createInternalFunctionEqualOverloadResolver(bool decimal_check_overflow); -} diff --git a/src/Functions/extractTimeZoneFromFunctionArguments.cpp b/src/Functions/extractTimeZoneFromFunctionArguments.cpp index 7168c68c9c9..cb8a834ed3b 100644 --- a/src/Functions/extractTimeZoneFromFunctionArguments.cpp +++ b/src/Functions/extractTimeZoneFromFunctionArguments.cpp @@ -1,10 +1,11 @@ -#include -#include +#include #include #include #include -#include +#include +#include #include +#include namespace DB diff --git a/src/Functions/makeDate.cpp b/src/Functions/makeDate.cpp index c7f3c195578..8794283a856 100644 --- a/src/Functions/makeDate.cpp +++ b/src/Functions/makeDate.cpp @@ -13,6 +13,7 @@ #include #include +#include #include #include diff --git a/src/Functions/materialize.h b/src/Functions/materialize.h index 73bfdec48ab..41994509745 100644 --- a/src/Functions/materialize.h +++ b/src/Functions/materialize.h @@ -36,6 +36,8 @@ public: bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } + bool isSuitableForConstantFolding() const override { return false; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } size_t getNumberOfArguments() const override diff --git a/src/Functions/notEquals.cpp b/src/Functions/notEquals.cpp index 744a0997d95..3a63db46711 100644 --- a/src/Functions/notEquals.cpp +++ b/src/Functions/notEquals.cpp @@ -12,11 +12,6 @@ REGISTER_FUNCTION(NotEquals) factory.registerFunction(); } -FunctionOverloadResolverPtr createInternalFunctionNotEqualOverloadResolver(bool decimal_check_overflow) -{ - return std::make_unique(std::make_shared(decimal_check_overflow)); -} - template <> ColumnPtr FunctionComparison::executeTupleImpl( const ColumnsWithTypeAndName & x, const ColumnsWithTypeAndName & y, size_t tuple_size, size_t input_rows_count) const diff --git a/src/Functions/notEquals.h b/src/Functions/notEquals.h deleted file mode 100644 index 961889d68d7..00000000000 --- a/src/Functions/notEquals.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once -#include - -namespace DB -{ - -class IFunctionOverloadResolver; -using FunctionOverloadResolverPtr = std::shared_ptr; - -FunctionOverloadResolverPtr createInternalFunctionNotEqualOverloadResolver(bool decimal_check_overflow); -} diff --git a/src/Functions/today.cpp b/src/Functions/today.cpp index 16a5b98d7ec..356660fa7b5 100644 --- a/src/Functions/today.cpp +++ b/src/Functions/today.cpp @@ -1,11 +1,9 @@ -#include - #include - #include - -#include #include +#include +#include +#include namespace DB diff --git a/src/Functions/visitParamExtractString.cpp b/src/Functions/visitParamExtractString.cpp index 8dae10638f8..92af725eb90 100644 --- a/src/Functions/visitParamExtractString.cpp +++ b/src/Functions/visitParamExtractString.cpp @@ -12,9 +12,11 @@ struct ExtractString { size_t old_size = res_data.size(); ReadBufferFromMemory in(pos, end - pos); - if (!tryReadJSONStringInto(res_data, in)) + if (!tryReadJSONStringInto(res_data, in, default_json_settings)) res_data.resize(old_size); } + + static const FormatSettings::JSON constexpr default_json_settings; }; struct NameSimpleJSONExtractString { static constexpr auto name = "simpleJSONExtractString"; }; diff --git a/src/Functions/yesterday.cpp b/src/Functions/yesterday.cpp index 43832c1faaa..4d79f1fef79 100644 --- a/src/Functions/yesterday.cpp +++ b/src/Functions/yesterday.cpp @@ -1,11 +1,9 @@ -#include - #include - #include - -#include #include +#include +#include +#include namespace DB diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index bcfe5fd5230..68b61e96c51 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -380,7 +380,7 @@ bool parseComplexEscapeSequence(String & s, ReadBuffer & buf) } template -static ReturnType parseJSONEscapeSequence(Vector & s, ReadBuffer & buf) +static ReturnType parseJSONEscapeSequence(Vector & s, ReadBuffer & buf, bool keep_bad_sequences) { static constexpr bool throw_exception = std::is_same_v; @@ -394,7 +394,11 @@ static ReturnType parseJSONEscapeSequence(Vector & s, ReadBuffer & buf) ++buf.position(); if (buf.eof()) - return error("Cannot parse escape sequence", ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE); + { + if (keep_bad_sequences) + return ReturnType(true); + return error("Cannot parse escape sequence: unexpected eof", ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE); + } assert(buf.hasPendingData()); @@ -429,8 +433,32 @@ static ReturnType parseJSONEscapeSequence(Vector & s, ReadBuffer & buf) ++buf.position(); char hex_code[4]; - if (4 != buf.read(hex_code, 4)) - return error("Cannot parse escape sequence: less than four bytes after \\u", ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE); + if (keep_bad_sequences) + { + for (size_t i = 0; i != 4; ++i) + { + if (buf.eof() || *buf.position() == '"') + { + /// Save initial data without parsing of escape sequence. + s.push_back('\\'); + s.push_back('u'); + for (size_t j = 0; j != i; ++j) + s.push_back(hex_code[j]); + return ReturnType(true); + } + + hex_code[i] = *buf.position(); + ++buf.position(); + } + } + else + { + if (4 != buf.read(hex_code, 4)) + return error( + "Cannot parse escape sequence: less than four bytes after \\u. In JSON input formats you can disable setting " + "input_format_json_throw_on_bad_escape_sequence to save bad escape sequences as is", + ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE); + } /// \u0000 - special case if (0 == memcmp(hex_code, "0000", 4)) @@ -455,13 +483,70 @@ static ReturnType parseJSONEscapeSequence(Vector & s, ReadBuffer & buf) /// Surrogate pair. if (code_point >= 0xD800 && code_point <= 0xDBFF) { - if (!checkString("\\u", buf)) - return error("Cannot parse escape sequence: missing second part of surrogate pair", ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE); + auto restore_first_unicode = [&]() + { + s.push_back('\\'); + s.push_back('u'); + for (char & c : hex_code) + s.push_back(c); + }; + + if (keep_bad_sequences) + { + if (buf.eof() || *buf.position() != '\\') + { + restore_first_unicode(); + return ReturnType(true); + } + + ++buf.position(); + if (buf.eof() || *buf.position() != 'u') + { + restore_first_unicode(); + s.push_back('\\'); + return ReturnType(true); + } + + ++buf.position(); + } + else + { + if (!checkString("\\u", buf)) + return error( + "Cannot parse escape sequence: missing second part of surrogate pair. In JSON input formats you can " + "disable setting input_format_json_throw_on_bad_escape_sequence to save bad escape sequences as is", + ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE); + } char second_hex_code[4]; - if (4 != buf.read(second_hex_code, 4)) - return error("Cannot parse escape sequence: less than four bytes after \\u of second part of surrogate pair", - ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE); + if (keep_bad_sequences) + { + for (size_t i = 0; i != 4; ++i) + { + if (buf.eof() || *buf.position() == '"') + { + /// Save initial data without parsing of escape sequence. + restore_first_unicode(); + s.push_back('\\'); + s.push_back('u'); + for (size_t j = 0; j != i; ++j) + s.push_back(second_hex_code[j]); + return ReturnType(true); + } + + second_hex_code[i] = *buf.position(); + ++buf.position(); + } + } + else + { + if (4 != buf.read(second_hex_code, 4)) + return error( + "Cannot parse escape sequence: less than four bytes after \\u of second part of surrogate pair. In JSON " + "input formats you can disable setting input_format_json_throw_on_bad_escape_sequence to save bad escape " + "sequences as is", + ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE); + } UInt16 second_code_point = unhex4(second_hex_code); @@ -475,7 +560,21 @@ static ReturnType parseJSONEscapeSequence(Vector & s, ReadBuffer & buf) s.push_back((full_code_point & 0x3F) | 0x80); } else - return error("Incorrect surrogate pair of unicode escape sequences in JSON", ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE); + { + if (!keep_bad_sequences) + return error( + "Incorrect surrogate pair of unicode escape sequences in JSON. In JSON input formats you can disable " + "setting input_format_json_throw_on_bad_escape_sequence to save bad escape sequences as is", + ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE); + + /// Save initial data without parsing of escape sequence. + restore_first_unicode(); + s.push_back('\\'); + s.push_back('u'); + for (char & c : second_hex_code) + s.push_back(c); + return ReturnType(true); + } } else { @@ -1000,7 +1099,7 @@ template void readCSVStringInto, false, false>(PaddedPODAr template -ReturnType readJSONStringInto(Vector & s, ReadBuffer & buf) +ReturnType readJSONStringInto(Vector & s, ReadBuffer & buf, const FormatSettings::JSON & settings) { static constexpr bool throw_exception = std::is_same_v; @@ -1032,23 +1131,23 @@ ReturnType readJSONStringInto(Vector & s, ReadBuffer & buf) } if (*buf.position() == '\\') - parseJSONEscapeSequence(s, buf); + parseJSONEscapeSequence(s, buf, !settings.throw_on_bad_escape_sequence); } return error("Cannot parse JSON string: expected closing quote", ErrorCodes::CANNOT_PARSE_QUOTED_STRING); } -void readJSONString(String & s, ReadBuffer & buf) +void readJSONString(String & s, ReadBuffer & buf, const FormatSettings::JSON & settings) { s.clear(); - readJSONStringInto(s, buf); + readJSONStringInto(s, buf, settings); } -template void readJSONStringInto, void>(PaddedPODArray & s, ReadBuffer & buf); -template bool readJSONStringInto, bool>(PaddedPODArray & s, ReadBuffer & buf); -template void readJSONStringInto(NullOutput & s, ReadBuffer & buf); -template void readJSONStringInto(String & s, ReadBuffer & buf); -template bool readJSONStringInto(String & s, ReadBuffer & buf); +template void readJSONStringInto, void>(PaddedPODArray & s, ReadBuffer & buf, const FormatSettings::JSON & settings); +template bool readJSONStringInto, bool>(PaddedPODArray & s, ReadBuffer & buf, const FormatSettings::JSON & settings); +template void readJSONStringInto(NullOutput & s, ReadBuffer & buf, const FormatSettings::JSON & settings); +template void readJSONStringInto(String & s, ReadBuffer & buf, const FormatSettings::JSON & settings); +template bool readJSONStringInto(String & s, ReadBuffer & buf, const FormatSettings::JSON & settings); template ReturnType readJSONObjectOrArrayPossiblyInvalid(Vector & s, ReadBuffer & buf) @@ -1356,7 +1455,7 @@ template bool readDateTimeTextFallback(time_t &, ReadBuffer &, const template -ReturnType skipJSONFieldImpl(ReadBuffer & buf, StringRef name_of_field) +ReturnType skipJSONFieldImpl(ReadBuffer & buf, StringRef name_of_field, const FormatSettings::JSON & settings) { static constexpr bool throw_exception = std::is_same_v; @@ -1370,8 +1469,8 @@ ReturnType skipJSONFieldImpl(ReadBuffer & buf, StringRef name_of_field) { NullOutput sink; if constexpr (throw_exception) - readJSONStringInto(sink, buf); - else if (!tryReadJSONStringInto(sink, buf)) + readJSONStringInto(sink, buf, settings); + else if (!tryReadJSONStringInto(sink, buf, settings)) return ReturnType(false); } else if (isNumericASCII(*buf.position()) || *buf.position() == '-' || *buf.position() == '+' || *buf.position() == '.') /// skip number @@ -1422,8 +1521,8 @@ ReturnType skipJSONFieldImpl(ReadBuffer & buf, StringRef name_of_field) while (true) { if constexpr (throw_exception) - skipJSONFieldImpl(buf, name_of_field); - else if (!skipJSONFieldImpl(buf, name_of_field)) + skipJSONFieldImpl(buf, name_of_field, settings); + else if (!skipJSONFieldImpl(buf, name_of_field, settings)) return ReturnType(false); skipWhitespaceIfAny(buf); @@ -1458,8 +1557,8 @@ ReturnType skipJSONFieldImpl(ReadBuffer & buf, StringRef name_of_field) { NullOutput sink; if constexpr (throw_exception) - readJSONStringInto(sink, buf); - else if (!tryReadJSONStringInto(sink, buf)) + readJSONStringInto(sink, buf, settings); + else if (!tryReadJSONStringInto(sink, buf, settings)) return ReturnType(false); } else @@ -1481,8 +1580,8 @@ ReturnType skipJSONFieldImpl(ReadBuffer & buf, StringRef name_of_field) skipWhitespaceIfAny(buf); if constexpr (throw_exception) - skipJSONFieldImpl(buf, name_of_field); - else if (!skipJSONFieldImpl(buf, name_of_field)) + skipJSONFieldImpl(buf, name_of_field, settings); + else if (!skipJSONFieldImpl(buf, name_of_field, settings)) return ReturnType(false); skipWhitespaceIfAny(buf); @@ -1519,14 +1618,14 @@ ReturnType skipJSONFieldImpl(ReadBuffer & buf, StringRef name_of_field) return ReturnType(true); } -void skipJSONField(ReadBuffer & buf, StringRef name_of_field) +void skipJSONField(ReadBuffer & buf, StringRef name_of_field, const FormatSettings::JSON & settings) { - skipJSONFieldImpl(buf, name_of_field); + skipJSONFieldImpl(buf, name_of_field, settings); } -bool trySkipJSONField(ReadBuffer & buf, StringRef name_of_field) +bool trySkipJSONField(ReadBuffer & buf, StringRef name_of_field, const FormatSettings::JSON & settings) { - return skipJSONFieldImpl(buf, name_of_field); + return skipJSONFieldImpl(buf, name_of_field, settings); } @@ -1941,17 +2040,17 @@ bool tryReadQuotedField(String & s, ReadBuffer & buf) return readQuotedFieldInto(s, buf); } -void readJSONField(String & s, ReadBuffer & buf) +void readJSONField(String & s, ReadBuffer & buf, const FormatSettings::JSON & settings) { s.clear(); - auto parse_func = [](ReadBuffer & in) { skipJSONField(in, ""); }; + auto parse_func = [&settings](ReadBuffer & in) { skipJSONField(in, "", settings); }; readParsedValueInto(s, buf, parse_func); } -bool tryReadJSONField(String & s, ReadBuffer & buf) +bool tryReadJSONField(String & s, ReadBuffer & buf, const FormatSettings::JSON & settings) { s.clear(); - auto parse_func = [](ReadBuffer & in) { return trySkipJSONField(in, ""); }; + auto parse_func = [&settings](ReadBuffer & in) { return trySkipJSONField(in, "", settings); }; return readParsedValueInto(s, buf, parse_func); } diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index ca568c469b4..fc105539061 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -596,7 +596,7 @@ void readDoubleQuotedStringWithSQLStyle(String & s, ReadBuffer & buf); bool tryReadDoubleQuotedString(String & s, ReadBuffer & buf); bool tryReadDoubleQuotedStringWithSQLStyle(String & s, ReadBuffer & buf); -void readJSONString(String & s, ReadBuffer & buf); +void readJSONString(String & s, ReadBuffer & buf, const FormatSettings::JSON & settings); void readBackQuotedString(String & s, ReadBuffer & buf); void readBackQuotedStringWithSQLStyle(String & s, ReadBuffer & buf); @@ -663,12 +663,12 @@ void readCSVStringInto(Vector & s, ReadBuffer & buf, const FormatSettings::CSV & /// ReturnType is either bool or void. If bool, the function will return false instead of throwing an exception. template -ReturnType readJSONStringInto(Vector & s, ReadBuffer & buf); +ReturnType readJSONStringInto(Vector & s, ReadBuffer & buf, const FormatSettings::JSON & settings); template -bool tryReadJSONStringInto(Vector & s, ReadBuffer & buf) +bool tryReadJSONStringInto(Vector & s, ReadBuffer & buf, const FormatSettings::JSON & settings) { - return readJSONStringInto(s, buf); + return readJSONStringInto(s, buf, settings); } template @@ -1673,8 +1673,8 @@ inline void skipWhitespaceIfAny(ReadBuffer & buf, bool one_line = false) } /// Skips json value. -void skipJSONField(ReadBuffer & buf, StringRef name_of_field); -bool trySkipJSONField(ReadBuffer & buf, StringRef name_of_field); +void skipJSONField(ReadBuffer & buf, StringRef name_of_field, const FormatSettings::JSON & settings); +bool trySkipJSONField(ReadBuffer & buf, StringRef name_of_field, const FormatSettings::JSON & settings); /** Read serialized exception. @@ -1895,8 +1895,8 @@ ReturnType readQuotedFieldInto(Vector & s, ReadBuffer & buf); void readQuotedField(String & s, ReadBuffer & buf); bool tryReadQuotedField(String & s, ReadBuffer & buf); -void readJSONField(String & s, ReadBuffer & buf); -bool tryReadJSONField(String & s, ReadBuffer & buf); +void readJSONField(String & s, ReadBuffer & buf, const FormatSettings::JSON & settings); +bool tryReadJSONField(String & s, ReadBuffer & buf, const FormatSettings::JSON & settings); void readTSVField(String & s, ReadBuffer & buf); diff --git a/src/IO/ReadWriteBufferFromHTTP.cpp b/src/IO/ReadWriteBufferFromHTTP.cpp index fdc8ef04d2e..c99b08d0c9d 100644 --- a/src/IO/ReadWriteBufferFromHTTP.cpp +++ b/src/IO/ReadWriteBufferFromHTTP.cpp @@ -8,6 +8,8 @@ namespace ProfileEvents { extern const Event ReadBufferSeekCancelConnection; + extern const Event ReadWriteBufferFromHTTPRequestsSent; + extern const Event ReadWriteBufferFromHTTPBytes; } @@ -95,7 +97,7 @@ size_t ReadWriteBufferFromHTTP::getOffset() const void ReadWriteBufferFromHTTP::prepareRequest(Poco::Net::HTTPRequest & request, std::optional range) const { - request.setHost(initial_uri.getHost()); // use original, not resolved host name in header + request.setHost(current_uri.getHost()); if (out_stream_callback) request.setChunkedTransferEncoding(true); @@ -235,15 +237,17 @@ ReadWriteBufferFromHTTP::ReadWriteBufferFromHTTP( } ReadWriteBufferFromHTTP::CallResult ReadWriteBufferFromHTTP::callImpl( - Poco::Net::HTTPResponse & response, const Poco::URI & uri_, const std::string & method_, const std::optional & range, bool allow_redirects) const + Poco::Net::HTTPResponse & response, const std::string & method_, const std::optional & range, bool allow_redirects) const { if (remote_host_filter) - remote_host_filter->checkURL(uri_); + remote_host_filter->checkURL(current_uri); - Poco::Net::HTTPRequest request(method_, uri_.getPathAndQuery(), Poco::Net::HTTPRequest::HTTP_1_1); + Poco::Net::HTTPRequest request(method_, current_uri.getPathAndQuery(), Poco::Net::HTTPRequest::HTTP_1_1); prepareRequest(request, range); - auto session = makeHTTPSession(connection_group, uri_, timeouts, proxy_config); + auto session = makeHTTPSession(connection_group, current_uri, timeouts, proxy_config); + + ProfileEvents::increment(ProfileEvents::ReadWriteBufferFromHTTPRequestsSent); auto & stream_out = session->sendRequest(request); if (out_stream_callback) @@ -259,7 +263,7 @@ ReadWriteBufferFromHTTP::CallResult ReadWriteBufferFromHTTP::callImpl( ReadWriteBufferFromHTTP::CallResult ReadWriteBufferFromHTTP::callWithRedirects( Poco::Net::HTTPResponse & response, const String & method_, const std::optional & range) { - auto result = callImpl(response, current_uri, method_, range, true); + auto result = callImpl(response, method_, range, true); while (isRedirect(response.getStatus())) { @@ -275,8 +279,7 @@ ReadWriteBufferFromHTTP::CallResult ReadWriteBufferFromHTTP::callWithRedirects( initial_uri.toString(), max_redirects ? "increase the allowed maximum number of" : "allow"); current_uri = uri_redirect; - - result = callImpl(response, uri_redirect, method_, range, true); + result = callImpl(response, method_, range, true); } return result; @@ -343,9 +346,11 @@ void ReadWriteBufferFromHTTP::doWithRetries(std::function && callable, { if (!mute_logging) LOG_ERROR(log, - "Failed to make request to '{}'. Error: '{}'. " + "Failed to make request to '{}'{}. " + "Error: '{}'. " "Failed at try {}/{}.", - initial_uri.toString(), error_message, + initial_uri.toString(), current_uri == initial_uri ? String() : fmt::format(" redirect to '{}'", current_uri.toString()), + error_message, attempt, read_settings.http_max_tries); std::rethrow_exception(exception); @@ -357,10 +362,12 @@ void ReadWriteBufferFromHTTP::doWithRetries(std::function && callable, if (!mute_logging) LOG_INFO(log, - "Failed to make request to `{}`. Error: {}. " + "Failed to make request to '{}'{}. " + "Error: {}. " "Failed at try {}/{}. " "Will retry with current backoff wait is {}/{} ms.", - initial_uri.toString(), error_message, + initial_uri.toString(), current_uri == initial_uri ? String() : fmt::format(" redirect to '{}'", current_uri.toString()), + error_message, attempt + 1, read_settings.http_max_tries, milliseconds_to_wait, read_settings.http_retry_max_backoff_ms); @@ -480,6 +487,8 @@ bool ReadWriteBufferFromHTTP::nextImpl() BufferBase::set(impl->buffer().begin(), impl->buffer().size(), impl->offset()); offset_from_begin_pos += working_buffer.size(); + + ProfileEvents::increment(ProfileEvents::ReadWriteBufferFromHTTPBytes, working_buffer.size()); }, /*on_retry=*/ [&] () { @@ -506,7 +515,7 @@ size_t ReadWriteBufferFromHTTP::readBigAt(char * to, size_t n, size_t offset, co auto range = HTTPRange{offset, offset + n - 1}; Poco::Net::HTTPResponse response; - auto result = callImpl(response, current_uri, method, range, false); + auto result = callImpl(response, method, range, false); if (response.getStatus() != Poco::Net::HTTPResponse::HTTPStatus::HTTP_PARTIAL_CONTENT && (offset != 0 || offset + n < *file_info->file_size)) @@ -528,6 +537,8 @@ size_t ReadWriteBufferFromHTTP::readBigAt(char * to, size_t n, size_t offset, co copyFromIStreamWithProgressCallback(*result.response_stream, to, n, progress_callback, &bytes_copied, &is_canceled); + ProfileEvents::increment(ProfileEvents::ReadWriteBufferFromHTTPBytes, bytes_copied); + offset += bytes_copied; total_bytes_copied += bytes_copied; to += bytes_copied; @@ -536,6 +547,8 @@ size_t ReadWriteBufferFromHTTP::readBigAt(char * to, size_t n, size_t offset, co }, /*on_retry=*/ [&] () { + ProfileEvents::increment(ProfileEvents::ReadWriteBufferFromHTTPBytes, bytes_copied); + offset += bytes_copied; total_bytes_copied += bytes_copied; to += bytes_copied; @@ -574,7 +587,7 @@ off_t ReadWriteBufferFromHTTP::seek(off_t offset_, int whence) if (impl) { auto position = getPosition(); - if (offset_ > position) + if (offset_ >= position) { size_t diff = offset_ - position; if (diff < read_settings.remote_read_min_bytes_for_seek) diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index 224aac809a4..f496fe3ddcd 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -107,7 +107,6 @@ private: CallResult callImpl( Poco::Net::HTTPResponse & response, - const Poco::URI & uri_, const std::string & method_, const std::optional & range, bool allow_redirects) const; diff --git a/src/IO/S3/BlobStorageLogWriter.cpp b/src/IO/S3/BlobStorageLogWriter.cpp index fe33f1c8799..aaf4aea5a8e 100644 --- a/src/IO/S3/BlobStorageLogWriter.cpp +++ b/src/IO/S3/BlobStorageLogWriter.cpp @@ -3,6 +3,7 @@ #if USE_AWS_S3 #include +#include #include #include #include diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index 4f93aba2f84..b2ad4668095 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -1,4 +1,5 @@ #include +#include #include #if USE_AWS_S3 diff --git a/src/IO/TimeoutSetter.cpp b/src/IO/TimeoutSetter.cpp index b355a119462..2e732782700 100644 --- a/src/IO/TimeoutSetter.cpp +++ b/src/IO/TimeoutSetter.cpp @@ -1,5 +1,5 @@ #include - +#include #include diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 7f3b961a598..ab8cec864ae 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1435,13 +1435,14 @@ void NO_INLINE Aggregator::mergeOnIntervalWithoutKey( AggregatedDataVariants & data_variants, size_t row_begin, size_t row_end, - const AggregateColumnsConstData & aggregate_columns_data) const + const AggregateColumnsConstData & aggregate_columns_data, + std::atomic & is_cancelled) const { /// `data_variants` will destroy the states of aggregate functions in the destructor data_variants.aggregator = this; data_variants.init(AggregatedDataVariants::Type::without_key); - mergeWithoutKeyStreamsImpl(data_variants, row_begin, row_end, aggregate_columns_data); + mergeWithoutKeyStreamsImpl(data_variants, row_begin, row_end, aggregate_columns_data, is_cancelled); } @@ -1751,7 +1752,7 @@ Block Aggregator::mergeAndConvertOneBucketToBlock( Arena * arena, bool final, Int32 bucket, - std::atomic * is_cancelled) const + std::atomic & is_cancelled) const { auto & merged_data = *variants[0]; auto method = merged_data.type; @@ -1761,8 +1762,8 @@ Block Aggregator::mergeAndConvertOneBucketToBlock( #define M(NAME) \ else if (method == AggregatedDataVariants::Type::NAME) \ { \ - mergeBucketImpl(variants, bucket, arena); \ - if (is_cancelled && is_cancelled->load(std::memory_order_seq_cst)) \ + mergeBucketImpl(variants, bucket, arena, is_cancelled); \ + if (is_cancelled.load(std::memory_order_seq_cst)) \ return {}; \ block = convertOneBucketToBlock(merged_data, *merged_data.NAME, arena, final, bucket); \ } @@ -2636,7 +2637,8 @@ void NO_INLINE Aggregator::mergeDataOnlyExistingKeysImpl( void NO_INLINE Aggregator::mergeWithoutKeyDataImpl( - ManyAggregatedDataVariants & non_empty_data) const + ManyAggregatedDataVariants & non_empty_data, + std::atomic & is_cancelled) const { ThreadPool thread_pool{CurrentMetrics::AggregatorThreads, CurrentMetrics::AggregatorThreadsActive, CurrentMetrics::AggregatorThreadsScheduled, params.max_threads}; @@ -2652,7 +2654,7 @@ void NO_INLINE Aggregator::mergeWithoutKeyDataImpl( for (size_t result_num = 0; result_num < size; ++result_num) data_vec.emplace_back(non_empty_data[result_num]->without_key + offsets_of_aggregate_states[i]); - aggregate_functions[i]->parallelizeMergePrepare(data_vec, thread_pool); + aggregate_functions[i]->parallelizeMergePrepare(data_vec, thread_pool, is_cancelled); } } @@ -2668,6 +2670,7 @@ void NO_INLINE Aggregator::mergeWithoutKeyDataImpl( res_data + offsets_of_aggregate_states[i], current_data + offsets_of_aggregate_states[i], thread_pool, + is_cancelled, res->aggregates_pool); else aggregate_functions[i]->merge( @@ -2743,7 +2746,7 @@ void NO_INLINE Aggregator::mergeSingleLevelDataImpl( template void NO_INLINE Aggregator::mergeBucketImpl( - ManyAggregatedDataVariants & data, Int32 bucket, Arena * arena, std::atomic * is_cancelled) const + ManyAggregatedDataVariants & data, Int32 bucket, Arena * arena, std::atomic & is_cancelled) const { /// We merge all aggregation results to the first. AggregatedDataVariantsPtr & res = data[0]; @@ -2753,7 +2756,7 @@ void NO_INLINE Aggregator::mergeBucketImpl( for (size_t result_num = 1, size = data.size(); result_num < size; ++result_num) { - if (is_cancelled && is_cancelled->load(std::memory_order_seq_cst)) + if (is_cancelled.load(std::memory_order_seq_cst)) return; AggregatedDataVariants & current = *data[result_num]; @@ -2954,17 +2957,19 @@ void NO_INLINE Aggregator::mergeStreamsImpl( void NO_INLINE Aggregator::mergeBlockWithoutKeyStreamsImpl( Block block, - AggregatedDataVariants & result) const + AggregatedDataVariants & result, + std::atomic & is_cancelled) const { AggregateColumnsConstData aggregate_columns = params.makeAggregateColumnsData(block); - mergeWithoutKeyStreamsImpl(result, 0, block.rows(), aggregate_columns); + mergeWithoutKeyStreamsImpl(result, 0, block.rows(), aggregate_columns, is_cancelled); } void NO_INLINE Aggregator::mergeWithoutKeyStreamsImpl( AggregatedDataVariants & result, size_t row_begin, size_t row_end, - const AggregateColumnsConstData & aggregate_columns_data) const + const AggregateColumnsConstData & aggregate_columns_data, + std::atomic & is_cancelled) const { using namespace CurrentMetrics; @@ -2986,12 +2991,12 @@ void NO_INLINE Aggregator::mergeWithoutKeyStreamsImpl( if (aggregate_functions[i]->isParallelizeMergePrepareNeeded()) { std::vector data_vec{res + offsets_of_aggregate_states[i], (*aggregate_columns_data[i])[row]}; - aggregate_functions[i]->parallelizeMergePrepare(data_vec, thread_pool); + aggregate_functions[i]->parallelizeMergePrepare(data_vec, thread_pool, is_cancelled); } if (aggregate_functions[i]->isAbleToParallelizeMerge()) aggregate_functions[i]->merge( - res + offsets_of_aggregate_states[i], (*aggregate_columns_data[i])[row], thread_pool, result.aggregates_pool); + res + offsets_of_aggregate_states[i], (*aggregate_columns_data[i])[row], thread_pool, is_cancelled, result.aggregates_pool); else aggregate_functions[i]->merge( res + offsets_of_aggregate_states[i], (*aggregate_columns_data[i])[row], result.aggregates_pool); @@ -3000,7 +3005,7 @@ void NO_INLINE Aggregator::mergeWithoutKeyStreamsImpl( } -bool Aggregator::mergeOnBlock(Block block, AggregatedDataVariants & result, bool & no_more_keys) const +bool Aggregator::mergeOnBlock(Block block, AggregatedDataVariants & result, bool & no_more_keys, std::atomic & is_cancelled) const { /// `result` will destroy the states of aggregate functions in the destructor result.aggregator = this; @@ -3022,7 +3027,7 @@ bool Aggregator::mergeOnBlock(Block block, AggregatedDataVariants & result, bool } if (result.type == AggregatedDataVariants::Type::without_key || block.info.is_overflows) - mergeBlockWithoutKeyStreamsImpl(std::move(block), result); + mergeBlockWithoutKeyStreamsImpl(std::move(block), result, is_cancelled); #define M(NAME, IS_TWO_LEVEL) \ else if (result.type == AggregatedDataVariants::Type::NAME) \ mergeStreamsImpl(std::move(block), result.aggregates_pool, *result.NAME, result.NAME->data, result.without_key, result.consecutive_keys_cache_stats, no_more_keys); @@ -3070,7 +3075,7 @@ bool Aggregator::mergeOnBlock(Block block, AggregatedDataVariants & result, bool } -void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVariants & result, size_t max_threads) +void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVariants & result, size_t max_threads, std::atomic & is_cancelled) { if (bucket_to_blocks.empty()) return; @@ -3183,7 +3188,7 @@ void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVari break; if (result.type == AggregatedDataVariants::Type::without_key || block.info.is_overflows) - mergeBlockWithoutKeyStreamsImpl(std::move(block), result); + mergeBlockWithoutKeyStreamsImpl(std::move(block), result, is_cancelled); #define M(NAME, IS_TWO_LEVEL) \ else if (result.type == AggregatedDataVariants::Type::NAME) \ @@ -3202,7 +3207,7 @@ void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVari } -Block Aggregator::mergeBlocks(BlocksList & blocks, bool final) +Block Aggregator::mergeBlocks(BlocksList & blocks, bool final, std::atomic & is_cancelled) { if (blocks.empty()) return {}; @@ -3264,7 +3269,7 @@ Block Aggregator::mergeBlocks(BlocksList & blocks, bool final) bucket_num = -1; if (result.type == AggregatedDataVariants::Type::without_key || is_overflows) - mergeBlockWithoutKeyStreamsImpl(std::move(block), result); + mergeBlockWithoutKeyStreamsImpl(std::move(block), result, is_cancelled); #define M(NAME, IS_TWO_LEVEL) \ else if (result.type == AggregatedDataVariants::Type::NAME) \ diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index e339047063c..406d28597cf 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -266,7 +266,10 @@ public: AggregateFunctionInstruction * aggregate_instructions) const; /// Used for aggregate projection. - bool mergeOnBlock(Block block, AggregatedDataVariants & result, bool & no_more_keys) const; + bool mergeOnBlock(Block block, + AggregatedDataVariants & result, + bool & no_more_keys, + std::atomic & is_cancelled) const; void mergeOnBlockSmall( AggregatedDataVariants & result, @@ -279,7 +282,8 @@ public: AggregatedDataVariants & data_variants, size_t row_begin, size_t row_end, - const AggregateColumnsConstData & aggregate_columns_data) const; + const AggregateColumnsConstData & aggregate_columns_data, + std::atomic & is_cancelled) const; /** Convert the aggregation data structure into a block. * If overflow_row = true, then aggregates for rows that are not included in max_rows_to_group_by are put in the first block. @@ -294,13 +298,13 @@ public: using BucketToBlocks = std::map; /// Merge partially aggregated blocks separated to buckets into one data structure. - void mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVariants & result, size_t max_threads); + void mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVariants & result, size_t max_threads, std::atomic & is_cancelled); /// Merge several partially aggregated blocks into one. /// Precondition: for all blocks block.info.is_overflows flag must be the same. /// (either all blocks are from overflow data or none blocks are). /// The resulting block has the same value of is_overflows flag. - Block mergeBlocks(BlocksList & blocks, bool final); + Block mergeBlocks(BlocksList & blocks, bool final, std::atomic & is_cancelled); /** Split block with partially-aggregated data to many blocks, as if two-level method of aggregation was used. * This is needed to simplify merging of that data with other results, that are already two-level. @@ -486,7 +490,8 @@ private: Arena * arena) const; void mergeWithoutKeyDataImpl( - ManyAggregatedDataVariants & non_empty_data) const; + ManyAggregatedDataVariants & non_empty_data, + std::atomic & is_cancelled) const; template void mergeSingleLevelDataImpl( @@ -541,7 +546,7 @@ private: Arena * arena, bool final, Int32 bucket, - std::atomic * is_cancelled = nullptr) const; + std::atomic & is_cancelled) const; Block prepareBlockAndFillWithoutKey(AggregatedDataVariants & data_variants, bool final, bool is_overflows) const; BlocksList prepareBlocksAndFillTwoLevel(AggregatedDataVariants & data_variants, bool final, ThreadPool * thread_pool) const; @@ -597,17 +602,19 @@ private: void mergeBlockWithoutKeyStreamsImpl( Block block, - AggregatedDataVariants & result) const; + AggregatedDataVariants & result, + std::atomic & is_cancelled) const; void mergeWithoutKeyStreamsImpl( AggregatedDataVariants & result, size_t row_begin, size_t row_end, - const AggregateColumnsConstData & aggregate_columns_data) const; + const AggregateColumnsConstData & aggregate_columns_data, + std::atomic & is_cancelled) const; template void mergeBucketImpl( - ManyAggregatedDataVariants & data, Int32 bucket, Arena * arena, std::atomic * is_cancelled = nullptr) const; + ManyAggregatedDataVariants & data, Int32 bucket, Arena * arena, std::atomic & is_cancelled) const; template void convertBlockToTwoLevelImpl( diff --git a/src/Interpreters/AsynchronousMetricLog.cpp b/src/Interpreters/AsynchronousMetricLog.cpp index dc67bd91550..4287798c4ca 100644 --- a/src/Interpreters/AsynchronousMetricLog.cpp +++ b/src/Interpreters/AsynchronousMetricLog.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -6,8 +5,10 @@ #include #include #include -#include #include +#include +#include +#include namespace DB diff --git a/src/Interpreters/Cache/EvictionCandidates.cpp b/src/Interpreters/Cache/EvictionCandidates.cpp index f1ae2baa347..f21c5f3a508 100644 --- a/src/Interpreters/Cache/EvictionCandidates.cpp +++ b/src/Interpreters/Cache/EvictionCandidates.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace ProfileEvents @@ -11,62 +12,158 @@ namespace ProfileEvents namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} EvictionCandidates::~EvictionCandidates() { + /// Here `queue_entries_to_invalidate` contains queue entries + /// for file segments which were successfully removed in evict(). + /// This set is non-empty in destructor only if there was + /// an exception before we called finalize() or in the middle of finalize(). + for (const auto & iterator : queue_entries_to_invalidate) + { + /// In this case we need to finalize the state of queue entries + /// which correspond to removed files segments to make sure + /// consistent state of cache. + iterator->invalidate(); + } + + /// Here `candidates` contain only those file segments + /// which failed to be removed during evict() + /// because there was some exception before evict() + /// or in the middle of evict(). for (const auto & [key, key_candidates] : candidates) { + // Reset the evicting state + // (as the corresponding file segments were not yet removed). for (const auto & candidate : key_candidates.candidates) - candidate->removal_candidate = false; + candidate->resetEvictingFlag(); } } -void EvictionCandidates::add(LockedKey & locked_key, const FileSegmentMetadataPtr & candidate) +void EvictionCandidates::add( + const FileSegmentMetadataPtr & candidate, + LockedKey & locked_key, + const CachePriorityGuard::Lock & lock) { auto [it, inserted] = candidates.emplace(locked_key.getKey(), KeyCandidates{}); if (inserted) it->second.key_metadata = locked_key.getKeyMetadata(); - it->second.candidates.push_back(candidate); - candidate->removal_candidate = true; + it->second.candidates.push_back(candidate); + candidate->setEvictingFlag(locked_key, lock); ++candidates_size; } -void EvictionCandidates::evict(FileCacheQueryLimit::QueryContext * query_context, const CachePriorityGuard::Lock & lock) +void EvictionCandidates::evict() { if (candidates.empty()) return; auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::FilesystemCacheEvictMicroseconds); + queue_entries_to_invalidate.reserve(candidates_size); for (auto & [key, key_candidates] : candidates) { auto locked_key = key_candidates.key_metadata->tryLock(); if (!locked_key) - continue; /// key could become invalid after we released the key lock above, just skip it. - - auto & to_evict = key_candidates.candidates; - while (!to_evict.empty()) { - auto & candidate = to_evict.back(); + /// key could become invalid after we released + /// the key lock above, just skip it. + continue; + } + + while (!key_candidates.candidates.empty()) + { + auto & candidate = key_candidates.candidates.back(); chassert(candidate->releasable()); const auto segment = candidate->file_segment; - auto queue_it = segment->getQueueIterator(); - chassert(queue_it); + auto iterator = segment->getQueueIterator(); + chassert(iterator); ProfileEvents::increment(ProfileEvents::FilesystemCacheEvictedFileSegments); ProfileEvents::increment(ProfileEvents::FilesystemCacheEvictedBytes, segment->range().size()); - locked_key->removeFileSegment(segment->offset(), segment->lock()); - queue_it->remove(lock); + locked_key->removeFileSegment( + segment->offset(), segment->lock(), + false/* can_be_broken */, false/* invalidate_queue_entry */); - if (query_context) - query_context->remove(segment->key(), segment->offset(), lock); + /// We set invalidate_queue_entry = false in removeFileSegment() above, because: + /// evict() is done without a cache priority lock while finalize() is done under the lock. + /// In evict() we: + /// - remove file segment from filesystem + /// - remove it from cache metadata + /// In finalize() we: + /// - remove corresponding queue entry from priority queue + /// + /// We do not invalidate queue entry now in evict(), + /// because invalidation of queue entries needs to be done under cache lock. + /// Why? Firstly, as long as queue entry exists, + /// the corresponding space in cache is considered to be hold, + /// and once queue entry is removed/invalidated - the space is released. + /// Secondly, after evict() and finalize() stages we will also add back the + /// "reserved size" (<= actually released size), + /// but until we do this - we cannot allow other threads to think that + /// this released space is free to take, as it is not - + /// it was freed in favour of some reserver, so we can make it visibly + /// free only for that particular reserver. - to_evict.pop_back(); + queue_entries_to_invalidate.push_back(iterator); + key_candidates.candidates.pop_back(); } } } +void EvictionCandidates::finalize( + FileCacheQueryLimit::QueryContext * query_context, + const CachePriorityGuard::Lock & lock) +{ + chassert(lock.owns_lock()); + + /// Release the hold space. It was hold only for the duration of evict() phase, + /// now we can release. It might also be needed for on_finalize func, + /// so release the space it firtst. + if (hold_space) + hold_space->release(); + + while (!queue_entries_to_invalidate.empty()) + { + auto iterator = queue_entries_to_invalidate.back(); + iterator->invalidate(); + queue_entries_to_invalidate.pop_back(); + + /// Remove entry from per query priority queue. + if (query_context) + { + const auto & entry = iterator->getEntry(); + query_context->remove(entry->key, entry->offset, lock); + } + /// Remove entry from main priority queue. + iterator->remove(lock); + } + + for (auto & func : on_finalize) + func(lock); + + /// Finalize functions might hold something (like HoldSpace object), + /// so we need to clear them now. + on_finalize.clear(); +} + +void EvictionCandidates::setSpaceHolder( + size_t size, + size_t elements, + IFileCachePriority & priority, + const CachePriorityGuard::Lock & lock) +{ + if (hold_space) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Space hold is already set"); + else + hold_space = std::make_unique(size, elements, priority, lock); +} + } diff --git a/src/Interpreters/Cache/EvictionCandidates.h b/src/Interpreters/Cache/EvictionCandidates.h index e817d33d5fe..2745d508a5d 100644 --- a/src/Interpreters/Cache/EvictionCandidates.h +++ b/src/Interpreters/Cache/EvictionCandidates.h @@ -7,11 +7,22 @@ namespace DB class EvictionCandidates { public: + using FinalizeEvictionFunc = std::function; + ~EvictionCandidates(); - void add(LockedKey & locked_key, const FileSegmentMetadataPtr & candidate); + void add( + const FileSegmentMetadataPtr & candidate, + LockedKey & locked_key, + const CachePriorityGuard::Lock &); - void evict(FileCacheQueryLimit::QueryContext * query_context, const CachePriorityGuard::Lock &); + void evict(); + + void onFinalize(FinalizeEvictionFunc && func) { on_finalize.emplace_back(std::move(func)); } + + void finalize( + FileCacheQueryLimit::QueryContext * query_context, + const CachePriorityGuard::Lock &); size_t size() const { return candidates_size; } @@ -19,6 +30,12 @@ public: auto end() const { return candidates.end(); } + void setSpaceHolder( + size_t size, + size_t elements, + IFileCachePriority & priority, + const CachePriorityGuard::Lock &); + private: struct KeyCandidates { @@ -28,6 +45,10 @@ private: std::unordered_map candidates; size_t candidates_size = 0; + + std::vector on_finalize; + std::vector queue_entries_to_invalidate; + IFileCachePriority::HoldSpacePtr hold_space; }; using EvictionCandidatesPtr = std::unique_ptr; diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp index 916bdb8f898..71dc0cca3a7 100644 --- a/src/Interpreters/Cache/FileCache.cpp +++ b/src/Interpreters/Cache/FileCache.cpp @@ -64,16 +64,16 @@ void FileCacheReserveStat::update(size_t size, FileSegmentKind kind, bool releas auto & local_stat = stat_by_kind[kind]; if (releasable) { - stat.releasable_size += size; - ++stat.releasable_count; + total_stat.releasable_size += size; + ++total_stat.releasable_count; local_stat.releasable_size += size; ++local_stat.releasable_count; } else { - stat.non_releasable_size += size; - ++stat.non_releasable_count; + total_stat.non_releasable_size += size; + ++total_stat.non_releasable_count; local_stat.non_releasable_size += size; ++local_stat.non_releasable_count; @@ -216,7 +216,7 @@ FileSegments FileCache::getImpl(const LockedKey & locked_key, const FileSegment: return false; FileSegmentPtr file_segment; - if (!file_segment_metadata.evicting()) + if (!file_segment_metadata.isEvicting(locked_key)) { file_segment = file_segment_metadata.file_segment; } @@ -802,9 +802,8 @@ bool FileCache::tryReserve( } LOG_TEST( - log, "Trying to reserve space ({} bytes) for {}:{}, current usage {}/{}", - size, file_segment.key(), file_segment.offset(), - main_priority->getSize(cache_lock), main_priority->getSizeLimit(cache_lock)); + log, "Trying to reserve space ({} bytes) for {}:{}, current usage: {}", + size, file_segment.key(), file_segment.offset(), main_priority->getStateInfoForLog(cache_lock)); /// In case of per query cache limit (by default disabled), we add/remove entries from both /// (main_priority and query_priority) priority queues, but iterate entries in order of query_priority, @@ -831,61 +830,104 @@ bool FileCache::tryReserve( file_segment.key(), file_segment.offset()); } - EvictionCandidates eviction_candidates; - IFileCachePriority::FinalizeEvictionFunc finalize_eviction_func; + auto queue_iterator = file_segment.getQueueIterator(); + /// A file_segment_metadata acquires a priority iterator + /// on first successful space reservation attempt, + /// so queue_iterator == nullptr, if no space reservation took place yet. + chassert(!queue_iterator || file_segment.getReservedSize() > 0); + + /// If it is the first space reservatiob attempt for a file segment + /// we need to make space for 1 element in cache, + /// otherwise space is already taken and we need 0 elements to free. + size_t required_elements_num = queue_iterator ? 0 : 1; + + EvictionCandidates eviction_candidates; + + /// If user has configured fs cache limit per query, + /// we take into account query limits here. if (query_priority) { if (!query_priority->collectCandidatesForEviction( - size, reserve_stat, eviction_candidates, {}, finalize_eviction_func, user.user_id, cache_lock)) + size, required_elements_num, reserve_stat, eviction_candidates, {}, user.user_id, cache_lock)) + { return false; + } + + LOG_TEST(log, "Query limits satisfied (while reserving for {}:{})", + file_segment.key(), file_segment.offset()); - LOG_TEST(log, "Query limits satisfied (while reserving for {}:{})", file_segment.key(), file_segment.offset()); /// If we have enough space in query_priority, we are not interested about stat there anymore. /// Clean the stat before iterating main_priority to avoid calculating any segment stat twice. reserve_stat.stat_by_kind.clear(); } - /// A file_segment_metadata acquires a priority iterator on first successful space reservation attempt, - auto queue_iterator = file_segment.getQueueIterator(); - chassert(!queue_iterator || file_segment.getReservedSize() > 0); - if (!main_priority->collectCandidatesForEviction( - size, reserve_stat, eviction_candidates, queue_iterator, finalize_eviction_func, user.user_id, cache_lock)) + size, required_elements_num, reserve_stat, eviction_candidates, queue_iterator, user.user_id, cache_lock)) + { return false; + } if (!file_segment.getKeyMetadata()->createBaseDirectory()) return false; - eviction_candidates.evict(query_context.get(), cache_lock); + if (eviction_candidates.size() > 0) + { + cache_lock.unlock(); + try + { + /// Remove eviction candidates from filesystem. + eviction_candidates.evict(); + } + catch (...) + { + cache_lock.lock(); + /// Invalidate queue entries if some succeeded to be removed. + eviction_candidates.finalize(query_context.get(), cache_lock); + throw; + } - if (finalize_eviction_func) - finalize_eviction_func(cache_lock); + cache_lock.lock(); + + /// Invalidate and remove queue entries and execute finalize func. + eviction_candidates.finalize(query_context.get(), cache_lock); + } + else if (!main_priority->canFit(size, required_elements_num, cache_lock, queue_iterator)) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Cannot fit {} in cache, but collection of eviction candidates succeeded with no candidates. " + "This is a bug. Queue entry type: {}. Cache info: {}", + size, queue_iterator ? queue_iterator->getType() : FileCacheQueueEntryType::None, + main_priority->getStateInfoForLog(cache_lock)); + } if (queue_iterator) { - queue_iterator->updateSize(size); + /// Increase size of queue entry. + queue_iterator->incrementSize(size, cache_lock); } else { - /// Space reservation is incremental, so file_segment_metadata is created first (with state empty), - /// and getQueueIterator() is assigned on first space reservation attempt. + /// Create a new queue entry and assign currently reserved size to it. queue_iterator = main_priority->add(file_segment.getKeyMetadata(), file_segment.offset(), size, user, cache_lock); file_segment.setQueueIterator(queue_iterator); } - file_segment.reserved_size += size; - chassert(file_segment.reserved_size == queue_iterator->getEntry()->size); + main_priority->check(cache_lock); if (query_context) { auto query_queue_it = query_context->tryGet(file_segment.key(), file_segment.offset(), cache_lock); if (query_queue_it) - query_queue_it->updateSize(size); + query_queue_it->incrementSize(size, cache_lock); else query_context->add(file_segment.getKeyMetadata(), file_segment.offset(), size, user, cache_lock); } + file_segment.reserved_size += size; + chassert(file_segment.reserved_size == queue_iterator->getEntry()->size); + if (main_priority->getSize(cache_lock) > (1ull << 63)) throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache became inconsistent. There must be a bug"); @@ -1148,7 +1190,7 @@ void FileCache::loadMetadataForKeys(const fs::path & keys_dir) auto lock = lockCache(); size_limit = main_priority->getSizeLimit(lock); - limits_satisfied = main_priority->canFit(size, lock, nullptr, true); + limits_satisfied = main_priority->canFit(size, 1, lock, nullptr, true); if (limits_satisfied) cache_it = main_priority->add(key_metadata, offset, size, user, lock, /* best_effort */true); diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h index 1433a067e7e..4ee456cce72 100644 --- a/src/Interpreters/Cache/FileCache.h +++ b/src/Interpreters/Cache/FileCache.h @@ -46,14 +46,14 @@ struct FileCacheReserveStat } }; - Stat stat; + Stat total_stat; std::unordered_map stat_by_kind; void update(size_t size, FileSegmentKind kind, bool releasable); FileCacheReserveStat & operator +=(const FileCacheReserveStat & other) { - stat += other.stat; + total_stat += other.total_stat; for (const auto & [name, stat_] : other.stat_by_kind) stat_by_kind[name] += stat_; return *this; diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp index 9ec2b090dc7..e474e24c6f1 100644 --- a/src/Interpreters/Cache/FileSegment.cpp +++ b/src/Interpreters/Cache/FileSegment.cpp @@ -6,10 +6,11 @@ #include #include #include +#include +#include #include #include #include -#include #include #include diff --git a/src/Interpreters/Cache/IFileCachePriority.cpp b/src/Interpreters/Cache/IFileCachePriority.cpp index eb396a1e323..6532af55ed2 100644 --- a/src/Interpreters/Cache/IFileCachePriority.cpp +++ b/src/Interpreters/Cache/IFileCachePriority.cpp @@ -10,6 +10,11 @@ namespace CurrentMetrics namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + IFileCachePriority::IFileCachePriority(size_t max_size_, size_t max_elements_) : max_size(max_size_), max_elements(max_elements_) { @@ -37,4 +42,13 @@ IFileCachePriority::Entry::Entry(const Entry & other) { } +void IFileCachePriority::check(const CachePriorityGuard::Lock & lock) const +{ + if (getSize(lock) > max_size || getElementsCount(lock) > max_elements) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache limits violated. " + "{}", getStateInfoForLog(lock)); + } +} + } diff --git a/src/Interpreters/Cache/IFileCachePriority.h b/src/Interpreters/Cache/IFileCachePriority.h index 09d71cebb01..ff06f17ce36 100644 --- a/src/Interpreters/Cache/IFileCachePriority.h +++ b/src/Interpreters/Cache/IFileCachePriority.h @@ -33,6 +33,36 @@ public: std::atomic size; size_t hits = 0; + + std::string toString() const { return fmt::format("{}:{}:{}", key, offset, size); } + + bool isEvicting(const CachePriorityGuard::Lock &) const { return evicting; } + bool isEvicting(const LockedKey &) const { return evicting; } + /// This does not look good to have isEvicting with two options for locks, + /// but still it is valid as we do setEvicting always under both of them. + /// (Well, not always - only always for setting it to True, + /// but for False we have lower guarantees and allow a logical race, + /// physical race is not possible because the value is atomic). + /// We can avoid this ambiguity for isEvicting by introducing + /// a separate lock `EntryGuard::Lock`, it will make this part of code more coherent, + /// but it will introduce one more mutex while it is avoidable. + /// Introducing one more mutex just for coherency does not win the trade-off (isn't it?). + void setEvictingFlag(const LockedKey &, const CachePriorityGuard::Lock &) const + { + auto prev = evicting.exchange(true, std::memory_order_relaxed); + chassert(!prev); + UNUSED(prev); + } + + void resetEvictingFlag() const + { + auto prev = evicting.exchange(false, std::memory_order_relaxed); + chassert(prev); + UNUSED(prev); + } + + private: + mutable std::atomic evicting = false; }; using EntryPtr = std::shared_ptr; @@ -45,7 +75,12 @@ public: virtual size_t increasePriority(const CachePriorityGuard::Lock &) = 0; - virtual void updateSize(int64_t size) = 0; + /// Note: IncrementSize unlike decrementSize requires a cache lock, because + /// it requires more consistency guarantees for eviction. + + virtual void incrementSize(size_t size, const CachePriorityGuard::Lock &) = 0; + + virtual void decrementSize(size_t size) = 0; virtual void remove(const CachePriorityGuard::Lock &) = 0; @@ -69,6 +104,10 @@ public: virtual size_t getElementsCountApprox() const = 0; + virtual std::string getStateInfoForLog(const CachePriorityGuard::Lock &) const = 0; + + virtual void check(const CachePriorityGuard::Lock &) const; + /// Throws exception if there is not enough size to fit it. virtual IteratorPtr add( /// NOLINT KeyMetadataPtr key_metadata, @@ -83,6 +122,7 @@ public: /// for the corresponding file segment. virtual bool canFit( /// NOLINT size_t size, + size_t elements, const CachePriorityGuard::Lock &, IteratorPtr reservee = nullptr, bool best_effort = false) const = 0; @@ -97,21 +137,61 @@ public: virtual PriorityDumpPtr dump(const CachePriorityGuard::Lock &) = 0; - using FinalizeEvictionFunc = std::function; virtual bool collectCandidatesForEviction( size_t size, + size_t elements, FileCacheReserveStat & stat, EvictionCandidates & res, - IFileCachePriority::IteratorPtr reservee, - FinalizeEvictionFunc & finalize_eviction_func, + IteratorPtr reservee, const UserID & user_id, const CachePriorityGuard::Lock &) = 0; virtual bool modifySizeLimits(size_t max_size_, size_t max_elements_, double size_ratio_, const CachePriorityGuard::Lock &) = 0; + /// A space holder implementation, which allows to take hold of + /// some space in cache given that this space was freed. + /// Takes hold of the space in constructor and releases it in destructor. + struct HoldSpace : private boost::noncopyable + { + HoldSpace( + size_t size_, + size_t elements_, + IFileCachePriority & priority_, + const CachePriorityGuard::Lock & lock) + : size(size_), elements(elements_), priority(priority_) + { + priority.holdImpl(size, elements, lock); + } + + void release() + { + if (released) + return; + released = true; + priority.releaseImpl(size, elements); + } + + ~HoldSpace() + { + if (!released) + release(); + } + + private: + const size_t size; + const size_t elements; + IFileCachePriority & priority; + bool released = false; + }; + using HoldSpacePtr = std::unique_ptr; + protected: IFileCachePriority(size_t max_size_, size_t max_elements_); + virtual void holdImpl(size_t /* size */, size_t /* elements */, const CachePriorityGuard::Lock &) {} + + virtual void releaseImpl(size_t /* size */, size_t /* elements */) {} + size_t max_size = 0; size_t max_elements = 0; }; diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp index 08e65b577ca..ddc30755409 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp @@ -1,10 +1,11 @@ -#include -#include #include -#include -#include -#include +#include +#include #include +#include +#include +#include +#include namespace CurrentMetrics { @@ -19,6 +20,7 @@ namespace ProfileEvents extern const Event FilesystemCacheEvictMicroseconds; extern const Event FilesystemCacheEvictedBytes; extern const Event FilesystemCacheEvictedFileSegments; + extern const Event FilesystemCacheEvictionSkippedEvictingFileSegments; } namespace DB @@ -29,8 +31,14 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -LRUFileCachePriority::LRUFileCachePriority(size_t max_size_, size_t max_elements_, StatePtr state_) +LRUFileCachePriority::LRUFileCachePriority( + size_t max_size_, + size_t max_elements_, + StatePtr state_, + const std::string & description_) : IFileCachePriority(max_size_, max_elements_) + , description(description_) + , log(getLogger("LRUFileCachePriority" + (description.empty() ? "" : "(" + description + ")"))) { if (state_) state = state_; @@ -63,22 +71,24 @@ LRUFileCachePriority::LRUIterator LRUFileCachePriority::add(EntryPtr entry, cons for (const auto & queue_entry : queue) { /// entry.size == 0 means entry was invalidated. - if (queue_entry->size != 0 && queue_entry->key == entry->key && queue_entry->offset == entry->offset) + if (queue_entry->size != 0 + && !queue_entry->isEvicting(lock) + && queue_entry->key == entry->key && queue_entry->offset == entry->offset) + { throw Exception( ErrorCodes::LOGICAL_ERROR, - "Attempt to add duplicate queue entry to queue. " - "(Key: {}, offset: {}, size: {})", - entry->key, entry->offset, entry->size); + "Attempt to add duplicate queue entry to queue: {}", + entry->toString()); + } } #endif - const auto & size_limit = getSizeLimit(lock); - if (size_limit && state->current_size + entry->size > size_limit) + if (!canFit(entry->size, 1, lock)) { throw Exception( ErrorCodes::LOGICAL_ERROR, - "Not enough space to add {}:{} with size {}: current size: {}/{}", - entry->key, entry->offset, entry->size, state->current_size, size_limit); + "Not enough space to add a new entry {}. Current state: {}", + entry->toString(), getStateInfoForLog(lock)); } auto iterator = queue.insert(queue.end(), entry); @@ -93,7 +103,8 @@ LRUFileCachePriority::LRUIterator LRUFileCachePriority::add(EntryPtr entry, cons return LRUIterator(this, iterator); } -LRUFileCachePriority::LRUQueue::iterator LRUFileCachePriority::remove(LRUQueue::iterator it, const CachePriorityGuard::Lock &) +LRUFileCachePriority::LRUQueue::iterator +LRUFileCachePriority::remove(LRUQueue::iterator it, const CachePriorityGuard::Lock &) { /// If size is 0, entry is invalidated, current_elements_num was already updated. const auto & entry = **it; @@ -112,6 +123,9 @@ LRUFileCachePriority::LRUQueue::iterator LRUFileCachePriority::remove(LRUQueue:: void LRUFileCachePriority::updateSize(int64_t size) { + chassert(size != 0); + chassert(size > 0 || state->current_size >= size_t(-size)); + state->current_size += size; CurrentMetrics::add(CurrentMetrics::FilesystemCacheSize, size); } @@ -155,9 +169,32 @@ void LRUFileCachePriority::iterate(IterateFunc && func, const CachePriorityGuard for (auto it = queue.begin(); it != queue.end();) { const auto & entry = **it; + + if (entry.size == 0) + { + /// entry.size == 0 means that queue entry was invalidated, + /// valid (active) queue entries always have size > 0, + /// so we can safely remove it. + it = remove(it, lock); + continue; + } + + if (entry.isEvicting(lock)) + { + /// Skip queue entries which are in evicting state. + /// We threat them the same way as deleted entries. + ++it; + ProfileEvents::increment(ProfileEvents::FilesystemCacheEvictionSkippedEvictingFileSegments); + continue; + } + auto locked_key = entry.key_metadata->tryLock(); if (!locked_key || entry.size == 0) { + /// locked_key == nullptr means that the cache key of + /// the file segment of this queue entry no longer exists. + /// This is normal if the key was removed from metadata, + /// while queue entries can be removed lazily (with delay). it = remove(it, lock); continue; } @@ -165,6 +202,9 @@ void LRUFileCachePriority::iterate(IterateFunc && func, const CachePriorityGuard auto metadata = locked_key->tryGetByOffset(entry.offset); if (!metadata) { + /// Same as explained in comment above, metadata == nullptr, + /// if file segment was removed from cache metadata, + /// but queue entry still exists because it is lazily removed. it = remove(it, lock); continue; } @@ -201,34 +241,38 @@ void LRUFileCachePriority::iterate(IterateFunc && func, const CachePriorityGuard bool LRUFileCachePriority::canFit( /// NOLINT size_t size, + size_t elements, const CachePriorityGuard::Lock & lock, IteratorPtr, bool) const { - return canFit(size, 0, 0, lock); + return canFit(size, elements, 0, 0, lock); } bool LRUFileCachePriority::canFit( size_t size, + size_t elements, size_t released_size_assumption, size_t released_elements_assumption, const CachePriorityGuard::Lock &) const { - return (max_size == 0 || (state->current_size + size - released_size_assumption <= max_size)) - && (max_elements == 0 || state->current_elements_num + 1 - released_elements_assumption <= max_elements); + return (max_size == 0 || state->current_size + size - released_size_assumption <= max_size) + && (max_elements == 0 || state->current_elements_num + elements - released_elements_assumption <= max_elements); } bool LRUFileCachePriority::collectCandidatesForEviction( size_t size, + size_t elements, FileCacheReserveStat & stat, EvictionCandidates & res, - IFileCachePriority::IteratorPtr, - FinalizeEvictionFunc &, + IFileCachePriority::IteratorPtr /* reservee */, const UserID &, const CachePriorityGuard::Lock & lock) { - if (canFit(size, lock)) + if (canFit(size, elements, 0, 0, lock)) + { return true; + } ProfileEvents::increment(ProfileEvents::FilesystemCacheEvictionTries); @@ -239,7 +283,7 @@ bool LRUFileCachePriority::collectCandidatesForEviction( if (segment_metadata->releasable()) { - res.add(locked_key, segment_metadata); + res.add(segment_metadata, locked_key, lock); stat.update(segment_metadata->size(), file_segment->getKind(), true); } else @@ -253,7 +297,7 @@ bool LRUFileCachePriority::collectCandidatesForEviction( auto can_fit = [&] { - return canFit(size, stat.stat.releasable_size, stat.stat.releasable_count, lock); + return canFit(size, elements, stat.total_stat.releasable_size, stat.total_stat.releasable_count, lock); }; iterate([&](LockedKey & locked_key, const FileSegmentMetadataPtr & segment_metadata) @@ -261,10 +305,42 @@ bool LRUFileCachePriority::collectCandidatesForEviction( return can_fit() ? IterationResult::BREAK : iterate_func(locked_key, segment_metadata); }, lock); - return can_fit(); + if (can_fit()) + { + /// As eviction is done without a cache priority lock, + /// then if some space was partially available and some needed + /// to be freed via eviction, we need to make sure that this + /// partially available space is still available + /// after we finish with eviction for non-available space. + /// So we create a space holder for the currently available part + /// of the required space for the duration of eviction of the other + /// currently non-available part of the space. + + const size_t hold_size = size > stat.total_stat.releasable_size + ? size - stat.total_stat.releasable_size + : 0; + + const size_t hold_elements = elements > stat.total_stat.releasable_count + ? elements - stat.total_stat.releasable_count + : 0; + + if (hold_size || hold_elements) + res.setSpaceHolder(hold_size, hold_elements, *this, lock); + + // LOG_TEST(log, "Collected {} candidates for eviction (total size: {}). " + // "Took hold of size {} and elements {}", + // res.size(), stat.total_stat.releasable_size, hold_size, hold_elements); + + return true; + } + else + { + return false; + } } -LRUFileCachePriority::LRUIterator LRUFileCachePriority::move(LRUIterator & it, LRUFileCachePriority & other, const CachePriorityGuard::Lock &) +LRUFileCachePriority::LRUIterator +LRUFileCachePriority::move(LRUIterator & it, LRUFileCachePriority & other, const CachePriorityGuard::Lock &) { const auto & entry = *it.getEntry(); if (entry.size == 0) @@ -281,9 +357,8 @@ LRUFileCachePriority::LRUIterator LRUFileCachePriority::move(LRUIterator & it, L if (queue_entry->size != 0 && queue_entry->key == entry.key && queue_entry->offset == entry.offset) throw Exception( ErrorCodes::LOGICAL_ERROR, - "Attempt to add duplicate queue entry to queue. " - "(Key: {}, offset: {}, size: {})", - entry.key, entry.offset, entry.size); + "Attempt to add duplicate queue entry to queue: {}", + entry.toString()); } #endif @@ -365,34 +440,61 @@ void LRUFileCachePriority::LRUIterator::invalidate() assertValid(); const auto & entry = *iterator; - LOG_TEST( - cache_priority->log, - "Invalidating entry in LRU queue. Key: {}, offset: {}, previous size: {}", - entry->key, entry->offset, entry->size); + LOG_TEST(cache_priority->log, + "Invalidating entry in LRU queue entry {}", entry->toString()); + chassert(entry->size != 0); cache_priority->updateSize(-entry->size); cache_priority->updateElementsCount(-1); entry->size = 0; } -void LRUFileCachePriority::LRUIterator::updateSize(int64_t size) +void LRUFileCachePriority::LRUIterator::incrementSize(size_t size, const CachePriorityGuard::Lock & lock) +{ + chassert(size); + assertValid(); + + const auto & entry = *iterator; + + if (!cache_priority->canFit(size, /* elements */0, lock)) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot increment size by {} for entry {}. Current state: {}", + size, entry->toString(), cache_priority->getStateInfoForLog(lock)); + } + + LOG_TEST( + cache_priority->log, + "Incrementing size with {} in LRU queue for entry {}", + size, entry->toString()); + + cache_priority->updateSize(size); + entry->size += size; + + cache_priority->check(lock); +} + +void LRUFileCachePriority::LRUIterator::decrementSize(size_t size) { assertValid(); const auto & entry = *iterator; - LOG_TEST( - cache_priority->log, - "Update size with {} in LRU queue for key: {}, offset: {}, previous size: {}", - size, entry->key, entry->offset, entry->size); + LOG_TEST(cache_priority->log, + "Decrement size with {} in LRU queue entry {}", + size, entry->toString()); - cache_priority->updateSize(size); - entry->size += size; + chassert(size); + chassert(entry->size >= size); + + cache_priority->updateSize(-size); + entry->size -= size; } -size_t LRUFileCachePriority::LRUIterator::increasePriority(const CachePriorityGuard::Lock &) +size_t LRUFileCachePriority::LRUIterator::increasePriority(const CachePriorityGuard::Lock & lock) { assertValid(); cache_priority->queue.splice(cache_priority->queue.end(), cache_priority->queue, iterator); + cache_priority->check(lock); return ++((*iterator)->hits); } @@ -414,4 +516,42 @@ void LRUFileCachePriority::shuffle(const CachePriorityGuard::Lock &) queue.splice(queue.end(), queue, it); } +std::string LRUFileCachePriority::getStateInfoForLog(const CachePriorityGuard::Lock & lock) const +{ + return fmt::format("size: {}/{}, elements: {}/{} (description: {})", + getSize(lock), max_size, getElementsCount(lock), max_elements, description); +} + +void LRUFileCachePriority::holdImpl( + size_t size, + size_t elements, + const CachePriorityGuard::Lock & lock) +{ + chassert(size || elements); + + if (!canFit(size, elements, lock)) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot take space {} in size and {} in elements. " + "Current state {}/{} in size, {}/{} in elements", + size, elements, state->current_size, max_size, + state->current_elements_num, max_elements); + } + + state->current_size += size; + state->current_elements_num += elements; + + // LOG_TEST(log, "Hold {} by size and {} by elements", size, elements); +} + +void LRUFileCachePriority::releaseImpl(size_t size, size_t elements) +{ + chassert(size || elements); + + state->current_size -= size; + state->current_elements_num -= elements; + + // LOG_TEST(log, "Released {} by size and {} by elements", size, elements); +} + } diff --git a/src/Interpreters/Cache/LRUFileCachePriority.h b/src/Interpreters/Cache/LRUFileCachePriority.h index 49977c79b81..31968d61196 100644 --- a/src/Interpreters/Cache/LRUFileCachePriority.h +++ b/src/Interpreters/Cache/LRUFileCachePriority.h @@ -22,7 +22,11 @@ protected: using StatePtr = std::shared_ptr; public: - LRUFileCachePriority(size_t max_size_, size_t max_elements_, StatePtr state_ = nullptr); + LRUFileCachePriority( + size_t max_size_, + size_t max_elements_, + StatePtr state_ = nullptr, + const std::string & description_ = "none"); size_t getSize(const CachePriorityGuard::Lock &) const override { return state->current_size; } @@ -32,8 +36,11 @@ public: size_t getElementsCountApprox() const override { return state->current_elements_num; } + std::string getStateInfoForLog(const CachePriorityGuard::Lock & lock) const override; + bool canFit( /// NOLINT size_t size, + size_t elements, const CachePriorityGuard::Lock &, IteratorPtr reservee = nullptr, bool best_effort = false) const override; @@ -48,10 +55,10 @@ public: bool collectCandidatesForEviction( size_t size, + size_t elements, FileCacheReserveStat & stat, EvictionCandidates & res, IFileCachePriority::IteratorPtr reservee, - FinalizeEvictionFunc & finalize_eviction_func, const UserID & user_id, const CachePriorityGuard::Lock &) override; @@ -75,13 +82,19 @@ private: friend class SLRUFileCachePriority; LRUQueue queue; - LoggerPtr log = getLogger("LRUFileCachePriority"); + const std::string description; + LoggerPtr log; StatePtr state; void updateElementsCount(int64_t num); void updateSize(int64_t size); - bool canFit(size_t size, size_t released_size_assumption, size_t released_elements_assumption, const CachePriorityGuard::Lock &) const; + bool canFit( + size_t size, + size_t elements, + size_t released_size_assumption, + size_t released_elements_assumption, + const CachePriorityGuard::Lock &) const; LRUQueue::iterator remove(LRUQueue::iterator it, const CachePriorityGuard::Lock &); @@ -96,6 +109,13 @@ private: LRUIterator move(LRUIterator & it, LRUFileCachePriority & other, const CachePriorityGuard::Lock &); LRUIterator add(EntryPtr entry, const CachePriorityGuard::Lock &); + + void holdImpl( + size_t size, + size_t elements, + const CachePriorityGuard::Lock & lock) override; + + void releaseImpl(size_t size, size_t elements) override; }; class LRUFileCachePriority::LRUIterator : public IFileCachePriority::Iterator @@ -118,7 +138,9 @@ public: void invalidate() override; - void updateSize(int64_t size) override; + void incrementSize(size_t size, const CachePriorityGuard::Lock &) override; + + void decrementSize(size_t size) override; QueueEntryType getType() const override { return QueueEntryType::LRU; } diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp index b79605622b6..2cbd56ba0bc 100644 --- a/src/Interpreters/Cache/Metadata.cpp +++ b/src/Interpreters/Cache/Metadata.cpp @@ -615,7 +615,7 @@ void CacheMetadata::downloadThreadFunc(const bool & stop_flag) continue; auto file_segment_metadata = locked_key->tryGetByOffset(offset); - if (!file_segment_metadata || file_segment_metadata->evicting()) + if (!file_segment_metadata || file_segment_metadata->isEvicting(*locked_key)) continue; auto file_segment = file_segment_weak.lock(); @@ -881,7 +881,7 @@ bool LockedKey::removeAllFileSegments(bool if_releasable) removed_all = false; continue; } - else if (it->second->evicting()) + else if (it->second->isEvicting(*this)) { /// File segment is currently a removal candidate, /// we do not know if it will be removed or not yet, @@ -899,32 +899,34 @@ bool LockedKey::removeAllFileSegments(bool if_releasable) return removed_all; } -KeyMetadata::iterator LockedKey::removeFileSegment(size_t offset, bool can_be_broken) +KeyMetadata::iterator LockedKey::removeFileSegment(size_t offset, bool can_be_broken, bool invalidate_queue_entry) { auto it = key_metadata->find(offset); if (it == key_metadata->end()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "There is no offset {}", offset); auto file_segment = it->second->file_segment; - return removeFileSegmentImpl(it, file_segment->lock(), can_be_broken); + return removeFileSegmentImpl(it, file_segment->lock(), can_be_broken, invalidate_queue_entry); } KeyMetadata::iterator LockedKey::removeFileSegment( size_t offset, const FileSegmentGuard::Lock & segment_lock, - bool can_be_broken) + bool can_be_broken, + bool invalidate_queue_entry) { auto it = key_metadata->find(offset); if (it == key_metadata->end()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "There is no offset {} in key {}", offset, getKey()); - return removeFileSegmentImpl(it, segment_lock, can_be_broken); + return removeFileSegmentImpl(it, segment_lock, can_be_broken, invalidate_queue_entry); } KeyMetadata::iterator LockedKey::removeFileSegmentImpl( KeyMetadata::iterator it, const FileSegmentGuard::Lock & segment_lock, - bool can_be_broken) + bool can_be_broken, + bool invalidate_queue_entry) { auto file_segment = it->second->file_segment; @@ -934,7 +936,7 @@ KeyMetadata::iterator LockedKey::removeFileSegmentImpl( chassert(can_be_broken || file_segment->assertCorrectnessUnlocked(segment_lock)); - if (file_segment->queue_iterator) + if (file_segment->queue_iterator && invalidate_queue_entry) file_segment->queue_iterator->invalidate(); file_segment->detach(segment_lock, *this); @@ -1016,7 +1018,7 @@ void LockedKey::shrinkFileSegmentToDownloadedSize( file_segment->cache, key_metadata, file_segment->queue_iterator); if (diff) - metadata->getQueueIterator()->updateSize(-diff); + metadata->getQueueIterator()->decrementSize(diff); chassert(file_segment->assertCorrectnessUnlocked(segment_lock)); } @@ -1102,7 +1104,7 @@ std::vector LockedKey::sync() std::vector broken; for (auto it = key_metadata->begin(); it != key_metadata->end();) { - if (it->second->evicting() || !it->second->releasable()) + if (it->second->isEvicting(*this) || !it->second->releasable()) { ++it; continue; diff --git a/src/Interpreters/Cache/Metadata.h b/src/Interpreters/Cache/Metadata.h index c02127cdef3..7efc83c331a 100644 --- a/src/Interpreters/Cache/Metadata.h +++ b/src/Interpreters/Cache/Metadata.h @@ -19,6 +19,10 @@ using DownloadQueuePtr = std::shared_ptr; using FileSegmentsHolderPtr = std::unique_ptr; class CacheMetadata; +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} struct FileSegmentMetadata : private boost::noncopyable { @@ -30,12 +34,41 @@ struct FileSegmentMetadata : private boost::noncopyable size_t size() const; - bool evicting() const { return removal_candidate.load(); } + bool isEvicting(const CachePriorityGuard::Lock & lock) const + { + auto iterator = getQueueIterator(); + if (!iterator) + return false; + return iterator->getEntry()->isEvicting(lock); + } + + bool isEvicting(const LockedKey & lock) const + { + auto iterator = getQueueIterator(); + if (!iterator) + return false; + return iterator->getEntry()->isEvicting(lock); + } + + void setEvictingFlag(const LockedKey & locked_key, const CachePriorityGuard::Lock & lock) const + { + auto iterator = getQueueIterator(); + if (!iterator) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Iterator is not set"); + iterator->getEntry()->setEvictingFlag(locked_key, lock); + } + + void resetEvictingFlag() const + { + auto iterator = getQueueIterator(); + if (!iterator) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Iterator is not set"); + iterator->getEntry()->resetEvictingFlag(); + } Priority::IteratorPtr getQueueIterator() const { return file_segment->getQueueIterator(); } FileSegmentPtr file_segment; - std::atomic removal_candidate{false}; }; using FileSegmentMetadataPtr = std::shared_ptr; @@ -269,8 +302,16 @@ struct LockedKey : private boost::noncopyable bool removeAllFileSegments(bool if_releasable = true); - KeyMetadata::iterator removeFileSegment(size_t offset, const FileSegmentGuard::Lock &, bool can_be_broken = false); - KeyMetadata::iterator removeFileSegment(size_t offset, bool can_be_broken = false); + KeyMetadata::iterator removeFileSegment( + size_t offset, + const FileSegmentGuard::Lock &, + bool can_be_broken = false, + bool invalidate_queue_entry = true); + + KeyMetadata::iterator removeFileSegment( + size_t offset, + bool can_be_broken = false, + bool invalidate_queue_entry = true); void shrinkFileSegmentToDownloadedSize(size_t offset, const FileSegmentGuard::Lock &); @@ -289,7 +330,11 @@ struct LockedKey : private boost::noncopyable std::string toString() const; private: - KeyMetadata::iterator removeFileSegmentImpl(KeyMetadata::iterator it, const FileSegmentGuard::Lock &, bool can_be_broken = false); + KeyMetadata::iterator removeFileSegmentImpl( + KeyMetadata::iterator it, + const FileSegmentGuard::Lock &, + bool can_be_broken = false, + bool invalidate_queue_entry = true); const std::shared_ptr key_metadata; KeyGuard::Lock lock; /// `lock` must be destructed before `key_metadata`. diff --git a/src/Interpreters/Cache/QueryLimit.cpp b/src/Interpreters/Cache/QueryLimit.cpp index 9421005dc92..6a5b5bf67ca 100644 --- a/src/Interpreters/Cache/QueryLimit.cpp +++ b/src/Interpreters/Cache/QueryLimit.cpp @@ -1,6 +1,7 @@ -#include -#include #include +#include +#include +#include namespace DB { diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.cpp b/src/Interpreters/Cache/SLRUFileCachePriority.cpp index c97d05d4b84..1400d3219c6 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.cpp +++ b/src/Interpreters/Cache/SLRUFileCachePriority.cpp @@ -10,11 +10,16 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + namespace { size_t getRatio(size_t total, double ratio) { - return static_cast(total * std::clamp(ratio, 0.0, 1.0)); + return std::lround(total * std::clamp(ratio, 0.0, 1.0)); } } @@ -26,12 +31,20 @@ SLRUFileCachePriority::SLRUFileCachePriority( LRUFileCachePriority::StatePtr protected_state_) : IFileCachePriority(max_size_, max_elements_) , size_ratio(size_ratio_) - , protected_queue(LRUFileCachePriority(getRatio(max_size_, size_ratio), getRatio(max_elements_, size_ratio), protected_state_)) - , probationary_queue(LRUFileCachePriority(getRatio(max_size_, 1 - size_ratio), getRatio(max_elements_, 1 - size_ratio), probationary_state_)) + , protected_queue(LRUFileCachePriority(getRatio(max_size_, size_ratio), + getRatio(max_elements_, size_ratio), + protected_state_, + "protected")) + , probationary_queue(LRUFileCachePriority(getRatio(max_size_, 1 - size_ratio), + getRatio(max_elements_, 1 - size_ratio), + probationary_state_, + "probationary")) { LOG_DEBUG( - log, "Using probationary queue size: {}, protected queue size: {}", - probationary_queue.max_size, protected_queue.max_elements); + log, "Probationary queue {} in size and {} in elements. " + "Protected queue {} in size and {} in elements", + probationary_queue.max_size, probationary_queue.max_elements, + protected_queue.max_size, protected_queue.max_elements); } size_t SLRUFileCachePriority::getSize(const CachePriorityGuard::Lock & lock) const @@ -56,23 +69,24 @@ size_t SLRUFileCachePriority::getElementsCountApprox() const bool SLRUFileCachePriority::canFit( /// NOLINT size_t size, + size_t elements, const CachePriorityGuard::Lock & lock, IteratorPtr reservee, bool best_effort) const { if (best_effort) - return probationary_queue.canFit(size, lock) || protected_queue.canFit(size, lock); + return probationary_queue.canFit(size, elements, lock) || protected_queue.canFit(size, elements, lock); if (reservee) { const auto * slru_iterator = assert_cast(reservee.get()); if (slru_iterator->is_protected) - return protected_queue.canFit(size, lock); + return protected_queue.canFit(size, elements, lock); else - return probationary_queue.canFit(size, lock); + return probationary_queue.canFit(size, elements, lock); } else - return probationary_queue.canFit(size, lock); + return probationary_queue.canFit(size, elements, lock); } IFileCachePriority::IteratorPtr SLRUFileCachePriority::add( /// NOLINT @@ -83,92 +97,187 @@ IFileCachePriority::IteratorPtr SLRUFileCachePriority::add( /// NOLINT const CachePriorityGuard::Lock & lock, bool is_startup) { + IteratorPtr iterator; if (is_startup) { /// If it is server startup, we put entries in any queue it will fit in, /// but with preference for probationary queue, /// because we do not know the distribution between queues after server restart. - if (probationary_queue.canFit(size, lock)) + if (probationary_queue.canFit(size, /* elements */1, lock)) { auto lru_iterator = probationary_queue.add(std::make_shared(key_metadata->key, offset, size, key_metadata), lock); - return std::make_shared(this, std::move(lru_iterator), false); + iterator = std::make_shared(this, std::move(lru_iterator), false); } else { auto lru_iterator = protected_queue.add(std::make_shared(key_metadata->key, offset, size, key_metadata), lock); - return std::make_shared(this, std::move(lru_iterator), true); + iterator = std::make_shared(this, std::move(lru_iterator), true); } } else { auto lru_iterator = probationary_queue.add(std::make_shared(key_metadata->key, offset, size, key_metadata), lock); - return std::make_shared(this, std::move(lru_iterator), false); + iterator = std::make_shared(this, std::move(lru_iterator), false); } + + if (getSize(lock) > max_size || getElementsCount(lock) > max_elements) + { + throw Exception( + ErrorCodes::LOGICAL_ERROR, "Violated cache limits. " + "Added {} for {} element ({}:{}). Current state: {}", + size, iterator->getType(), key_metadata->key, offset, getStateInfoForLog(lock)); + } + + return iterator; } bool SLRUFileCachePriority::collectCandidatesForEviction( size_t size, + size_t elements, FileCacheReserveStat & stat, EvictionCandidates & res, IFileCachePriority::IteratorPtr reservee, - FinalizeEvictionFunc & finalize_eviction_func, const UserID & user_id, const CachePriorityGuard::Lock & lock) { - /// If `it` is nullptr, then it is the first space reservation attempt + /// If `reservee` is nullptr, then it is the first space reservation attempt /// for a corresponding file segment, so it will be directly put into probationary queue. if (!reservee) { - return probationary_queue.collectCandidatesForEviction(size, stat, res, reservee, finalize_eviction_func, user_id, lock); + return probationary_queue.collectCandidatesForEviction(size, elements, stat, res, reservee, user_id, lock); } - /// If `it` not nullptr (e.g. is already in some queue), + auto * slru_iterator = assert_cast(reservee.get()); + bool success = false; + + /// If `reservee` is not nullptr (e.g. is already in some queue), /// we need to check in which queue (protected/probationary) it currently is /// (in order to know where we need to free space). - if (!assert_cast(reservee.get())->is_protected) + if (!slru_iterator->is_protected) { - return probationary_queue.collectCandidatesForEviction(size, stat, res, reservee, finalize_eviction_func, user_id, lock); + chassert(slru_iterator->lru_iterator.cache_priority == &probationary_queue); + success = probationary_queue.collectCandidatesForEviction(size, elements, stat, res, reservee, user_id, lock); + } + else + { + chassert(slru_iterator->lru_iterator.cache_priority == &protected_queue); + /// Entry is in protected queue. + /// Check if we have enough space in protected queue to fit a new size of entry. + /// `size` is the increment to the current entry.size we want to increase. + success = collectCandidatesForEvictionInProtected(size, elements, stat, res, reservee, user_id, lock); } - /// Entry is in protected queue. - /// Check if we have enough space in protected queue to fit a new size of entry. - /// `size` is the increment to the current entry.size we want to increase. - if (protected_queue.canFit(size, lock)) - return true; + /// We eviction_candidates (res) set is non-empty and + /// space reservation was successful, we will do eviction from filesystem + /// which is executed without a cache priority lock. + /// So we made space reservation from a certain queue (protected or probationary), + /// but we should remember that there is an increasePriority operation + /// which can be called concurrently to space reservation. + /// This operation can move elements from one queue to another. + /// We need to make sure that this does not happen for the elements + /// which are in the process of unfinished space reservation. + if (success && res.size() > 0) + { + slru_iterator->movable = false; + res.onFinalize([=](const CachePriorityGuard::Lock &){ slru_iterator->movable = true; }); + } + return success; +} + +bool SLRUFileCachePriority::collectCandidatesForEvictionInProtected( + size_t size, + size_t elements, + FileCacheReserveStat & stat, + EvictionCandidates & res, + IFileCachePriority::IteratorPtr reservee, + const UserID & user_id, + const CachePriorityGuard::Lock & lock) +{ + if (protected_queue.canFit(size, elements, lock)) + { + return true; + } + + /// If not enough space - we need to "downgrade" lowest priority entries + /// from protected queue to probationary queue. - /// If not enough space - we need to "downgrade" lowest priority entries from protected - /// queue to probationary queue. - /// The amount of such "downgraded" entries is equal to the amount - /// required to make space for additionary `size` bytes for entry. auto downgrade_candidates = std::make_shared(); FileCacheReserveStat downgrade_stat; - FinalizeEvictionFunc noop; - - if (!protected_queue.collectCandidatesForEviction(size, downgrade_stat, *downgrade_candidates, reservee, noop, user_id, lock)) + if (!protected_queue.collectCandidatesForEviction(size, elements, downgrade_stat, *downgrade_candidates, reservee, user_id, lock)) + { return false; + } + else + chassert(downgrade_candidates->size() > 0); - const size_t size_to_downgrade = downgrade_stat.stat.releasable_size; - - if (!probationary_queue.canFit(size_to_downgrade, lock) - && !probationary_queue.collectCandidatesForEviction(size_to_downgrade, stat, res, reservee, noop, user_id, lock)) + if (!probationary_queue.collectCandidatesForEviction( + downgrade_stat.total_stat.releasable_size, downgrade_stat.total_stat.releasable_count, + stat, res, reservee, user_id, lock)) + { return false; + } - finalize_eviction_func = [=, this](const CachePriorityGuard::Lock & lk) mutable + auto downgrade_func = [=, this](const CachePriorityGuard::Lock & lk) { for (const auto & [key, key_candidates] : *downgrade_candidates) { for (const auto & candidate : key_candidates.candidates) - { - auto * candidate_it = assert_cast(candidate->getQueueIterator().get()); - candidate_it->lru_iterator = probationary_queue.move(candidate_it->lru_iterator, protected_queue, lk); - candidate_it->is_protected = false; - } + downgrade(candidate->getQueueIterator(), lk); } }; + if (res.size() > 0) + { + LOG_TEST(log, "Setting up delayed downgrade for {} elements " + "from protected to probationary. Total size: {}", + downgrade_candidates->size(), downgrade_stat.total_stat.releasable_size); + + /// Downgrade from protected to probationary only after + /// we free up space in probationary (in order to fit these downgrade candidates). + res.onFinalize(std::move(downgrade_func)); + } + else + { + LOG_TEST(log, "Downgrading {} elements from protected to probationary. " + "Total size: {}", + downgrade_candidates->size(), downgrade_stat.total_stat.releasable_size); + + /// Enough space in probationary queue already to fit our downgrade candidates. + downgrade_func(lock); + } + return true; } +void SLRUFileCachePriority::downgrade(IteratorPtr iterator, const CachePriorityGuard::Lock & lock) +{ + auto * candidate_it = assert_cast(iterator.get()); + if (!candidate_it->is_protected) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot downgrade {}: it is already in probationary queue", + candidate_it->getEntry()->toString()); + } + if (!candidate_it->movable) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot downgrade {}: it is non-movable", + candidate_it->getEntry()->toString()); + } + + const size_t entry_size = candidate_it->entry->size; + if (!probationary_queue.canFit(entry_size, 1, lock)) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Cannot downgrade {}: not enough space: {}", + candidate_it->getEntry()->toString(), + probationary_queue.getStateInfoForLog(lock)); + } + + candidate_it->lru_iterator = probationary_queue.move(candidate_it->lru_iterator, protected_queue, lock); + candidate_it->is_protected = false; +} + void SLRUFileCachePriority::increasePriority(SLRUIterator & iterator, const CachePriorityGuard::Lock & lock) { /// If entry is already in protected queue, @@ -179,10 +288,18 @@ void SLRUFileCachePriority::increasePriority(SLRUIterator & iterator, const Cach return; } + if (!iterator.movable) + { + iterator.lru_iterator.increasePriority(lock); + return; + } + + chassert(iterator.lru_iterator.cache_priority == &probationary_queue); + /// Entry is in probationary queue. /// We need to move it to protected queue. - const size_t size = iterator.getEntry()->size; - if (size > protected_queue.getSizeLimit(lock)) + const size_t entry_size = iterator.getEntry()->size; + if (entry_size > protected_queue.getSizeLimit(lock)) { /// Entry size is bigger than the whole protected queue limit. /// This is only possible if protected_queue_size_limit is less than max_file_segment_size, @@ -191,67 +308,79 @@ void SLRUFileCachePriority::increasePriority(SLRUIterator & iterator, const Cach return; } - /// Check if there is enough space in protected queue to move entry there. - /// If not - we need to "downgrade" lowest priority entries from protected - /// queue to probationary queue. - EvictionCandidates downgrade_candidates; - FileCacheReserveStat downgrade_stat; - FinalizeEvictionFunc noop; + EntryPtr entry = iterator.getEntry(); + /// We need to remove the entry from probationary first + /// in order to make space for downgrade from protected. + iterator.lru_iterator.remove(lock); - if (!protected_queue.collectCandidatesForEviction(size, downgrade_stat, downgrade_candidates, {}, noop, "", lock)) - { - /// We cannot make space for entry to be moved to protected queue - /// (not enough releasable file segments). - /// Then just increase its priority within probationary queue. - iterator.lru_iterator.increasePriority(lock); - return; - } - - /// The amount of such "downgraded" entries is equal to the amount - /// required to make space for entry we want to insert. - const size_t size_to_downgrade = downgrade_stat.stat.releasable_count; - size_t size_to_free = 0; - if (size_to_downgrade && size_to_downgrade > size) - size_to_free = size_to_downgrade - size; - - /// Now we need to check if those "downgrade" candidates can actually - /// be moved to probationary queue. EvictionCandidates eviction_candidates; FileCacheReserveStat stat; - - if (size_to_free) + try { - if (!probationary_queue.collectCandidatesForEviction(size_to_free, stat, eviction_candidates, {}, noop, {}, lock)) + /// Check if there is enough space in protected queue to move entry there. + /// If not - we need to "downgrade" lowest priority entries from protected + /// queue to probationary queue. + /// + if (!collectCandidatesForEvictionInProtected( + entry->size, /* elements */1, stat, eviction_candidates, nullptr, FileCache::getInternalUser().user_id, lock)) { /// "downgrade" candidates cannot be moved to probationary queue, /// so entry cannot be moved to protected queue as well. /// Then just increase its priority within probationary queue. - iterator.lru_iterator.increasePriority(lock); + iterator.lru_iterator = addOrThrow(entry, probationary_queue, lock); return; } - /// Make space for "downgrade" candidates. - eviction_candidates.evict(nullptr, lock); + + eviction_candidates.evict(); + eviction_candidates.finalize(nullptr, lock); } - - /// All checks passed, now we can move downgrade candidates to - /// probationary queue and our entry to protected queue. - EntryPtr entry = iterator.getEntry(); - iterator.lru_iterator.remove(lock); - - for (const auto & [key, key_candidates] : downgrade_candidates) + catch (...) { - for (const auto & candidate : key_candidates.candidates) - { - auto * candidate_it = assert_cast(candidate->getQueueIterator().get()); - candidate_it->lru_iterator = probationary_queue.move(candidate_it->lru_iterator, protected_queue, lock); - candidate_it->is_protected = false; - } + iterator.lru_iterator = addOrThrow(entry, probationary_queue, lock); + throw; } - iterator.lru_iterator = protected_queue.add(entry, lock); + iterator.lru_iterator = addOrThrow(entry, protected_queue, lock); iterator.is_protected = true; } +LRUFileCachePriority::LRUIterator SLRUFileCachePriority::addOrThrow( + EntryPtr entry, LRUFileCachePriority & queue, const CachePriorityGuard::Lock & lock) +{ + try + { + return queue.add(entry, lock); + } + catch (...) + { + const auto initial_exception = getCurrentExceptionMessage(true); + try + { + /// We cannot allow a situation that a file exists on filesystem, but + /// there is no corresponding entry in priority queue for it, + /// because it will mean that cache became inconsistent. + /// So let's try to fix the situation. + auto metadata = entry->key_metadata->tryLock(); + chassert(metadata); + if (metadata) + { + auto segment_metadata = metadata->tryGetByOffset(entry->offset); + metadata->removeFileSegment(entry->offset, segment_metadata->file_segment->lock()); + } + } + catch (...) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Unexpected exception: {} (Initial exception: {}). Cache will become inconsistent", + getCurrentExceptionMessage(true), initial_exception); + } + + /// Let's try to catch such cases in CI. + chassert(false); + throw; + } +} + IFileCachePriority::PriorityDumpPtr SLRUFileCachePriority::dump(const CachePriorityGuard::Lock & lock) { auto res = dynamic_pointer_cast(probationary_queue.dump(lock)); @@ -301,13 +430,20 @@ size_t SLRUFileCachePriority::SLRUIterator::increasePriority(const CachePriority { assertValid(); cache_priority->increasePriority(*this, lock); + cache_priority->check(lock); return getEntry()->hits; } -void SLRUFileCachePriority::SLRUIterator::updateSize(int64_t size) +void SLRUFileCachePriority::SLRUIterator::incrementSize(size_t size, const CachePriorityGuard::Lock & lock) { assertValid(); - lru_iterator.updateSize(size); + lru_iterator.incrementSize(size, lock); +} + +void SLRUFileCachePriority::SLRUIterator::decrementSize(size_t size) +{ + assertValid(); + lru_iterator.decrementSize(size); } void SLRUFileCachePriority::SLRUIterator::invalidate() @@ -327,4 +463,23 @@ void SLRUFileCachePriority::SLRUIterator::assertValid() const lru_iterator.assertValid(); } +std::string SLRUFileCachePriority::getStateInfoForLog(const CachePriorityGuard::Lock & lock) const +{ + return fmt::format("total size {}/{}, elements {}/{}, " + "probationary queue size {}/{}, elements {}/{}, " + "protected queue size {}/{}, elements {}/{}", + getSize(lock), max_size, getElementsCount(lock), max_elements, + probationary_queue.getSize(lock), probationary_queue.max_size, + probationary_queue.getElementsCount(lock), probationary_queue.max_elements, + protected_queue.getSize(lock), protected_queue.max_size, + protected_queue.getElementsCount(lock), protected_queue.max_elements); +} + +void SLRUFileCachePriority::check(const CachePriorityGuard::Lock & lock) const +{ + probationary_queue.check(lock); + protected_queue.check(lock); + IFileCachePriority::check(lock); +} + } diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.h b/src/Interpreters/Cache/SLRUFileCachePriority.h index f90918f7878..4cf5bb0f199 100644 --- a/src/Interpreters/Cache/SLRUFileCachePriority.h +++ b/src/Interpreters/Cache/SLRUFileCachePriority.h @@ -29,8 +29,13 @@ public: size_t getElementsCountApprox() const override; + std::string getStateInfoForLog(const CachePriorityGuard::Lock & lock) const override; + + void check(const CachePriorityGuard::Lock &) const override; + bool canFit( /// NOLINT size_t size, + size_t elements, const CachePriorityGuard::Lock &, IteratorPtr reservee = nullptr, bool best_effort = false) const override; @@ -45,10 +50,10 @@ public: bool collectCandidatesForEviction( size_t size, + size_t elements, FileCacheReserveStat & stat, EvictionCandidates & res, IFileCachePriority::IteratorPtr reservee, - FinalizeEvictionFunc & finalize_eviction_func, const UserID & user_id, const CachePriorityGuard::Lock &) override; @@ -65,6 +70,22 @@ private: LoggerPtr log = getLogger("SLRUFileCachePriority"); void increasePriority(SLRUIterator & iterator, const CachePriorityGuard::Lock & lock); + + void downgrade(IteratorPtr iterator, const CachePriorityGuard::Lock &); + + bool collectCandidatesForEvictionInProtected( + size_t size, + size_t elements, + FileCacheReserveStat & stat, + EvictionCandidates & res, + IFileCachePriority::IteratorPtr reservee, + const UserID & user_id, + const CachePriorityGuard::Lock & lock); + + LRUFileCachePriority::LRUIterator addOrThrow( + EntryPtr entry, + LRUFileCachePriority & queue, + const CachePriorityGuard::Lock & lock); }; class SLRUFileCachePriority::SLRUIterator : public IFileCachePriority::Iterator @@ -84,7 +105,9 @@ public: void invalidate() override; - void updateSize(int64_t size) override; + void incrementSize(size_t size, const CachePriorityGuard::Lock &) override; + + void decrementSize(size_t size) override; QueueEntryType getType() const override { return is_protected ? QueueEntryType::SLRU_Protected : QueueEntryType::SLRU_Probationary; } @@ -98,6 +121,16 @@ private: /// but needed only in order to do FileSegment::getInfo() without any lock, /// which is done for system tables and logging. std::atomic is_protected; + /// Iterator can me marked as non-movable in case we are reserving + /// space for it. It means that we start space reservation + /// and prepare space in probationary queue, then do eviction without lock, + /// then take the lock again to finalize the eviction and we need to be sure + /// that the element is still in probationary queue. + /// Therefore we forbid concurrent priority increase for probationary entries. + /// Same goes for the downgrade of queue entries from protected to probationary. + /// (For downgrade there is no explicit check because it will fall into unreleasable state, + /// e.g. will not be taken for eviction anyway). + bool movable{true}; }; } diff --git a/src/Interpreters/Cache/WriteBufferToFileSegment.cpp b/src/Interpreters/Cache/WriteBufferToFileSegment.cpp index 51914c0a14e..acdfa0d5437 100644 --- a/src/Interpreters/Cache/WriteBufferToFileSegment.cpp +++ b/src/Interpreters/Cache/WriteBufferToFileSegment.cpp @@ -7,8 +7,9 @@ #include -#include +#include #include +#include namespace DB { diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 7030522dd2a..65fcd51529b 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1219,7 +1219,7 @@ void Context::addWarningMessageAboutDatabaseOrdinary(const String & database_nam /// We don't use getFlagsPath method, because it takes a shared lock. auto convert_databases_flag = fs::path(shared->flags_path) / "convert_ordinary_to_atomic"; auto message = fmt::format("Server has databases (for example `{}`) with Ordinary engine, which was deprecated. " - "To convert this database to a new Atomic engine, create a flag {} and make sure that ClickHouse has write permission for it. " + "To convert this database to the new Atomic engine, create a flag {} and make sure that ClickHouse has write permission for it. " "Example: sudo touch '{}' && sudo chmod 666 '{}'", database_name, convert_databases_flag.string(), convert_databases_flag.string(), convert_databases_flag.string()); diff --git a/src/Interpreters/CrashLog.cpp b/src/Interpreters/CrashLog.cpp index 410ea922429..4a8ef84fd5c 100644 --- a/src/Interpreters/CrashLog.cpp +++ b/src/Interpreters/CrashLog.cpp @@ -1,14 +1,15 @@ -#include -#include #include +#include +#include #include #include #include -#include -#include +#include +#include #include -#include +#include #include +#include #include diff --git a/src/Interpreters/DDLTask.cpp b/src/Interpreters/DDLTask.cpp index a37b4db029a..37954850851 100644 --- a/src/Interpreters/DDLTask.cpp +++ b/src/Interpreters/DDLTask.cpp @@ -16,6 +16,7 @@ #include #include #include +#include namespace DB @@ -200,6 +201,14 @@ void DDLTaskBase::parseQueryFromEntry(ContextPtr context) ParserQuery parser_query(end, settings.allow_settings_after_format_in_insert); String description; query = parseQuery(parser_query, begin, end, description, 0, settings.max_parser_depth, settings.max_parser_backtracks); + if (auto * query_drop = query->as()) + { + ASTs drops = query_drop->getRewrittenASTsOfSingleTable(); + if (drops.size() > 1) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Not supports drop multiple tables for ddl task."); + + query = drops[0]; + } } void DDLTaskBase::formatRewrittenQuery(ContextPtr context) diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 3dc700e9f93..ec6c8b5924f 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -1143,7 +1143,7 @@ void DatabaseCatalog::dequeueDroppedTableCleanup(StorageID table_id) TableMarkedAsDropped dropped_table; { std::lock_guard lock(tables_marked_dropped_mutex); - time_t latest_drop_time = std::numeric_limits::min(); + auto latest_drop_time = std::numeric_limits::min(); auto it_dropped_table = tables_marked_dropped.end(); for (auto it = tables_marked_dropped.begin(); it != tables_marked_dropped.end(); ++it) { @@ -1168,7 +1168,7 @@ void DatabaseCatalog::dequeueDroppedTableCleanup(StorageID table_id) } if (it_dropped_table == tables_marked_dropped.end()) throw Exception(ErrorCodes::UNKNOWN_TABLE, - "The drop task of table {} is in progress, has been dropped or the database engine doesn't support it", + "Table {} is being dropped, has been dropped, or the database engine does not support UNDROP", table_id.getNameForLogs()); latest_metadata_dropped_path = it_dropped_table->metadata_path; String table_metadata_path = getPathForMetadata(it_dropped_table->table_id); diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index bd56a540128..f9e24e2de70 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -1,19 +1,20 @@ #include "ExternalLoader.h" #include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include #include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include namespace DB diff --git a/src/Interpreters/FilesystemReadPrefetchesLog.cpp b/src/Interpreters/FilesystemReadPrefetchesLog.cpp index 7fb2e3d1f4c..8cea05a1857 100644 --- a/src/Interpreters/FilesystemReadPrefetchesLog.cpp +++ b/src/Interpreters/FilesystemReadPrefetchesLog.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -6,6 +5,8 @@ #include #include #include +#include +#include namespace DB diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index ddd65f95627..12a906526f6 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -3,15 +3,15 @@ #include #include -#include -#include - #include -#include -#include #include #include +#include #include +#include +#include +#include +#include #include diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index d90d2446fca..7c3bed7388c 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -513,7 +513,7 @@ ASTPtr InterpreterCreateQuery::formatProjections(const ProjectionsDescription & } ColumnsDescription InterpreterCreateQuery::getColumnsDescription( - const ASTExpressionList & columns_ast, ContextPtr context_, bool attach, bool is_restore_from_backup) + const ASTExpressionList & columns_ast, ContextPtr context_, LoadingStrictnessLevel mode) { /// First, deduce implicit types. @@ -522,7 +522,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( ASTPtr default_expr_list = std::make_shared(); NamesAndTypesList column_names_and_types; - bool make_columns_nullable = !attach && !is_restore_from_backup && context_->getSettingsRef().data_type_default_nullable; + bool make_columns_nullable = mode <= LoadingStrictnessLevel::CREATE && context_->getSettingsRef().data_type_default_nullable; for (const auto & ast : columns_ast.children) { @@ -538,7 +538,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( { column_type = DataTypeFactory::instance().get(col_decl.type); - if (attach) + if (LoadingStrictnessLevel::ATTACH <= mode) setVersionToAggregateFunctions(column_type, true); if (col_decl.null_modifier) @@ -606,10 +606,11 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( if (!default_expr_list->children.empty()) defaults_sample_block = validateColumnsDefaultsAndGetSampleBlock(default_expr_list, column_names_and_types, context_); - bool sanity_check_compression_codecs = !attach && !context_->getSettingsRef().allow_suspicious_codecs; - bool allow_experimental_codecs = attach || context_->getSettingsRef().allow_experimental_codecs; - bool enable_deflate_qpl_codec = attach || context_->getSettingsRef().enable_deflate_qpl_codec; - bool enable_zstd_qat_codec = attach || context_->getSettingsRef().enable_zstd_qat_codec; + bool skip_checks = LoadingStrictnessLevel::SECONDARY_CREATE <= mode; + bool sanity_check_compression_codecs = !skip_checks && !context_->getSettingsRef().allow_suspicious_codecs; + bool allow_experimental_codecs = skip_checks || context_->getSettingsRef().allow_experimental_codecs; + bool enable_deflate_qpl_codec = skip_checks || context_->getSettingsRef().enable_deflate_qpl_codec; + bool enable_zstd_qat_codec = skip_checks || context_->getSettingsRef().enable_zstd_qat_codec; ColumnsDescription res; auto name_type_it = column_names_and_types.begin(); @@ -675,7 +676,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( if (col_decl.stat_type) { - if (!attach && !context_->getSettingsRef().allow_experimental_statistic) + if (!skip_checks && !context_->getSettingsRef().allow_experimental_statistic) throw Exception(ErrorCodes::INCORRECT_QUERY, "Create table with statistic is now disabled. Turn on allow_experimental_statistic"); column.stat = StatisticDescription::getStatisticFromColumnDeclaration(col_decl); } @@ -692,7 +693,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( res.add(std::move(column)); } - if (!attach && !is_restore_from_backup && context_->getSettingsRef().flatten_nested) + if (mode <= LoadingStrictnessLevel::CREATE && context_->getSettingsRef().flatten_nested) res.flattenNested(); @@ -714,7 +715,8 @@ ConstraintsDescription InterpreterCreateQuery::getConstraintsDescription(const A } -InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTablePropertiesAndNormalizeCreateQuery(ASTCreateQuery & create) const +InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTablePropertiesAndNormalizeCreateQuery( + ASTCreateQuery & create, LoadingStrictnessLevel mode) const { /// Set the table engine if it was not specified explicitly. setEngine(create); @@ -740,7 +742,7 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti if (create.columns_list->columns) { - properties.columns = getColumnsDescription(*create.columns_list->columns, getContext(), create.attach, is_restore_from_backup); + properties.columns = getColumnsDescription(*create.columns_list->columns, getContext(), mode); } if (create.columns_list->indices) @@ -1207,11 +1209,14 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (!UserDefinedSQLFunctionFactory::instance().empty()) UserDefinedSQLFunctionVisitor::visit(query_ptr); + bool is_secondary_query = getContext()->getZooKeeperMetadataTransaction() && !getContext()->getZooKeeperMetadataTransaction()->isInitialQuery(); + auto mode = getLoadingStrictnessLevel(create.attach, /*force_attach*/ false, /*has_force_restore_data_flag*/ false, is_secondary_query || is_restore_from_backup); + /// Set and retrieve list of columns, indices and constraints. Set table engine if needed. Rewrite query in canonical way. - TableProperties properties = getTablePropertiesAndNormalizeCreateQuery(create); + TableProperties properties = getTablePropertiesAndNormalizeCreateQuery(create, mode); /// Check type compatible for materialized dest table and select columns - if (create.select && create.is_materialized_view && create.to_table_id && !create.attach && !is_restore_from_backup) + if (create.select && create.is_materialized_view && create.to_table_id && mode <= LoadingStrictnessLevel::CREATE) { if (StoragePtr to_table = DatabaseCatalog::instance().tryGetTable( {create.to_table_id.database_name, create.to_table_id.table_name, create.to_table_id.uuid}, @@ -1296,11 +1301,11 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) if (create.replace_table) { chassert(!ddl_guard); - return doCreateOrReplaceTable(create, properties); + return doCreateOrReplaceTable(create, properties, mode); } /// Actually creates table - bool created = doCreateTable(create, properties, ddl_guard); + bool created = doCreateTable(create, properties, ddl_guard, mode); ddl_guard.reset(); if (!created) /// Table already exists @@ -1317,11 +1322,8 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, const InterpreterCreateQuery::TableProperties & properties, - DDLGuardPtr & ddl_guard) + DDLGuardPtr & ddl_guard, LoadingStrictnessLevel mode) { - bool is_secondary_query = getContext()->getZooKeeperMetadataTransaction() && !getContext()->getZooKeeperMetadataTransaction()->isInitialQuery(); - auto mode = getLoadingStrictnessLevel(create.attach, /*force_attach*/ false, /*has_force_restore_data_flag*/ false, is_secondary_query); - if (create.temporary) { if (create.if_not_exists && getContext()->tryResolveStorageID({"", create.getTable()}, Context::ResolveExternal)) @@ -1557,7 +1559,7 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create, - const InterpreterCreateQuery::TableProperties & properties) + const InterpreterCreateQuery::TableProperties & properties, LoadingStrictnessLevel mode) { /// Replicated database requires separate contexts for each DDL query ContextPtr current_context = getContext(); @@ -1607,7 +1609,7 @@ BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create, { /// Create temporary table (random name will be generated) DDLGuardPtr ddl_guard; - [[maybe_unused]] bool done = InterpreterCreateQuery(query_ptr, create_context).doCreateTable(create, properties, ddl_guard); + [[maybe_unused]] bool done = InterpreterCreateQuery(query_ptr, create_context).doCreateTable(create, properties, ddl_guard, mode); ddl_guard.reset(); assert(done); created = true; diff --git a/src/Interpreters/InterpreterCreateQuery.h b/src/Interpreters/InterpreterCreateQuery.h index 865f2736784..71bdeda05df 100644 --- a/src/Interpreters/InterpreterCreateQuery.h +++ b/src/Interpreters/InterpreterCreateQuery.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -73,7 +74,7 @@ public: /// Obtain information about columns, their types, default values and column comments, /// for case when columns in CREATE query is specified explicitly. - static ColumnsDescription getColumnsDescription(const ASTExpressionList & columns, ContextPtr context, bool attach, bool is_restore_from_backup); + static ColumnsDescription getColumnsDescription(const ASTExpressionList & columns, ContextPtr context, LoadingStrictnessLevel mode); static ConstraintsDescription getConstraintsDescription(const ASTExpressionList * constraints); static void prepareOnClusterQuery(ASTCreateQuery & create, ContextPtr context, const String & cluster_name); @@ -96,14 +97,14 @@ private: BlockIO createTable(ASTCreateQuery & create); /// Calculate list of columns, constraints, indices, etc... of table. Rewrite query in canonical way. - TableProperties getTablePropertiesAndNormalizeCreateQuery(ASTCreateQuery & create) const; + TableProperties getTablePropertiesAndNormalizeCreateQuery(ASTCreateQuery & create, LoadingStrictnessLevel mode) const; void validateTableStructure(const ASTCreateQuery & create, const TableProperties & properties) const; void setEngine(ASTCreateQuery & create) const; AccessRightsElements getRequiredAccess() const; /// Create IStorage and add it to database. If table already exists and IF NOT EXISTS specified, do nothing and return false. - bool doCreateTable(ASTCreateQuery & create, const TableProperties & properties, DDLGuardPtr & ddl_guard); - BlockIO doCreateOrReplaceTable(ASTCreateQuery & create, const InterpreterCreateQuery::TableProperties & properties); + bool doCreateTable(ASTCreateQuery & create, const TableProperties & properties, DDLGuardPtr & ddl_guard, LoadingStrictnessLevel mode); + BlockIO doCreateOrReplaceTable(ASTCreateQuery & create, const InterpreterCreateQuery::TableProperties & properties, LoadingStrictnessLevel mode); /// Inserts data in created table if it's CREATE ... SELECT BlockIO fillTableIfNeeded(const ASTCreateQuery & create); diff --git a/src/Interpreters/InterpreterDeleteQuery.cpp b/src/Interpreters/InterpreterDeleteQuery.cpp index 8fb0dabb5b5..07d23be78a7 100644 --- a/src/Interpreters/InterpreterDeleteQuery.cpp +++ b/src/Interpreters/InterpreterDeleteQuery.cpp @@ -101,7 +101,7 @@ BlockIO InterpreterDeleteQuery::execute() DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); auto context = Context::createCopy(getContext()); - context->setSetting("mutations_sync", 2); /// Lightweight delete is always synchronous + context->setSetting("mutations_sync", Field(context->getSettingsRef().lightweight_deletes_sync)); InterpreterAlterQuery alter_interpreter(alter_ast, context); return alter_interpreter.execute(); } diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index a7040709034..e29e59ee4c3 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -54,16 +55,27 @@ InterpreterDropQuery::InterpreterDropQuery(const ASTPtr & query_ptr_, ContextMut { } - BlockIO InterpreterDropQuery::execute() { + BlockIO res; auto & drop = query_ptr->as(); + ASTs drops = drop.getRewrittenASTsOfSingleTable(); + for (const auto & drop_query_ptr : drops) + { + current_query_ptr = drop_query_ptr; + res = executeSingleDropQuery(drop_query_ptr); + } + return res; +} - if (!drop.cluster.empty() && drop.table && !drop.if_empty && !maybeRemoveOnCluster(query_ptr, getContext())) +BlockIO InterpreterDropQuery::executeSingleDropQuery(const ASTPtr & drop_query_ptr) +{ + auto & drop = drop_query_ptr->as(); + if (!drop.cluster.empty() && drop.table && !drop.if_empty && !maybeRemoveOnCluster(current_query_ptr, getContext())) { DDLQueryOnClusterParams params; params.access_to_check = getRequiredAccessForDDLOnCluster(); - return executeDDLQueryOnCluster(query_ptr, getContext(), params); + return executeDDLQueryOnCluster(current_query_ptr, getContext(), params); } if (getContext()->getSettingsRef().database_atomic_wait_for_drop_and_detach_synchronously) @@ -71,11 +83,11 @@ BlockIO InterpreterDropQuery::execute() if (drop.table) return executeToTable(drop); - else if (drop.database && !drop.cluster.empty() && !maybeRemoveOnCluster(query_ptr, getContext())) + else if (drop.database && !drop.cluster.empty() && !maybeRemoveOnCluster(current_query_ptr, getContext())) { - DDLQueryOnClusterParams params; - params.access_to_check = getRequiredAccessForDDLOnCluster(); - return executeDDLQueryOnCluster(query_ptr, getContext(), params); + DDLQueryOnClusterParams params; + params.access_to_check = getRequiredAccessForDDLOnCluster(); + return executeDDLQueryOnCluster(current_query_ptr, getContext(), params); } else if (drop.database) return executeToDatabase(drop); @@ -83,7 +95,6 @@ BlockIO InterpreterDropQuery::execute() throw Exception(ErrorCodes::LOGICAL_ERROR, "Nothing to drop, both names are empty"); } - void InterpreterDropQuery::waitForTableToBeActuallyDroppedOrDetached(const ASTDropQuery & query, const DatabasePtr & db, const UUID & uuid_to_wait) { if (uuid_to_wait == UUIDHelpers::Nil) @@ -156,7 +167,7 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue table_id.uuid = database->tryGetTableUUID(table_id.table_name); /// Prevents recursive drop from drop database query. The original query must specify a table. - bool is_drop_or_detach_database = !query_ptr->as()->table; + bool is_drop_or_detach_database = !current_query_ptr->as()->table; AccessFlags drop_storage; @@ -179,7 +190,7 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue return executeDDLQueryOnCluster(new_query_ptr, getContext(), params); } - if (database->shouldReplicateQuery(getContext(), query_ptr)) + if (database->shouldReplicateQuery(getContext(), current_query_ptr)) { if (query.kind == ASTDropQuery::Kind::Detach) context_->checkAccess(drop_storage, table_id); @@ -249,7 +260,7 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue auto metadata_snapshot = table->getInMemoryMetadataPtr(); /// Drop table data, don't touch metadata - table->truncate(query_ptr, metadata_snapshot, context_, table_excl_lock); + table->truncate(current_query_ptr, metadata_snapshot, context_, table_excl_lock); } else if (query.kind == ASTDropQuery::Kind::Drop) { @@ -308,7 +319,7 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(const String & table_name, = table->lockExclusively(getContext()->getCurrentQueryId(), getContext()->getSettingsRef().lock_acquire_timeout); /// Drop table data, don't touch metadata auto metadata_snapshot = table->getInMemoryMetadataPtr(); - table->truncate(query_ptr, metadata_snapshot, getContext(), table_lock); + table->truncate(current_query_ptr, metadata_snapshot, getContext(), table_lock); } else if (kind == ASTDropQuery::Kind::Drop) { @@ -388,7 +399,7 @@ BlockIO InterpreterDropQuery::executeToDatabaseImpl(const ASTDropQuery & query, query_for_table.kind = query.kind; // For truncate operation on database, drop the tables if (truncate) - query_for_table.kind = ASTDropQuery::Kind::Drop; + query_for_table.kind = query.has_all_tables ? ASTDropQuery::Kind::Truncate : ASTDropQuery::Kind::Drop; query_for_table.if_exists = true; query_for_table.if_empty = false; query_for_table.setDatabase(database_name); @@ -441,11 +452,39 @@ BlockIO InterpreterDropQuery::executeToDatabaseImpl(const ASTDropQuery & query, return {}; } +void InterpreterDropQuery::extendQueryLogElemImpl(DB::QueryLogElement & elem, const DB::ASTPtr & ast, DB::ContextPtr context_) const +{ + auto & drop = ast->as(); + if (drop.database_and_tables) + { + auto & list = drop.database_and_tables->as(); + for (auto & child : list.children) + { + auto identifier = dynamic_pointer_cast(child); + if (!identifier) + throw Exception(ErrorCodes::SYNTAX_ERROR, "Unexpected type for list of table names."); + + String query_database = identifier->getDatabaseName(); + String query_table = identifier->shortName(); + if (!query_database.empty() && query_table.empty()) + { + elem.query_databases.insert(backQuoteIfNeed(query_database)); + } + else if (!query_table.empty()) + { + auto quoted_database = query_database.empty() ? backQuoteIfNeed(context_->getCurrentDatabase()) + : backQuoteIfNeed(query_database); + elem.query_databases.insert(quoted_database); + elem.query_tables.insert(quoted_database + "." + backQuoteIfNeed(query_table)); + } + } + } +} AccessRightsElements InterpreterDropQuery::getRequiredAccessForDDLOnCluster() const { AccessRightsElements required_access; - const auto & drop = query_ptr->as(); + const auto & drop = current_query_ptr->as(); if (!drop.table) { @@ -518,7 +557,7 @@ bool InterpreterDropQuery::supportsTransactions() const return drop.cluster.empty() && !drop.temporary && drop.kind == ASTDropQuery::Kind::Truncate - && drop.table; + && drop.database_and_tables; } void registerInterpreterDropQuery(InterpreterFactory & factory) diff --git a/src/Interpreters/InterpreterDropQuery.h b/src/Interpreters/InterpreterDropQuery.h index 7ae544a7356..08668f47225 100644 --- a/src/Interpreters/InterpreterDropQuery.h +++ b/src/Interpreters/InterpreterDropQuery.h @@ -29,10 +29,14 @@ public: bool supportsTransactions() const override; + void extendQueryLogElemImpl(QueryLogElement & elem, const ASTPtr & ast, ContextPtr context_) const override; + private: AccessRightsElements getRequiredAccessForDDLOnCluster() const; ASTPtr query_ptr; + ASTPtr current_query_ptr; + BlockIO executeSingleDropQuery(const ASTPtr & drop_query_ptr); BlockIO executeToDatabase(const ASTDropQuery & query); BlockIO executeToDatabaseImpl(const ASTDropQuery & query, DatabasePtr & database, std::vector & uuids_to_wait); diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 6bbf03bb1e0..c47e3bdc49f 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -798,7 +798,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( != parallel_replicas_before_analysis) { context->setSetting("allow_experimental_parallel_reading_from_replicas", Field(0)); - context->setSetting("max_parallel_replicas", UInt64{0}); + context->setSetting("max_parallel_replicas", UInt64{1}); need_analyze_again = true; } @@ -945,7 +945,7 @@ bool InterpreterSelectQuery::adjustParallelReplicasAfterAnalysis() if (number_of_replicas_to_use <= 1) { context->setSetting("allow_experimental_parallel_reading_from_replicas", Field(0)); - context->setSetting("max_parallel_replicas", UInt64{0}); + context->setSetting("max_parallel_replicas", UInt64{1}); LOG_DEBUG(log, "Disabling parallel replicas because there aren't enough rows to read"); return true; } @@ -2298,7 +2298,7 @@ std::optional InterpreterSelectQuery::getTrivialCount(UInt64 max_paralle && !settings.allow_experimental_query_deduplication && !settings.empty_result_for_aggregation_by_empty_set && storage - && storage->supportsTrivialCountOptimization() + && storage->supportsTrivialCountOptimization(storage_snapshot, getContext()) && query_info.filter_asts.empty() && query_analyzer->hasAggregation() && (query_analyzer->aggregates().size() == 1) diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 1cbc9c49631..9b4534601c3 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -755,6 +755,14 @@ BlockIO InterpreterSystemQuery::execute() FailPointInjection::disableFailPoint(query.fail_point_name); break; } + case Type::WAIT_FAILPOINT: + { + getContext()->checkAccess(AccessType::SYSTEM_FAILPOINT); + LOG_TRACE(log, "waiting for failpoint {}", query.fail_point_name); + FailPointInjection::pauseFailPoint(query.fail_point_name); + LOG_TRACE(log, "finished failpoint {}", query.fail_point_name); + break; + } case Type::RESET_COVERAGE: { getContext()->checkAccess(AccessType::SYSTEM); @@ -854,7 +862,7 @@ StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica, auto & create = create_ast->as(); create.attach = true; - auto columns = InterpreterCreateQuery::getColumnsDescription(*create.columns_list->columns, system_context, true, false); + auto columns = InterpreterCreateQuery::getColumnsDescription(*create.columns_list->columns, system_context, LoadingStrictnessLevel::ATTACH); auto constraints = InterpreterCreateQuery::getConstraintsDescription(create.columns_list->constraints); auto data_path = database->getTableDataPath(create); @@ -1454,6 +1462,7 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster() case Type::STOP_THREAD_FUZZER: case Type::START_THREAD_FUZZER: case Type::ENABLE_FAILPOINT: + case Type::WAIT_FAILPOINT: case Type::DISABLE_FAILPOINT: case Type::RESET_COVERAGE: case Type::UNKNOWN: diff --git a/src/Interpreters/InterpreterUndropQuery.cpp b/src/Interpreters/InterpreterUndropQuery.cpp index 1f2ff4b4461..920df3d6aed 100644 --- a/src/Interpreters/InterpreterUndropQuery.cpp +++ b/src/Interpreters/InterpreterUndropQuery.cpp @@ -18,14 +18,16 @@ namespace ErrorCodes extern const int SUPPORT_IS_DISABLED; } -InterpreterUndropQuery::InterpreterUndropQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) +InterpreterUndropQuery::InterpreterUndropQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) + : WithMutableContext(context_) + , query_ptr(query_ptr_) { } - BlockIO InterpreterUndropQuery::execute() { getContext()->checkAccess(AccessType::UNDROP_TABLE); + auto & undrop = query_ptr->as(); if (!undrop.cluster.empty() && !maybeRemoveOnCluster(query_ptr, getContext())) { diff --git a/src/Interpreters/MetricLog.cpp b/src/Interpreters/MetricLog.cpp index 5f6db0da520..6ed29cfadcb 100644 --- a/src/Interpreters/MetricLog.cpp +++ b/src/Interpreters/MetricLog.cpp @@ -1,12 +1,13 @@ -#include -#include -#include -#include -#include #include #include #include +#include #include +#include +#include +#include +#include +#include namespace DB diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 222447ca650..35fd549559b 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -409,12 +409,13 @@ MutationsInterpreter::MutationsInterpreter( , available_columns(std::move(available_columns_)) , settings(std::move(settings_)) , select_limits(SelectQueryOptions().analyze(!settings.can_execute).ignoreLimits()) + , logger(getLogger("MutationsInterpreter(" + source.getStorage()->getStorageID().getFullTableName() + ")")) { auto new_context = Context::createCopy(context_); if (new_context->getSettingsRef().allow_experimental_analyzer) { new_context->setSetting("allow_experimental_analyzer", false); - LOG_DEBUG(&Poco::Logger::get("MutationsInterpreter"), "Will use old analyzer to prepare mutation"); + LOG_DEBUG(logger, "Will use old analyzer to prepare mutation"); } context = std::move(new_context); @@ -997,6 +998,7 @@ void MutationsInterpreter::prepare(bool dry_run) /// Always rebuild broken projections. if (source.hasBrokenProjection(projection.name)) { + LOG_DEBUG(logger, "Will rebuild broken projection {}", projection.name); materialized_projections.insert(projection.name); continue; } diff --git a/src/Interpreters/MutationsInterpreter.h b/src/Interpreters/MutationsInterpreter.h index 4c35ec34b58..2d01c7154c8 100644 --- a/src/Interpreters/MutationsInterpreter.h +++ b/src/Interpreters/MutationsInterpreter.h @@ -175,6 +175,8 @@ private: Settings settings; SelectQueryOptions select_limits; + LoggerPtr logger; + /// A sequence of mutation commands is executed as a sequence of stages. Each stage consists of several /// filters, followed by updating values of some columns. Commands can reuse expressions calculated by the /// previous commands in the same stage, but at the end of each stage intermediate columns are thrown away diff --git a/src/Interpreters/ProcessorsProfileLog.cpp b/src/Interpreters/ProcessorsProfileLog.cpp index 015b4abc712..7dec2a3163a 100644 --- a/src/Interpreters/ProcessorsProfileLog.cpp +++ b/src/Interpreters/ProcessorsProfileLog.cpp @@ -1,15 +1,16 @@ #include -#include -#include +#include #include #include #include #include +#include #include #include -#include -#include +#include +#include +#include #include #include diff --git a/src/Interpreters/TablesStatus.cpp b/src/Interpreters/TablesStatus.cpp index 911a028f813..a830ac16de6 100644 --- a/src/Interpreters/TablesStatus.cpp +++ b/src/Interpreters/TablesStatus.cpp @@ -13,22 +13,26 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -void TableStatus::write(WriteBuffer & out) const +void TableStatus::write(WriteBuffer & out, UInt64 client_protocol_revision) const { writeBinary(is_replicated, out); if (is_replicated) { writeVarUInt(absolute_delay, out); + if (client_protocol_revision >= DBMS_MIN_REVISION_WITH_TABLE_READ_ONLY_CHECK) + writeVarUInt(is_readonly, out); } } -void TableStatus::read(ReadBuffer & in) +void TableStatus::read(ReadBuffer & in, UInt64 server_protocol_revision) { absolute_delay = 0; readBinary(is_replicated, in); if (is_replicated) { readVarUInt(absolute_delay, in); + if (server_protocol_revision >= DBMS_MIN_REVISION_WITH_TABLE_READ_ONLY_CHECK) + readVarUInt(is_readonly, in); } } @@ -71,14 +75,14 @@ void TablesStatusResponse::write(WriteBuffer & out, UInt64 client_protocol_revis throw Exception(ErrorCodes::LOGICAL_ERROR, "method TablesStatusResponse::write is called for unsupported client revision"); writeVarUInt(table_states_by_id.size(), out); - for (const auto & kv: table_states_by_id) + for (const auto & kv : table_states_by_id) { const QualifiedTableName & table_name = kv.first; writeBinary(table_name.database, out); writeBinary(table_name.table, out); const TableStatus & status = kv.second; - status.write(out); + status.write(out, client_protocol_revision); } } @@ -100,7 +104,7 @@ void TablesStatusResponse::read(ReadBuffer & in, UInt64 server_protocol_revision readBinary(table_name.table, in); TableStatus status; - status.read(in); + status.read(in, server_protocol_revision); table_states_by_id.emplace(std::move(table_name), std::move(status)); } } diff --git a/src/Interpreters/TablesStatus.h b/src/Interpreters/TablesStatus.h index 2323e751fc9..ddd0a721701 100644 --- a/src/Interpreters/TablesStatus.h +++ b/src/Interpreters/TablesStatus.h @@ -28,9 +28,11 @@ struct TableStatus { bool is_replicated = false; UInt32 absolute_delay = 0; + /// Used to filter such nodes out for INSERTs + bool is_readonly = false; - void write(WriteBuffer & out) const; - void read(ReadBuffer & in); + void write(WriteBuffer & out, UInt64 client_protocol_revision) const; + void read(ReadBuffer & in, UInt64 server_protocol_revision); }; struct TablesStatusRequest diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 21889441b51..ea2f69bd2b1 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -1453,6 +1453,7 @@ void executeQuery( ASTPtr ast; BlockIO streams; OutputFormatPtr output_format; + String format_name; auto update_format_on_exception_if_needed = [&]() { @@ -1460,7 +1461,7 @@ void executeQuery( { try { - String format_name = context->getDefaultFormat(); + format_name = context->getDefaultFormat(); output_format = FormatFactory::instance().getOutputFormat(format_name, ostr, {}, context, output_format_settings); if (output_format && output_format->supportsWritingException()) { @@ -1501,7 +1502,7 @@ void executeQuery( { update_format_on_exception_if_needed(); if (output_format) - handle_exception_in_output_format(*output_format); + handle_exception_in_output_format(*output_format, format_name, context, output_format_settings); } throw; } @@ -1543,7 +1544,7 @@ void executeQuery( ); } - String format_name = ast_query_with_output && (ast_query_with_output->format != nullptr) + format_name = ast_query_with_output && (ast_query_with_output->format != nullptr) ? getIdentifierName(ast_query_with_output->format) : context->getDefaultFormat(); @@ -1609,7 +1610,7 @@ void executeQuery( { update_format_on_exception_if_needed(); if (output_format) - handle_exception_in_output_format(*output_format); + handle_exception_in_output_format(*output_format, format_name, context, output_format_settings); } throw; } diff --git a/src/Interpreters/executeQuery.h b/src/Interpreters/executeQuery.h index 0f599922668..c6b3e1fc34e 100644 --- a/src/Interpreters/executeQuery.h +++ b/src/Interpreters/executeQuery.h @@ -27,7 +27,7 @@ struct QueryResultDetails }; using SetResultDetailsFunc = std::function; -using HandleExceptionInOutputFormatFunc = std::function; +using HandleExceptionInOutputFormatFunc = std::function & format_settings)>; struct QueryFlags { diff --git a/src/Interpreters/parseColumnsListForTableFunction.cpp b/src/Interpreters/parseColumnsListForTableFunction.cpp index 30a41c090d5..27c364073ae 100644 --- a/src/Interpreters/parseColumnsListForTableFunction.cpp +++ b/src/Interpreters/parseColumnsListForTableFunction.cpp @@ -86,6 +86,10 @@ void validateDataType(const DataTypePtr & type_to_check, const DataTypeValidatio { for (size_t j = i + 1; j < variants.size(); ++j) { + /// Don't consider bool as similar to something (like number). + if (isBool(variants[i]) || isBool(variants[j])) + continue; + if (auto supertype = tryGetLeastSupertype(DataTypes{variants[i], variants[j]})) { throw Exception( @@ -121,7 +125,7 @@ ColumnsDescription parseColumnsListFromString(const std::string & structure, con if (!columns_list) throw Exception(ErrorCodes::LOGICAL_ERROR, "Could not cast AST to ASTExpressionList"); - auto columns = InterpreterCreateQuery::getColumnsDescription(*columns_list, context, false, false); + auto columns = InterpreterCreateQuery::getColumnsDescription(*columns_list, context, LoadingStrictnessLevel::CREATE); auto validation_settings = DataTypeValidationSettings(context->getSettingsRef()); for (const auto & [name, type] : columns.getAll()) validateDataType(type, validation_settings); @@ -149,7 +153,7 @@ bool tryParseColumnsListFromString(const std::string & structure, ColumnsDescrip try { - columns = InterpreterCreateQuery::getColumnsDescription(*columns_list, context, false, false); + columns = InterpreterCreateQuery::getColumnsDescription(*columns_list, context, LoadingStrictnessLevel::CREATE); auto validation_settings = DataTypeValidationSettings(context->getSettingsRef()); for (const auto & [name, type] : columns.getAll()) validateDataType(type, validation_settings); diff --git a/src/Interpreters/tests/gtest_page_cache.cpp b/src/Interpreters/tests/gtest_page_cache.cpp index 1e2688c0ca2..30fa3b921c9 100644 --- a/src/Interpreters/tests/gtest_page_cache.cpp +++ b/src/Interpreters/tests/gtest_page_cache.cpp @@ -1,4 +1,6 @@ +#include #include + #include #include diff --git a/src/Loggers/Loggers.cpp b/src/Loggers/Loggers.cpp index cc6e4691737..c5862b82f34 100644 --- a/src/Loggers/Loggers.cpp +++ b/src/Loggers/Loggers.cpp @@ -1,14 +1,17 @@ #include "Loggers.h" -#include -#include -#include #include "OwnFormattingChannel.h" #include "OwnPatternFormatter.h" #include "OwnSplitChannel.h" + +#include +#include + #include #include #include +#include +#include #ifndef WITHOUT_TEXT_LOG #include diff --git a/src/Parsers/ASTDropQuery.cpp b/src/Parsers/ASTDropQuery.cpp index ca47ceccb85..ecb9ad8169b 100644 --- a/src/Parsers/ASTDropQuery.cpp +++ b/src/Parsers/ASTDropQuery.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include @@ -47,7 +49,9 @@ void ASTDropQuery::formatQueryImpl(const FormatSettings & settings, FormatState if (temporary) settings.ostr << "TEMPORARY "; - if (!table && database) + if (has_all_tables) + settings.ostr << "ALL TABLES "; + else if (!table && !database_and_tables && database) settings.ostr << "DATABASE "; else if (is_dictionary) settings.ostr << "DICTIONARY "; @@ -64,10 +68,33 @@ void ASTDropQuery::formatQueryImpl(const FormatSettings & settings, FormatState settings.ostr << (settings.hilite ? hilite_none : ""); - if (!table && database) + if (!table && !database_and_tables && database) { database->formatImpl(settings, state, frame); } + else if (database_and_tables) + { + auto & list = database_and_tables->as(); + for (auto * it = list.children.begin(); it != list.children.end(); ++it) + { + if (it != list.children.begin()) + settings.ostr << ", "; + + auto identifier = dynamic_pointer_cast(*it); + if (!identifier) + throw Exception(ErrorCodes::SYNTAX_ERROR, "Unexpected type for list of table names."); + + if (auto db = identifier->getDatabase()) + { + db->formatImpl(settings, state, frame); + settings.ostr << '.'; + } + + auto tb = identifier->getTable(); + chassert(tb); + tb->formatImpl(settings, state, frame); + } + } else { if (database) @@ -89,4 +116,39 @@ void ASTDropQuery::formatQueryImpl(const FormatSettings & settings, FormatState settings.ostr << (settings.hilite ? hilite_keyword : "") << " SYNC" << (settings.hilite ? hilite_none : ""); } +ASTs ASTDropQuery::getRewrittenASTsOfSingleTable() +{ + ASTs res; + if (database_and_tables == nullptr) + { + res.push_back(shared_from_this()); + return res; + } + + auto & list = database_and_tables->as(); + for (const auto & child : list.children) + { + auto cloned = clone(); + auto & query = cloned->as(); + query.database_and_tables = nullptr; + query.children.clear(); + + auto database_and_table = dynamic_pointer_cast(child); + if (!database_and_table) + throw Exception(ErrorCodes::SYNTAX_ERROR, "Unexpected type for list of table names."); + + query.database = database_and_table->getDatabase(); + query.table = database_and_table->getTable(); + + if (query.database) + query.children.push_back(query.database); + + if (query.table) + query.children.push_back(query.table); + + res.push_back(cloned); + } + return res; +} + } diff --git a/src/Parsers/ASTDropQuery.h b/src/Parsers/ASTDropQuery.h index a732b354260..e0e908733e5 100644 --- a/src/Parsers/ASTDropQuery.h +++ b/src/Parsers/ASTDropQuery.h @@ -26,6 +26,9 @@ public: /// Useful if we already have a DDL lock bool no_ddl_lock{false}; + /// For `TRUNCATE ALL TABLES` query + bool has_all_tables{false}; + /// We dropping dictionary, so print correct word bool is_dictionary{false}; @@ -37,6 +40,9 @@ public: // We detach the object permanently, so it will not be reattached back during server restart. bool permanently{false}; + /// Example: Drop TABLE t1, t2, t3... + ASTPtr database_and_tables; + /** Get the text that identifies this element. */ String getID(char) const override; ASTPtr clone() const override; @@ -46,6 +52,8 @@ public: return removeOnCluster(clone(), params.default_database); } + ASTs getRewrittenASTsOfSingleTable(); + QueryKind getQueryKind() const override { return QueryKind::Drop; } protected: diff --git a/src/Parsers/ASTSystemQuery.cpp b/src/Parsers/ASTSystemQuery.cpp index 9215353e2b3..ed122b2b191 100644 --- a/src/Parsers/ASTSystemQuery.cpp +++ b/src/Parsers/ASTSystemQuery.cpp @@ -364,6 +364,7 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState & s } case Type::ENABLE_FAILPOINT: case Type::DISABLE_FAILPOINT: + case Type::WAIT_FAILPOINT: { settings.ostr << ' '; print_identifier(fail_point_name); diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h index 18a804ebc45..65c3f0eb328 100644 --- a/src/Parsers/ASTSystemQuery.h +++ b/src/Parsers/ASTSystemQuery.h @@ -87,6 +87,7 @@ public: UNFREEZE, ENABLE_FAILPOINT, DISABLE_FAILPOINT, + WAIT_FAILPOINT, SYNC_FILESYSTEM_CACHE, STOP_PULLING_REPLICATION_LOG, START_PULLING_REPLICATION_LOG, diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index f2c09e9b050..1510cc8e195 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -168,7 +168,7 @@ bool ParserColumnDeclarationList::parseImpl(Pos & pos, ASTPtr & node, Expected & bool ParserNameList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - return ParserList(std::make_unique(), std::make_unique(TokenType::Comma), false) + return ParserList(std::make_unique(true, true), std::make_unique(TokenType::Comma), false) .parse(pos, node, expected); } diff --git a/src/Parsers/ParserDropQuery.cpp b/src/Parsers/ParserDropQuery.cpp index fab9938d0b0..09f15e9649f 100644 --- a/src/Parsers/ParserDropQuery.cpp +++ b/src/Parsers/ParserDropQuery.cpp @@ -2,7 +2,7 @@ #include #include - +#include namespace DB { @@ -17,6 +17,8 @@ bool parseDropQuery(IParser::Pos & pos, ASTPtr & node, Expected & expected, cons ParserKeyword s_dictionary(Keyword::DICTIONARY); ParserKeyword s_view(Keyword::VIEW); ParserKeyword s_database(Keyword::DATABASE); + ParserKeyword s_all(Keyword::ALL); + ParserKeyword s_tables(Keyword::TABLES); ParserToken s_dot(TokenType::Dot); ParserKeyword s_if_exists(Keyword::IF_EXISTS); ParserKeyword s_if_empty(Keyword::IF_EMPTY); @@ -24,12 +26,14 @@ bool parseDropQuery(IParser::Pos & pos, ASTPtr & node, Expected & expected, cons ParserKeyword s_permanently(Keyword::PERMANENTLY); ParserKeyword s_no_delay(Keyword::NO_DELAY); ParserKeyword s_sync(Keyword::SYNC); + ParserNameList tables_p; ASTPtr database; - ASTPtr table; + ASTPtr database_and_tables; String cluster_str; bool if_exists = false; bool if_empty = false; + bool has_all_tables = false; bool temporary = false; bool is_dictionary = false; bool is_view = false; @@ -47,6 +51,16 @@ bool parseDropQuery(IParser::Pos & pos, ASTPtr & node, Expected & expected, cons if (!name_p.parse(pos, database, expected)) return false; } + else if (s_all.ignore(pos, expected) && s_tables.ignore(pos, expected) && kind == ASTDropQuery::Kind::Truncate) + { + has_all_tables = true; + + if (s_if_exists.ignore(pos, expected)) + if_exists = true; + + if (!name_p.parse(pos, database, expected)) + return false; + } else { if (s_view.ignore(pos, expected)) @@ -68,15 +82,8 @@ bool parseDropQuery(IParser::Pos & pos, ASTPtr & node, Expected & expected, cons if (s_if_empty.ignore(pos, expected)) if_empty = true; - if (!name_p.parse(pos, table, expected)) + if (!tables_p.parse(pos, database_and_tables, expected)) return false; - - if (s_dot.ignore(pos, expected)) - { - database = table; - if (!name_p.parse(pos, table, expected)) - return false; - } } /// common for tables / dictionaries / databases @@ -99,19 +106,20 @@ bool parseDropQuery(IParser::Pos & pos, ASTPtr & node, Expected & expected, cons query->kind = kind; query->if_exists = if_exists; query->if_empty = if_empty; + query->has_all_tables = has_all_tables; query->temporary = temporary; query->is_dictionary = is_dictionary; query->is_view = is_view; query->sync = sync; query->permanently = permanently; query->database = database; - query->table = table; + query->database_and_tables = database_and_tables; if (database) query->children.push_back(database); - if (table) - query->children.push_back(table); + if (database_and_tables) + query->children.push_back(database_and_tables); query->cluster = cluster_str; diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp index 81f9332c730..b660f947290 100644 --- a/src/Parsers/ParserSystemQuery.cpp +++ b/src/Parsers/ParserSystemQuery.cpp @@ -263,6 +263,7 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & } case Type::ENABLE_FAILPOINT: case Type::DISABLE_FAILPOINT: + case Type::WAIT_FAILPOINT: { ASTPtr ast; if (ParserIdentifier{}.parse(pos, ast, expected)) diff --git a/src/Parsers/tests/gtest_dictionary_parser.cpp b/src/Parsers/tests/gtest_dictionary_parser.cpp index a1ba46125a7..44205975cdc 100644 --- a/src/Parsers/tests/gtest_dictionary_parser.cpp +++ b/src/Parsers/tests/gtest_dictionary_parser.cpp @@ -301,8 +301,10 @@ TEST(ParserDictionaryDDL, ParseDropQuery) ASTDropQuery * drop1 = ast1->as(); EXPECT_TRUE(drop1->is_dictionary); - EXPECT_EQ(drop1->getDatabase(), "test"); - EXPECT_EQ(drop1->getTable(), "dict1"); + auto & database_and_tables1 = drop1->database_and_tables->as(); + auto identifier1 = dynamic_pointer_cast(database_and_tables1.children[0]); + EXPECT_EQ(identifier1->getDatabaseName(), "test"); + EXPECT_EQ(identifier1->shortName(), "dict1"); auto str1 = serializeAST(*drop1); EXPECT_EQ(input1, str1); @@ -312,8 +314,10 @@ TEST(ParserDictionaryDDL, ParseDropQuery) ASTDropQuery * drop2 = ast2->as(); EXPECT_TRUE(drop2->is_dictionary); - EXPECT_EQ(drop2->getDatabase(), ""); - EXPECT_EQ(drop2->getTable(), "dict2"); + auto & database_and_tables2 = drop2->database_and_tables->as(); + auto identifier2 = dynamic_pointer_cast(database_and_tables2.children[0]); + EXPECT_EQ(identifier2->getDatabaseName(), ""); + EXPECT_EQ(identifier2->shortName(), "dict2"); auto str2 = serializeAST(*drop2); EXPECT_EQ(input2, str2); } diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index dddab524101..e9c3795176a 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -225,7 +225,8 @@ bool applyTrivialCountIfPossible( return false; const auto & storage = table_node ? table_node->getStorage() : table_function_node->getStorage(); - if (!storage->supportsTrivialCountOptimization()) + if (!storage->supportsTrivialCountOptimization( + table_node ? table_node->getStorageSnapshot() : table_function_node->getStorageSnapshot(), query_context)) return false; auto storage_id = storage->getStorageID(); @@ -262,9 +263,6 @@ bool applyTrivialCountIfPossible( if (main_query_node.hasGroupBy() || main_query_node.hasPrewhere() || main_query_node.hasWhere()) return false; - if (storage->hasLightweightDeletedMask()) - return false; - if (settings.allow_experimental_query_deduplication || settings.empty_result_for_aggregation_by_empty_set) return false; @@ -296,7 +294,7 @@ bool applyTrivialCountIfPossible( /// The query could use trivial count if it didn't use parallel replicas, so let's disable it query_context->setSetting("allow_experimental_parallel_reading_from_replicas", Field(0)); - query_context->setSetting("max_parallel_replicas", UInt64{0}); + query_context->setSetting("max_parallel_replicas", UInt64{1}); LOG_TRACE(getLogger("Planner"), "Disabling parallel replicas to be able to use a trivial count optimization"); } @@ -779,7 +777,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres { planner_context->getMutableQueryContext()->setSetting( "allow_experimental_parallel_reading_from_replicas", Field(0)); - planner_context->getMutableQueryContext()->setSetting("max_parallel_replicas", UInt64{0}); + planner_context->getMutableQueryContext()->setSetting("max_parallel_replicas", UInt64{1}); LOG_DEBUG(getLogger("Planner"), "Disabling parallel replicas because there aren't enough rows to read"); } else if (number_of_replicas_to_use < settings.max_parallel_replicas) diff --git a/src/Processors/Formats/IRowInputFormat.cpp b/src/Processors/Formats/IRowInputFormat.cpp index 8c317a34a9d..0b6c81923db 100644 --- a/src/Processors/Formats/IRowInputFormat.cpp +++ b/src/Processors/Formats/IRowInputFormat.cpp @@ -29,6 +29,7 @@ namespace ErrorCodes extern const int CANNOT_PARSE_IPV4; extern const int CANNOT_PARSE_IPV6; extern const int UNKNOWN_ELEMENT_OF_ENUM; + extern const int CANNOT_PARSE_ESCAPE_SEQUENCE; } @@ -50,7 +51,8 @@ bool isParseError(int code) || code == ErrorCodes::CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING || code == ErrorCodes::CANNOT_PARSE_IPV4 || code == ErrorCodes::CANNOT_PARSE_IPV6 - || code == ErrorCodes::UNKNOWN_ELEMENT_OF_ENUM; + || code == ErrorCodes::UNKNOWN_ELEMENT_OF_ENUM + || code == ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE; } IRowInputFormat::IRowInputFormat(Block header, ReadBuffer & in_, Params params_) diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp index 9d6c8420069..2b40e796c5c 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp @@ -2,7 +2,6 @@ #if USE_ARROW || USE_PARQUET -// #include #include #include #include @@ -10,6 +9,7 @@ #include #include #include +#include #include #include #include diff --git a/src/Processors/Formats/Impl/JSONColumnsBlockInputFormat.cpp b/src/Processors/Formats/Impl/JSONColumnsBlockInputFormat.cpp index 3d003658e64..c1a1ef6ddf9 100644 --- a/src/Processors/Formats/Impl/JSONColumnsBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONColumnsBlockInputFormat.cpp @@ -7,7 +7,7 @@ namespace DB { -JSONColumnsReader::JSONColumnsReader(ReadBuffer & in_) : JSONColumnsReaderBase(in_) +JSONColumnsReader::JSONColumnsReader(ReadBuffer & in_, const FormatSettings & format_settings_) : JSONColumnsReaderBase(in_), format_settings(format_settings_) { } @@ -18,7 +18,7 @@ void JSONColumnsReader::readChunkStart() std::optional JSONColumnsReader::readColumnStart() { - auto name = JSONUtils::readFieldName(*in); + auto name = JSONUtils::readFieldName(*in, format_settings.json); JSONUtils::skipArrayStart(*in); return name; } @@ -38,7 +38,7 @@ void registerInputFormatJSONColumns(FormatFactory & factory) const RowInputFormatParams &, const FormatSettings & settings) { - return std::make_shared(buf, sample, settings, std::make_unique(buf)); + return std::make_shared(buf, sample, settings, std::make_unique(buf, settings)); } ); factory.markFormatSupportsSubsetOfColumns("JSONColumns"); @@ -50,7 +50,7 @@ void registerJSONColumnsSchemaReader(FormatFactory & factory) "JSONColumns", [](ReadBuffer & buf, const FormatSettings & settings) { - return std::make_shared(buf, settings, std::make_unique(buf)); + return std::make_shared(buf, settings, std::make_unique(buf, settings)); } ); factory.registerAdditionalInfoForSchemaCacheGetter("JSONColumns", [](const FormatSettings & settings) diff --git a/src/Processors/Formats/Impl/JSONColumnsBlockInputFormat.h b/src/Processors/Formats/Impl/JSONColumnsBlockInputFormat.h index eb3020036e2..3494d1b9289 100644 --- a/src/Processors/Formats/Impl/JSONColumnsBlockInputFormat.h +++ b/src/Processors/Formats/Impl/JSONColumnsBlockInputFormat.h @@ -15,11 +15,14 @@ namespace DB class JSONColumnsReader : public JSONColumnsReaderBase { public: - explicit JSONColumnsReader(ReadBuffer & in_); + explicit JSONColumnsReader(ReadBuffer & in_, const FormatSettings & format_settings_); void readChunkStart() override; std::optional readColumnStart() override; bool checkChunkEnd() override; + +protected: + const FormatSettings format_settings; }; } diff --git a/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.cpp b/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.cpp index 62d33d36206..faa4f36bbb0 100644 --- a/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.cpp +++ b/src/Processors/Formats/Impl/JSONColumnsBlockInputFormatBase.cpp @@ -134,7 +134,7 @@ Chunk JSONColumnsBlockInputFormatBase::read() { do { - skipJSONField(*in, "skip_field"); + skipJSONField(*in, "skip_field", format_settings.json); ++num_rows; } while (!reader->checkColumnEndOrSkipFieldDelimiter()); } @@ -326,7 +326,7 @@ DataTypePtr JSONColumnsSchemaReaderBase::readColumnAndGetDataType(const String & break; } - readJSONField(field, in); + readJSONField(field, in, format_settings.json); DataTypePtr field_type = tryInferDataTypeForSingleJSONField(field, format_settings, &inference_info); chooseResultColumnType(*this, column_type, field_type, nullptr, column_name, rows_read); ++rows_read; diff --git a/src/Processors/Formats/Impl/JSONColumnsWithMetadataBlockInputFormat.cpp b/src/Processors/Formats/Impl/JSONColumnsWithMetadataBlockInputFormat.cpp index 572b3b0703f..6541e5eddf4 100644 --- a/src/Processors/Formats/Impl/JSONColumnsWithMetadataBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONColumnsWithMetadataBlockInputFormat.cpp @@ -12,8 +12,8 @@ namespace ErrorCodes extern const int INCORRECT_DATA; } -JSONColumnsWithMetadataReader::JSONColumnsWithMetadataReader(ReadBuffer & in_, const Block & header_, const FormatSettings & settings) - : JSONColumnsReader(in_), header(header_), validate_types_from_metadata(settings.json.validate_types_from_metadata) +JSONColumnsWithMetadataReader::JSONColumnsWithMetadataReader(ReadBuffer & in_, const Block & header_, const FormatSettings & format_settings_) + : JSONColumnsReader(in_, format_settings_), header(header_) { } @@ -21,13 +21,13 @@ void JSONColumnsWithMetadataReader::readChunkStart() { skipBOMIfExists(*in); JSONUtils::skipObjectStart(*in); - if (validate_types_from_metadata) - JSONUtils::readMetadataAndValidateHeader(*in, header); + if (format_settings.json.validate_types_from_metadata) + JSONUtils::readMetadataAndValidateHeader(*in, header, format_settings.json); else - JSONUtils::readMetadata(*in); + JSONUtils::readMetadata(*in, format_settings.json); JSONUtils::skipComma(*in); - if (!JSONUtils::skipUntilFieldInObject(*in, "data")) + if (!JSONUtils::skipUntilFieldInObject(*in, "data", format_settings.json)) throw Exception(ErrorCodes::INCORRECT_DATA, "Expected field \"data\" with table content"); JSONUtils::skipObjectStart(*in); @@ -39,12 +39,12 @@ bool JSONColumnsWithMetadataReader::checkChunkEnd() if (!JSONUtils::checkAndSkipObjectEnd(*in)) return false; - JSONUtils::skipTheRestOfObject(*in); + JSONUtils::skipTheRestOfObject(*in, format_settings.json); assertEOF(*in); return true; } -JSONColumnsWithMetadataSchemaReader::JSONColumnsWithMetadataSchemaReader(ReadBuffer & in_) : ISchemaReader(in_) +JSONColumnsWithMetadataSchemaReader::JSONColumnsWithMetadataSchemaReader(ReadBuffer & in_, const FormatSettings & format_settings_) : ISchemaReader(in_), format_settings(format_settings_) { } @@ -52,7 +52,7 @@ NamesAndTypesList JSONColumnsWithMetadataSchemaReader::readSchema() { skipBOMIfExists(in); JSONUtils::skipObjectStart(in); - return JSONUtils::readMetadata(in); + return JSONUtils::readMetadata(in, format_settings.json); } void registerInputFormatJSONColumnsWithMetadata(FormatFactory & factory) @@ -74,9 +74,9 @@ void registerJSONColumnsWithMetadataSchemaReader(FormatFactory & factory) { factory.registerSchemaReader( "JSONColumnsWithMetadata", - [](ReadBuffer & buf, const FormatSettings &) + [](ReadBuffer & buf, const FormatSettings & format_settings) { - return std::make_shared(buf); + return std::make_shared(buf, format_settings); } ); } diff --git a/src/Processors/Formats/Impl/JSONColumnsWithMetadataBlockInputFormat.h b/src/Processors/Formats/Impl/JSONColumnsWithMetadataBlockInputFormat.h index e9db0b58223..1b3f6159bb1 100644 --- a/src/Processors/Formats/Impl/JSONColumnsWithMetadataBlockInputFormat.h +++ b/src/Processors/Formats/Impl/JSONColumnsWithMetadataBlockInputFormat.h @@ -9,22 +9,24 @@ namespace DB class JSONColumnsWithMetadataReader : public JSONColumnsReader { public: - JSONColumnsWithMetadataReader(ReadBuffer & in_, const Block & header_, const FormatSettings & settings); + JSONColumnsWithMetadataReader(ReadBuffer & in_, const Block & header_, const FormatSettings & format_settings_); void readChunkStart() override; bool checkChunkEnd() override; private: const Block header; - const bool validate_types_from_metadata; }; class JSONColumnsWithMetadataSchemaReader : public ISchemaReader { public: - explicit JSONColumnsWithMetadataSchemaReader(ReadBuffer & in_); + explicit JSONColumnsWithMetadataSchemaReader(ReadBuffer & in_, const FormatSettings & format_settings_); NamesAndTypesList readSchema() override; + +private: + const FormatSettings format_settings; }; diff --git a/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp index 6fbd9d7ad22..ee06ae91740 100644 --- a/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.cpp @@ -75,7 +75,7 @@ void JSONCompactEachRowFormatReader::skipRowBetweenDelimiter() void JSONCompactEachRowFormatReader::skipField() { skipWhitespaceIfAny(*in); - skipJSONField(*in, "skipped_field"); + skipJSONField(*in, "skipped_field", format_settings.json); } void JSONCompactEachRowFormatReader::skipHeaderRow() @@ -114,7 +114,7 @@ std::vector JSONCompactEachRowFormatReader::readHeaderRow() do { skipWhitespaceIfAny(*in); - readJSONString(field, *in); + readJSONString(field, *in, format_settings.json); fields.push_back(field); skipWhitespaceIfAny(*in); } diff --git a/src/Processors/Formats/Impl/JSONCompactRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONCompactRowInputFormat.cpp index fbb361f4b05..63066fc8220 100644 --- a/src/Processors/Formats/Impl/JSONCompactRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONCompactRowInputFormat.cpp @@ -16,7 +16,6 @@ JSONCompactRowInputFormat::JSONCompactRowInputFormat( const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings_) : RowInputFormatWithNamesAndTypes( header_, in_, params_, false, false, false, format_settings_, std::make_unique(in_, format_settings_)) - , validate_types_from_metadata(format_settings_.json.validate_types_from_metadata) { } @@ -24,9 +23,9 @@ void JSONCompactRowInputFormat::readPrefix() { skipBOMIfExists(*in); JSONUtils::skipObjectStart(*in); - if (validate_types_from_metadata) + if (format_settings.json.validate_types_from_metadata) { - auto names_and_types = JSONUtils::readMetadataAndValidateHeader(*in, getPort().getHeader()); + auto names_and_types = JSONUtils::readMetadataAndValidateHeader(*in, getPort().getHeader(), format_settings.json); Names column_names; for (const auto & [name, type] : names_and_types) column_names.push_back(name); @@ -34,12 +33,12 @@ void JSONCompactRowInputFormat::readPrefix() } else { - JSONUtils::readMetadata(*in); + JSONUtils::readMetadata(*in, format_settings.json); column_mapping->setupByHeader(getPort().getHeader()); } JSONUtils::skipComma(*in); - if (!JSONUtils::skipUntilFieldInObject(*in, "data")) + if (!JSONUtils::skipUntilFieldInObject(*in, "data", format_settings.json)) throw Exception(ErrorCodes::INCORRECT_DATA, "Expected field \"data\" with table content"); JSONUtils::skipArrayStart(*in); @@ -48,7 +47,7 @@ void JSONCompactRowInputFormat::readPrefix() void JSONCompactRowInputFormat::readSuffix() { /// Array end was skipped in JSONCompactFormatReader::checkForSuffix - JSONUtils::skipTheRestOfObject(*in); + JSONUtils::skipTheRestOfObject(*in, format_settings.json); } void JSONCompactRowInputFormat::syncAfterError() diff --git a/src/Processors/Formats/Impl/JSONCompactRowInputFormat.h b/src/Processors/Formats/Impl/JSONCompactRowInputFormat.h index 09f2f77268d..3a93e7149b0 100644 --- a/src/Processors/Formats/Impl/JSONCompactRowInputFormat.h +++ b/src/Processors/Formats/Impl/JSONCompactRowInputFormat.h @@ -20,8 +20,6 @@ private: void readPrefix() override; void readSuffix() override; - - const bool validate_types_from_metadata; }; class JSONCompactFormatReader : public JSONCompactEachRowFormatReader diff --git a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp index 6fa94356cd3..b4b40d5dcc6 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp @@ -114,7 +114,7 @@ StringRef JSONEachRowRowInputFormat::readColumnName(ReadBuffer & buf) } current_column_name.resize(nested_prefix_length); - readJSONStringInto(current_column_name, buf); + readJSONStringInto(current_column_name, buf, format_settings.json); return current_column_name; } @@ -123,7 +123,7 @@ void JSONEachRowRowInputFormat::skipUnknownField(StringRef name_ref) if (!format_settings.skip_unknown_fields) throw Exception(ErrorCodes::INCORRECT_DATA, "Unknown field found while parsing JSONEachRow format: {}", name_ref.toString()); - skipJSONField(*in, name_ref); + skipJSONField(*in, name_ref, format_settings.json); } void JSONEachRowRowInputFormat::readField(size_t index, MutableColumns & columns) diff --git a/src/Processors/Formats/Impl/JSONObjectEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONObjectEachRowRowInputFormat.cpp index ac3ef59e88b..b332dcfc6a6 100644 --- a/src/Processors/Formats/Impl/JSONObjectEachRowRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONObjectEachRowRowInputFormat.cpp @@ -46,7 +46,7 @@ void JSONObjectEachRowInputFormat::readPrefix() void JSONObjectEachRowInputFormat::readRowStart(MutableColumns & columns) { - auto object_name = JSONUtils::readFieldName(*in); + auto object_name = JSONUtils::readFieldName(*in, format_settings.json); if (field_index_for_object_name) { columns[*field_index_for_object_name]->insertData(object_name.data(), object_name.size()); @@ -57,7 +57,7 @@ void JSONObjectEachRowInputFormat::readRowStart(MutableColumns & columns) void JSONObjectEachRowInputFormat::skipRowStart() { - JSONUtils::readFieldName(*in); + JSONUtils::readFieldName(*in, format_settings.json); } bool JSONObjectEachRowInputFormat::checkEndOfData(bool is_first_row) @@ -90,7 +90,7 @@ NamesAndTypesList JSONObjectEachRowSchemaReader::readRowAndGetNamesAndDataTypes( else JSONUtils::skipComma(in); - JSONUtils::readFieldName(in); + JSONUtils::readFieldName(in, format_settings.json); return JSONUtils::readRowAndGetNamesAndDataTypesForJSONEachRow(in, format_settings, &inference_info); } diff --git a/src/Processors/Formats/Impl/JSONRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONRowInputFormat.cpp index 67652a2cb0d..615b85ab32c 100644 --- a/src/Processors/Formats/Impl/JSONRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONRowInputFormat.cpp @@ -27,9 +27,9 @@ void JSONRowInputFormat::readPrefix() /// Try to parse metadata, if failed, try to parse data as JSONEachRow format. if (JSONUtils::checkAndSkipObjectStart(*peekable_buf) - && JSONUtils::tryReadMetadata(*peekable_buf, names_and_types_from_metadata) + && JSONUtils::tryReadMetadata(*peekable_buf, names_and_types_from_metadata, format_settings.json) && JSONUtils::checkAndSkipComma(*peekable_buf) - && JSONUtils::skipUntilFieldInObject(*peekable_buf, "data") + && JSONUtils::skipUntilFieldInObject(*peekable_buf, "data", format_settings.json) && JSONUtils::checkAndSkipArrayStart(*peekable_buf)) { data_in_square_brackets = true; @@ -55,7 +55,7 @@ void JSONRowInputFormat::readSuffix() else { JSONUtils::skipArrayEnd(*peekable_buf); - JSONUtils::skipTheRestOfObject(*peekable_buf); + JSONUtils::skipTheRestOfObject(*peekable_buf, format_settings.json); } } @@ -90,7 +90,7 @@ NamesAndTypesList JSONRowSchemaReader::readSchema() PeekableReadBufferCheckpoint checkpoint(*peekable_buf); /// Try to parse metadata, if failed, try to parse data as JSONEachRow format NamesAndTypesList names_and_types; - if (JSONUtils::checkAndSkipObjectStart(*peekable_buf) && JSONUtils::tryReadMetadata(*peekable_buf, names_and_types)) + if (JSONUtils::checkAndSkipObjectStart(*peekable_buf) && JSONUtils::tryReadMetadata(*peekable_buf, names_and_types, format_settings.json)) return names_and_types; peekable_buf->rollbackToCheckpoint(true); @@ -99,7 +99,7 @@ NamesAndTypesList JSONRowSchemaReader::readSchema() else { JSONUtils::skipObjectStart(*peekable_buf); - return JSONUtils::readMetadata(*peekable_buf); + return JSONUtils::readMetadata(*peekable_buf, format_settings.json); } } diff --git a/src/Processors/IProcessor.h b/src/Processors/IProcessor.h index c6bef186877..56b4509fe00 100644 --- a/src/Processors/IProcessor.h +++ b/src/Processors/IProcessor.h @@ -369,6 +369,8 @@ public: protected: virtual void onCancel() {} + std::atomic is_cancelled{false}; + private: /// For: /// - elapsed_us @@ -378,8 +380,6 @@ private: /// - output_wait_elapsed_us friend class ExecutingGraph; - std::atomic is_cancelled{false}; - std::string processor_description; /// For processors_profile_log diff --git a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp index 28160b18269..79b5dae2d6e 100644 --- a/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/SummingSortedAlgorithm.cpp @@ -224,7 +224,7 @@ static SummingSortedAlgorithm::ColumnsDefinition defineColumns( const ColumnWithTypeAndName & column = header.safeGetByPosition(i); const auto * simple = dynamic_cast(column.type->getCustomName()); - if (column.name == BlockNumberColumn::name) + if (column.name == BlockNumberColumn::name || column.name == BlockOffsetColumn::name) { def.column_numbers_not_to_aggregate.push_back(i); continue; diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index a76bacdd97b..74f293e5682 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -191,20 +191,24 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B const size_t streams = pipeline.getNumStreams(); auto input_header = pipeline.getHeader(); - pipeline.transform([&](OutputPortRawPtrs ports) + + if (grouping_sets_size > 1) { - Processors copiers; - copiers.reserve(ports.size()); - - for (auto * port : ports) + pipeline.transform([&](OutputPortRawPtrs ports) { - auto copier = std::make_shared(input_header, grouping_sets_size); - connect(*port, copier->getInputPort()); - copiers.push_back(copier); - } + Processors copiers; + copiers.reserve(ports.size()); - return copiers; - }); + for (auto * port : ports) + { + auto copier = std::make_shared(input_header, grouping_sets_size); + connect(*port, copier->getInputPort()); + copiers.push_back(copier); + } + + return copiers; + }); + } pipeline.transform([&](OutputPortRawPtrs ports) { diff --git a/src/Processors/Transforms/AggregatingInOrderTransform.cpp b/src/Processors/Transforms/AggregatingInOrderTransform.cpp index a39a0db1311..9ffe15d0f85 100644 --- a/src/Processors/Transforms/AggregatingInOrderTransform.cpp +++ b/src/Processors/Transforms/AggregatingInOrderTransform.cpp @@ -160,7 +160,7 @@ void AggregatingInOrderTransform::consume(Chunk chunk) if (group_by_key) params->aggregator.mergeOnBlockSmall(variants, key_begin, key_end, aggregate_columns_data, key_columns_raw); else - params->aggregator.mergeOnIntervalWithoutKey(variants, key_begin, key_end, aggregate_columns_data); + params->aggregator.mergeOnIntervalWithoutKey(variants, key_begin, key_end, aggregate_columns_data, is_cancelled); } else { diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index ea5c525d5f2..b48d435720a 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -132,7 +132,7 @@ protected: return {}; } - Block block = params->aggregator.mergeAndConvertOneBucketToBlock(*data, arena, params->final, bucket_num, &shared_data->is_cancelled); + Block block = params->aggregator.mergeAndConvertOneBucketToBlock(*data, arena, params->final, bucket_num, shared_data->is_cancelled); Chunk chunk = convertToChunk(block); shared_data->is_bucket_processed[bucket_num] = true; @@ -285,7 +285,11 @@ class ConvertingAggregatedToChunksTransform : public IProcessor { public: ConvertingAggregatedToChunksTransform(AggregatingTransformParamsPtr params_, ManyAggregatedDataVariantsPtr data_, size_t num_threads_) - : IProcessor({}, {params_->getHeader()}), params(std::move(params_)), data(std::move(data_)), num_threads(num_threads_) + : IProcessor({}, {params_->getHeader()}) + , params(std::move(params_)) + , data(std::move(data_)) + , shared_data(std::make_shared()) + , num_threads(num_threads_) { } @@ -346,8 +350,7 @@ public: for (auto & input : inputs) input.close(); - if (shared_data) - shared_data->is_cancelled.store(true); + shared_data->is_cancelled.store(true, std::memory_order_seq_cst); return Status::Finished; } @@ -372,6 +375,11 @@ public: return prepareTwoLevel(); } + void onCancel() override + { + shared_data->is_cancelled.store(true, std::memory_order_seq_cst); + } + private: IProcessor::Status preparePushToOutput() { @@ -464,7 +472,7 @@ private: if (first->type == AggregatedDataVariants::Type::without_key || params->params.overflow_row) { - params->aggregator.mergeWithoutKeyDataImpl(*data); + params->aggregator.mergeWithoutKeyDataImpl(*data, shared_data->is_cancelled); auto block = params->aggregator.prepareBlockAndFillWithoutKey( *first, params->final, first->type != AggregatedDataVariants::Type::without_key); @@ -506,7 +514,7 @@ private: void createSources() { AggregatedDataVariantsPtr & first = data->at(0); - shared_data = std::make_shared(); + processors.reserve(num_threads); for (size_t thread = 0; thread < num_threads; ++thread) { @@ -684,7 +692,7 @@ void AggregatingTransform::consume(Chunk chunk) { auto block = getInputs().front().getHeader().cloneWithColumns(chunk.detachColumns()); block = materializeBlock(block); - if (!params->aggregator.mergeOnBlock(block, variants, no_more_keys)) + if (!params->aggregator.mergeOnBlock(block, variants, no_more_keys, is_cancelled)) is_consume_finished = true; } else @@ -704,7 +712,7 @@ void AggregatingTransform::initGenerate() if (variants.empty() && params->params.keys_size == 0 && !params->params.empty_result_for_aggregation_by_empty_set) { if (params->params.only_merge) - params->aggregator.mergeOnBlock(getInputs().front().getHeader(), variants, no_more_keys); + params->aggregator.mergeOnBlock(getInputs().front().getHeader(), variants, no_more_keys, is_cancelled); else params->aggregator.executeOnBlock(getInputs().front().getHeader(), variants, key_columns, aggregate_columns, no_more_keys); } diff --git a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp index a92e2253314..fc40c6894bb 100644 --- a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp @@ -363,7 +363,7 @@ void MergingAggregatedBucketTransform::transform(Chunk & chunk) res_info->chunk_num = chunks_to_merge->chunk_num; chunk.setChunkInfo(std::move(res_info)); - auto block = params->aggregator.mergeBlocks(blocks_list, params->final); + auto block = params->aggregator.mergeBlocks(blocks_list, params->final, is_cancelled); if (!required_sort_description.empty()) sortBlock(block, required_sort_description); diff --git a/src/Processors/Transforms/MergingAggregatedTransform.cpp b/src/Processors/Transforms/MergingAggregatedTransform.cpp index e4955d06859..ad723da7527 100644 --- a/src/Processors/Transforms/MergingAggregatedTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedTransform.cpp @@ -39,11 +39,10 @@ void MergingAggregatedTransform::consume(Chunk chunk) if (const auto * agg_info = typeid_cast(info.get())) { /** If the remote servers used a two-level aggregation method, - * then blocks will contain information about the number of the bucket. - * Then the calculations can be parallelized by buckets. - * We decompose the blocks to the bucket numbers indicated in them. - */ - + * then blocks will contain information about the number of the bucket. + * Then the calculations can be parallelized by buckets. + * We decompose the blocks to the bucket numbers indicated in them. + */ auto block = getInputPort().getHeader().cloneWithColumns(chunk.getColumns()); block.info.is_overflows = agg_info->is_overflows; block.info.bucket_num = agg_info->bucket_num; @@ -73,7 +72,7 @@ Chunk MergingAggregatedTransform::generate() next_block = blocks.begin(); /// TODO: this operation can be made async. Add async for IAccumulatingTransform. - params->aggregator.mergeBlocks(std::move(bucket_to_blocks), data_variants, max_threads); + params->aggregator.mergeBlocks(std::move(bucket_to_blocks), data_variants, max_threads, is_cancelled); blocks = params->aggregator.convertToBlocks(data_variants, params->final, max_threads); next_block = blocks.begin(); } diff --git a/src/Processors/Transforms/RollupTransform.cpp b/src/Processors/Transforms/RollupTransform.cpp index a5d67fb2f15..bc5a7aeb7bf 100644 --- a/src/Processors/Transforms/RollupTransform.cpp +++ b/src/Processors/Transforms/RollupTransform.cpp @@ -57,7 +57,8 @@ Chunk GroupByModifierTransform::merge(Chunks && chunks, bool is_input, bool fina for (auto & chunk : chunks) blocks.emplace_back(header.cloneWithColumns(chunk.detachColumns())); - auto current_block = is_input ? params->aggregator.mergeBlocks(blocks, final) : output_aggregator->mergeBlocks(blocks, final); + auto & aggregator = is_input ? params->aggregator : *output_aggregator; + auto current_block = aggregator.mergeBlocks(blocks, final, is_cancelled); auto num_rows = current_block.rows(); return Chunk(current_block.getColumns(), num_rows); } diff --git a/src/Server/CertificateReloader.cpp b/src/Server/CertificateReloader.cpp index c974f450c9a..98d7a362bd7 100644 --- a/src/Server/CertificateReloader.cpp +++ b/src/Server/CertificateReloader.cpp @@ -1,7 +1,8 @@ -#include "CertificateReloader.h" +#include #if USE_SSL +#include #include #include #include @@ -22,7 +23,6 @@ int callSetCertificate(SSL * ssl, [[maybe_unused]] void * arg) } - /// This is callback for OpenSSL. It will be called on every connection to obtain a certificate and private key. int CertificateReloader::setCertificate(SSL * ssl) { @@ -30,17 +30,37 @@ int CertificateReloader::setCertificate(SSL * ssl) if (!current) return -1; - SSL_use_certificate(ssl, const_cast(current->cert.certificate())); - SSL_use_PrivateKey(ssl, const_cast(static_cast(current->key))); + if (current->certs_chain.empty()) + return -1; - int err = SSL_check_private_key(ssl); - if (err != 1) + if (auto err = SSL_clear_chain_certs(ssl); err != 1) { - std::string msg = Poco::Net::Utility::getLastError(); - LOG_ERROR(log, "Unusable key-pair {}", msg); + LOG_ERROR(log, "Clear certificates {}", Poco::Net::Utility::getLastError()); + return -1; + } + if (auto err = SSL_use_certificate(ssl, const_cast(current->certs_chain[0].certificate())); err != 1) + { + LOG_ERROR(log, "Use certificate {}", Poco::Net::Utility::getLastError()); + return -1; + } + for (auto cert = current->certs_chain.begin() + 1; cert != current->certs_chain.end(); cert++) + { + if (auto err = SSL_add1_chain_cert(ssl, const_cast(cert->certificate())); err != 1) + { + LOG_ERROR(log, "Add certificate to chain {}", Poco::Net::Utility::getLastError()); + return -1; + } + } + if (auto err = SSL_use_PrivateKey(ssl, const_cast(static_cast(current->key))); err != 1) + { + LOG_ERROR(log, "Use private key {}", Poco::Net::Utility::getLastError()); + return -1; + } + if (auto err = SSL_check_private_key(ssl); err != 1) + { + LOG_ERROR(log, "Unusable key-pair {}", Poco::Net::Utility::getLastError()); return -1; } - return 1; } @@ -100,7 +120,7 @@ void CertificateReloader::tryLoad(const Poco::Util::AbstractConfiguration & conf CertificateReloader::Data::Data(std::string cert_path, std::string key_path, std::string pass_phrase) - : cert(cert_path), key(/* public key */ "", /* private key */ key_path, pass_phrase) + : certs_chain(Poco::Crypto::X509Certificate::readPEM(cert_path)), key(/* public key */ "", /* private key */ key_path, pass_phrase) { } diff --git a/src/Server/CertificateReloader.h b/src/Server/CertificateReloader.h index 028914e682f..5ab799037d5 100644 --- a/src/Server/CertificateReloader.h +++ b/src/Server/CertificateReloader.h @@ -70,7 +70,7 @@ private: struct Data { - Poco::Crypto::X509Certificate cert; + Poco::Crypto::X509Certificate::List certs_chain; Poco::Crypto::EVPPKey key; Data(std::string cert_path, std::string key_path, std::string pass_phrase); diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index fd9be999276..07a89b60a8f 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -873,18 +873,33 @@ void HTTPHandler::processQuery( response.add("X-ClickHouse-Timezone", *details.timezone); }; - auto handle_exception_in_output_format = [&](IOutputFormat & output_format) + auto handle_exception_in_output_format = [&](IOutputFormat & current_output_format, const String & format_name, const ContextPtr & context_, const std::optional & format_settings) { - if (settings.http_write_exception_in_output_format && output_format.supportsWritingException()) + if (settings.http_write_exception_in_output_format && current_output_format.supportsWritingException()) { - bool with_stacktrace = (params.getParsed("stacktrace", false) && server.config().getBool("enable_http_stacktrace", true)); - - ExecutionStatus status = ExecutionStatus::fromCurrentException("", with_stacktrace); - formatExceptionForClient(status.code, request, response, used_output); - - output_format.setException(getCurrentExceptionMessage(false)); - output_format.finalize(); - used_output.exception_is_written = true; + /// If wait_end_of_query=true in case of an exception all data written to output format during query execution will be + /// ignored, so we cannot write exception message in current output format as it will be also ignored. + /// Instead, we create exception_writer function that will write exception in required format + /// and will use it later in trySendExceptionToClient when all buffers will be prepared. + if (buffer_until_eof) + { + auto header = current_output_format.getPort(IOutputFormat::PortKind::Main).getHeader(); + used_output.exception_writer = [format_name, header, context_, format_settings](WriteBuffer & buf, const String & message) + { + auto output_format = FormatFactory::instance().getOutputFormat(format_name, buf, header, context_, format_settings); + output_format->setException(message); + output_format->finalize(); + }; + } + else + { + bool with_stacktrace = (params.getParsed("stacktrace", false) && server.config().getBool("enable_http_stacktrace", true)); + ExecutionStatus status = ExecutionStatus::fromCurrentException("", with_stacktrace); + formatExceptionForClient(status.code, request, response, used_output); + current_output_format.setException(status.message); + current_output_format.finalize(); + used_output.exception_is_written = true; + } } }; @@ -948,8 +963,16 @@ try used_output.out_holder->position() = used_output.out_holder->buffer().begin(); } - writeString(s, *used_output.out_maybe_compressed); - writeChar('\n', *used_output.out_maybe_compressed); + /// We might have special formatter for exception message. + if (used_output.exception_writer) + { + used_output.exception_writer(*used_output.out_maybe_compressed, s); + } + else + { + writeString(s, *used_output.out_maybe_compressed); + writeChar('\n', *used_output.out_maybe_compressed); + } } used_output.out_maybe_compressed->next(); diff --git a/src/Server/HTTPHandler.h b/src/Server/HTTPHandler.h index 39cef0e7112..ae4cf034276 100644 --- a/src/Server/HTTPHandler.h +++ b/src/Server/HTTPHandler.h @@ -75,6 +75,7 @@ private: bool finalized = false; bool exception_is_written = false; + std::function exception_writer; inline bool hasDelayed() const { diff --git a/src/Server/InterserverIOHTTPHandler.cpp b/src/Server/InterserverIOHTTPHandler.cpp index 28045380cd7..0d79aaa227b 100644 --- a/src/Server/InterserverIOHTTPHandler.cpp +++ b/src/Server/InterserverIOHTTPHandler.cpp @@ -8,8 +8,9 @@ #include #include #include -#include +#include #include +#include #include #include diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp index 32490627214..6456f6d24ff 100644 --- a/src/Server/MySQLHandler.cpp +++ b/src/Server/MySQLHandler.cpp @@ -21,8 +21,10 @@ #include #include #include +#include #include #include +#include #include #include #include diff --git a/src/Server/PostgreSQLHandler.cpp b/src/Server/PostgreSQLHandler.cpp index 83e06628185..473d681ddb2 100644 --- a/src/Server/PostgreSQLHandler.cpp +++ b/src/Server/PostgreSQLHandler.cpp @@ -1,17 +1,19 @@ +#include "PostgreSQLHandler.h" #include -#include #include +#include #include #include #include -#include "PostgreSQLHandler.h" #include #include -#include -#include #include #include +#include +#include #include +#include +#include #if USE_SSL # include diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 26d2390f1df..5c08c697434 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1124,6 +1124,7 @@ void TCPHandler::processTablesStatusRequest() { status.is_replicated = true; status.absolute_delay = static_cast(replicated_table->getAbsoluteDelay()); + status.is_readonly = replicated_table->isTableReadOnly(); } else status.is_replicated = false; diff --git a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp index 8d95e49de57..2d052255ac5 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp @@ -203,11 +203,10 @@ void DistributedAsyncInsertBatch::readText(ReadBuffer & in) void DistributedAsyncInsertBatch::sendBatch(const SettingsChanges & settings_changes) { + IConnectionPool::Entry connection; std::unique_ptr remote; bool compression_expected = false; - IConnectionPool::Entry connection; - /// Since the batch is sent as a whole (in case of failure, the whole batch /// will be repeated), we need to mark the whole batch as failed in case of /// error). @@ -232,7 +231,8 @@ void DistributedAsyncInsertBatch::sendBatch(const SettingsChanges & settings_cha insert_settings.applyChanges(settings_changes); auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(insert_settings); - connection = parent.pool->get(timeouts); + auto result = parent.pool->getManyCheckedForInsert(timeouts, insert_settings, PoolMode::GET_ONE, parent.storage.remote_storage.getQualifiedName()); + connection = std::move(result.front().entry); compression_expected = connection->getCompression() == Protocol::Compression::Enable; LOG_DEBUG(parent.log, "Sending a batch of {} files to {} ({} rows, {} bytes).", @@ -289,7 +289,8 @@ void DistributedAsyncInsertBatch::sendSeparateFiles(const SettingsChanges & sett parent.storage.getContext()->getOpenTelemetrySpanLog()); auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(insert_settings); - auto connection = parent.pool->get(timeouts); + auto result = parent.pool->getManyCheckedForInsert(timeouts, insert_settings, PoolMode::GET_ONE, parent.storage.remote_storage.getQualifiedName()); + auto connection = std::move(result.front().entry); bool compression_expected = connection->getCompression() == Protocol::Compression::Enable; RemoteInserter remote(*connection, timeouts, diff --git a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp index 7fed076713d..1ee77611191 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp @@ -101,7 +101,7 @@ DistributedAsyncInsertDirectoryQueue::DistributedAsyncInsertDirectoryQueue( StorageDistributed & storage_, const DiskPtr & disk_, const std::string & relative_path_, - ConnectionPoolPtr pool_, + ConnectionPoolWithFailoverPtr pool_, ActionBlocker & monitor_blocker_, BackgroundSchedulePool & bg_pool) : storage(storage_) @@ -237,7 +237,7 @@ void DistributedAsyncInsertDirectoryQueue::run() } -ConnectionPoolPtr DistributedAsyncInsertDirectoryQueue::createPool(const Cluster::Addresses & addresses, const StorageDistributed & storage) +ConnectionPoolWithFailoverPtr DistributedAsyncInsertDirectoryQueue::createPool(const Cluster::Addresses & addresses, const StorageDistributed & storage) { const auto pool_factory = [&storage] (const Cluster::Address & address) -> ConnectionPoolPtr { @@ -284,7 +284,7 @@ ConnectionPoolPtr DistributedAsyncInsertDirectoryQueue::createPool(const Cluster auto pools = createPoolsForAddresses(addresses, pool_factory, storage.log); const auto settings = storage.getContext()->getSettings(); - return pools.size() == 1 ? pools.front() : std::make_shared(pools, + return std::make_shared(std::move(pools), settings.load_balancing, settings.distributed_replica_error_half_life.totalSeconds(), settings.distributed_replica_error_cap); @@ -412,7 +412,9 @@ void DistributedAsyncInsertDirectoryQueue::processFile(std::string & file_path, insert_settings.applyChanges(settings_changes); auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(insert_settings); - auto connection = pool->get(timeouts, insert_settings); + auto result = pool->getManyCheckedForInsert(timeouts, insert_settings, PoolMode::GET_ONE, storage.remote_storage.getQualifiedName()); + auto connection = std::move(result.front().entry); + LOG_DEBUG(log, "Sending `{}` to {} ({} rows, {} bytes)", file_path, connection->getDescription(), diff --git a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.h b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.h index 4d6afe31d61..ba4d264f967 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.h +++ b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.h @@ -50,13 +50,13 @@ public: StorageDistributed & storage_, const DiskPtr & disk_, const std::string & relative_path_, - ConnectionPoolPtr pool_, + ConnectionPoolWithFailoverPtr pool_, ActionBlocker & monitor_blocker_, BackgroundSchedulePool & bg_pool); ~DistributedAsyncInsertDirectoryQueue(); - static ConnectionPoolPtr createPool(const Cluster::Addresses & addresses, const StorageDistributed & storage); + static ConnectionPoolWithFailoverPtr createPool(const Cluster::Addresses & addresses, const StorageDistributed & storage); void updatePath(const std::string & new_relative_path); @@ -111,7 +111,7 @@ private: std::string getLoggerName() const; StorageDistributed & storage; - const ConnectionPoolPtr pool; + const ConnectionPoolWithFailoverPtr pool; DiskPtr disk; std::string relative_path; diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index 1efa98d0c13..ddbcc6d473f 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -112,19 +112,17 @@ DistributedSink::DistributedSink( const ClusterPtr & cluster_, bool insert_sync_, UInt64 insert_timeout_, - StorageID main_table_, const Names & columns_to_send_) : SinkToStorage(metadata_snapshot_->getSampleBlock()) , context(Context::createCopy(context_)) , storage(storage_) , metadata_snapshot(metadata_snapshot_) - , query_ast(createInsertToRemoteTableQuery(main_table_.database_name, main_table_.table_name, columns_to_send_)) + , query_ast(createInsertToRemoteTableQuery(storage.remote_storage.database_name, storage.remote_storage.table_name, columns_to_send_)) , query_string(queryToString(query_ast)) , cluster(cluster_) , insert_sync(insert_sync_) , allow_materialized(context->getSettingsRef().insert_allow_materialized_columns) , insert_timeout(insert_timeout_) - , main_table(main_table_) , columns_to_send(columns_to_send_.begin(), columns_to_send_.end()) , log(getLogger("DistributedSink")) { @@ -372,10 +370,9 @@ DistributedSink::runWritingJob(JobReplica & job, const Block & current_block, si throw Exception(ErrorCodes::LOGICAL_ERROR, "There are several writing job for an automatically replicated shard"); /// TODO: it make sense to rewrite skip_unavailable_shards and max_parallel_replicas here - auto results = shard_info.pool->getManyChecked(timeouts, settings, PoolMode::GET_ONE, main_table.getQualifiedName()); - if (results.empty() || results.front().entry.isNull()) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected exactly one connection for shard {}", toString(job.shard_index)); - + /// NOTE: INSERT will also take into account max_replica_delay_for_distributed_queries + /// (anyway fallback_to_stale_replicas_for_distributed_queries=true by default) + auto results = shard_info.pool->getManyCheckedForInsert(timeouts, settings, PoolMode::GET_ONE, storage.remote_storage.getQualifiedName()); job.connection_entry = std::move(results.front().entry); } else diff --git a/src/Storages/Distributed/DistributedSink.h b/src/Storages/Distributed/DistributedSink.h index 654c1db354f..7a9e89c9e94 100644 --- a/src/Storages/Distributed/DistributedSink.h +++ b/src/Storages/Distributed/DistributedSink.h @@ -46,7 +46,6 @@ public: const ClusterPtr & cluster_, bool insert_sync_, UInt64 insert_timeout_, - StorageID main_table_, const Names & columns_to_send_); String getName() const override { return "DistributedSink"; } @@ -108,7 +107,6 @@ private: /// Sync-related stuff UInt64 insert_timeout; // in seconds - StorageID main_table; NameSet columns_to_send; Stopwatch watch; Stopwatch watch_current_block; diff --git a/src/Storages/FileLog/DirectoryWatcherBase.h b/src/Storages/FileLog/DirectoryWatcherBase.h index 0dfb58fbc5c..3bf93415b8f 100644 --- a/src/Storages/FileLog/DirectoryWatcherBase.h +++ b/src/Storages/FileLog/DirectoryWatcherBase.h @@ -2,6 +2,7 @@ #include #include +#include #include #include diff --git a/src/Storages/HDFS/StorageHDFS.h b/src/Storages/HDFS/StorageHDFS.h index b14bb7f997b..b8faa27d678 100644 --- a/src/Storages/HDFS/StorageHDFS.h +++ b/src/Storages/HDFS/StorageHDFS.h @@ -92,7 +92,7 @@ public: static SchemaCache & getSchemaCache(const ContextPtr & ctx); - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } protected: friend class HDFSSource; diff --git a/src/Storages/HDFS/StorageHDFSCluster.h b/src/Storages/HDFS/StorageHDFSCluster.h index 26ebc8601ee..0b5c6242aa9 100644 --- a/src/Storages/HDFS/StorageHDFSCluster.h +++ b/src/Storages/HDFS/StorageHDFSCluster.h @@ -36,7 +36,7 @@ public: bool supportsSubcolumns() const override { return true; } - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } private: void updateQueryToSendIfNeeded(ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const ContextPtr & context) override; diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 1108eafc6b6..87a04c3fcc6 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -265,7 +265,10 @@ public: /// Return true if the trivial count query could be optimized without reading the data at all /// in totalRows() or totalRowsByPartitionPredicate() methods or with optimized reading in read() method. - virtual bool supportsTrivialCountOptimization() const { return false; } + virtual bool supportsTrivialCountOptimization(const StorageSnapshotPtr & /*storage_snapshot*/, ContextPtr /*query_context*/) const + { + return false; + } private: StorageID storage_id; diff --git a/src/Storages/Kafka/StorageKafka.h b/src/Storages/Kafka/StorageKafka.h index 829e23faf77..6479902f0f2 100644 --- a/src/Storages/Kafka/StorageKafka.h +++ b/src/Storages/Kafka/StorageKafka.h @@ -19,8 +19,9 @@ namespace DB { -class StorageSystemKafkaConsumers; class ReadFromStorageKafka; +class StorageSystemKafkaConsumers; +class ThreadStatus; struct StorageKafkaInterceptors; diff --git a/src/Storages/MaterializedView/RefreshSet.cpp b/src/Storages/MaterializedView/RefreshSet.cpp index ac83d3d99c2..0125b958a41 100644 --- a/src/Storages/MaterializedView/RefreshSet.cpp +++ b/src/Storages/MaterializedView/RefreshSet.cpp @@ -73,7 +73,7 @@ void RefreshSet::Handle::reset() RefreshSet::RefreshSet() = default; -RefreshSet::Handle RefreshSet::emplace(StorageID id, const std::vector & dependencies, RefreshTaskHolder task) +void RefreshSet::emplace(StorageID id, const std::vector & dependencies, RefreshTaskHolder task) { std::lock_guard guard(mutex); auto [it, is_inserted] = tasks.emplace(id, task); @@ -81,7 +81,7 @@ RefreshSet::Handle RefreshSet::emplace(StorageID id, const std::vectorsetRefreshSetHandleUnlock(Handle(this, id, dependencies)); } void RefreshSet::addDependenciesLocked(const StorageID & id, const std::vector & dependencies) diff --git a/src/Storages/MaterializedView/RefreshSet.h b/src/Storages/MaterializedView/RefreshSet.h index 63e022b6d4d..eff445023a6 100644 --- a/src/Storages/MaterializedView/RefreshSet.h +++ b/src/Storages/MaterializedView/RefreshSet.h @@ -1,9 +1,9 @@ #pragma once +#include #include #include #include - #include namespace DB @@ -82,7 +82,7 @@ public: RefreshSet(); - Handle emplace(StorageID id, const std::vector & dependencies, RefreshTaskHolder task); + void emplace(StorageID id, const std::vector & dependencies, RefreshTaskHolder task); RefreshTaskHolder getTask(const StorageID & id) const; diff --git a/src/Storages/MaterializedView/RefreshTask.cpp b/src/Storages/MaterializedView/RefreshTask.cpp index daf7bd65784..bc8cb0ce69a 100644 --- a/src/Storages/MaterializedView/RefreshTask.cpp +++ b/src/Storages/MaterializedView/RefreshTask.cpp @@ -50,7 +50,7 @@ RefreshTaskHolder RefreshTask::create( for (auto && dependency : strategy.dependencies->children) deps.emplace_back(dependency->as()); - task->set_handle = context->getRefreshSet().emplace(view.getStorageID(), deps, task); + context->getRefreshSet().emplace(view.getStorageID(), deps, task); return task; } @@ -509,4 +509,9 @@ std::chrono::system_clock::time_point RefreshTask::currentTime() const return std::chrono::system_clock::time_point(std::chrono::seconds(fake)); } +void RefreshTask::setRefreshSetHandleUnlock(RefreshSet::Handle && set_handle_) +{ + set_handle = std::move(set_handle_); +} + } diff --git a/src/Storages/MaterializedView/RefreshTask.h b/src/Storages/MaterializedView/RefreshTask.h index 78599f4f4b4..1f050a97cd9 100644 --- a/src/Storages/MaterializedView/RefreshTask.h +++ b/src/Storages/MaterializedView/RefreshTask.h @@ -61,6 +61,9 @@ public: /// For tests void setFakeTime(std::optional t); + /// RefreshSet will set handle for refresh tasks, to avoid race condition. + void setRefreshSetHandleUnlock(RefreshSet::Handle && set_handle_); + private: LoggerPtr log = nullptr; std::weak_ptr view_to_refresh; diff --git a/src/Storages/MergeTree/BackgroundJobsAssignee.h b/src/Storages/MergeTree/BackgroundJobsAssignee.h index 65fefce0917..9369ebe9135 100644 --- a/src/Storages/MergeTree/BackgroundJobsAssignee.h +++ b/src/Storages/MergeTree/BackgroundJobsAssignee.h @@ -1,7 +1,9 @@ #pragma once -#include #include +#include +#include + #include diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index a26e2b725be..570175f6614 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -805,8 +805,8 @@ void IMergeTreeDataPart::loadProjections(bool require_columns_checksums, bool ch throw; auto message = getCurrentExceptionMessage(true); - LOG_ERROR(&Poco::Logger::get("IMergeTreeDataPart"), - "Cannot load projection {}, will consider it broken. Reason: {}", projection.name, message); + LOG_WARNING(storage.log, "Cannot load projection {}, " + "will consider it broken. Reason: {}", projection.name, message); has_broken_projection = true; part->setBrokenReason(message, getCurrentExceptionCode()); diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp index 4936f1d33c6..cf6b64aac85 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -95,7 +95,7 @@ void IMergeTreeReader::fillVirtualColumns(Columns & columns, size_t rows) const it->name, it->type->getName(), virtual_column->type->getName()); } - if (it->name == "_part_offset") + if (MergeTreeRangeReader::virtuals_to_fill.contains(it->name)) throw Exception(ErrorCodes::LOGICAL_ERROR, "Virtual column {} must be filled by range reader", it->name); Field field; diff --git a/src/Storages/MergeTree/MarkRange.cpp b/src/Storages/MergeTree/MarkRange.cpp index bd8546f04cc..c6e98b4e5a1 100644 --- a/src/Storages/MergeTree/MarkRange.cpp +++ b/src/Storages/MergeTree/MarkRange.cpp @@ -81,6 +81,11 @@ size_t MarkRanges::getNumberOfMarks() const return result; } +bool MarkRanges::isOneRangeForWholePart(size_t num_marks_in_part) const +{ + return size() == 1 && front().begin == 0 && front().end == num_marks_in_part; +} + void MarkRanges::serialize(WriteBuffer & out) const { writeBinaryLittleEndian(this->size(), out); diff --git a/src/Storages/MergeTree/MarkRange.h b/src/Storages/MergeTree/MarkRange.h index 1d9d0a1e27e..f36d5d89825 100644 --- a/src/Storages/MergeTree/MarkRange.h +++ b/src/Storages/MergeTree/MarkRange.h @@ -36,6 +36,7 @@ struct MarkRanges : public std::deque using std::deque::deque; /// NOLINT(modernize-type-traits) size_t getNumberOfMarks() const; + bool isOneRangeForWholePart(size_t num_marks_in_part) const; void serialize(WriteBuffer & out) const; String describe() const; diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 1776d1da27c..34e17e40a74 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -224,13 +224,11 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() ctx->need_remove_expired_values = false; ctx->force_ttl = false; - if (supportsBlockNumberColumn(global_ctx) && !global_ctx->storage_columns.contains(BlockNumberColumn::name)) - { - global_ctx->storage_columns.emplace_back(NameAndTypePair{BlockNumberColumn::name,BlockNumberColumn::type}); - global_ctx->all_column_names.emplace_back(BlockNumberColumn::name); - global_ctx->gathering_columns.emplace_back(NameAndTypePair{BlockNumberColumn::name,BlockNumberColumn::type}); - global_ctx->gathering_column_names.emplace_back(BlockNumberColumn::name); - } + if (enabledBlockNumberColumn(global_ctx)) + addGatheringColumn(global_ctx, BlockNumberColumn::name, BlockNumberColumn::type); + + if (enabledBlockOffsetColumn(global_ctx)) + addGatheringColumn(global_ctx, BlockOffsetColumn::name, BlockOffsetColumn::type); SerializationInfo::Settings info_settings = { @@ -402,6 +400,17 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() return false; } +void MergeTask::addGatheringColumn(GlobalRuntimeContextPtr global_ctx, const String & name, const DataTypePtr & type) +{ + if (global_ctx->storage_columns.contains(name)) + return; + + global_ctx->storage_columns.emplace_back(name, type); + global_ctx->all_column_names.emplace_back(name); + global_ctx->gathering_columns.emplace_back(name, type); + global_ctx->gathering_column_names.emplace_back(name); +} + MergeTask::StageRuntimeContextPtr MergeTask::ExecuteAndFinalizeHorizontalPart::getContextForNextStage() { diff --git a/src/Storages/MergeTree/MergeTask.h b/src/Storages/MergeTree/MergeTask.h index 1f50e55f8a0..f6268886b14 100644 --- a/src/Storages/MergeTree/MergeTask.h +++ b/src/Storages/MergeTree/MergeTask.h @@ -404,12 +404,17 @@ private: Stages::const_iterator stages_iterator = stages.begin(); - /// Check for persisting block number column - static bool supportsBlockNumberColumn(GlobalRuntimeContextPtr global_ctx) + static bool enabledBlockNumberColumn(GlobalRuntimeContextPtr global_ctx) { - return global_ctx->data->getSettings()->allow_experimental_block_number_column && global_ctx->metadata_snapshot->getGroupByTTLs().empty(); + return global_ctx->data->getSettings()->enable_block_number_column && global_ctx->metadata_snapshot->getGroupByTTLs().empty(); } + static bool enabledBlockOffsetColumn(GlobalRuntimeContextPtr global_ctx) + { + return global_ctx->data->getSettings()->enable_block_offset_column && global_ctx->metadata_snapshot->getGroupByTTLs().empty(); + } + + static void addGatheringColumn(GlobalRuntimeContextPtr global_ctx, const String & name, const DataTypePtr & type); }; /// FIXME diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 67f5e7a53e8..570387a7046 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -278,17 +278,17 @@ MergeTreeReadTaskColumns getReadTaskColumns( .withVirtuals() .withSubcolumns(with_subcolumns); - static const NameSet columns_to_read_at_first_step = {"_part_offset"}; - NameSet columns_from_previous_steps; auto add_step = [&](const PrewhereExprStep & step) { Names step_column_names; + /// Virtual columns that are filled by RangeReader + /// must be read in the first step before any filtering. if (columns_from_previous_steps.empty()) { for (const auto & required_column : required_columns) - if (columns_to_read_at_first_step.contains(required_column)) + if (MergeTreeRangeReader::virtuals_to_fill.contains(required_column)) step_column_names.push_back(required_column); } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 08a2ff89e7b..8faed72b198 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -440,6 +440,7 @@ VirtualColumnsDescription MergeTreeData::createVirtuals(const StorageInMemoryMet desc.addEphemeral("_partition_id", std::make_shared(std::make_shared()), "Name of partition"); desc.addEphemeral("_sample_factor", std::make_shared(), "Sample factor (from the query)"); desc.addEphemeral("_part_offset", std::make_shared(), "Number of row in the part"); + desc.addEphemeral("_part_data_version", std::make_shared(), "Data version of part (either min block number or mutation version)"); if (metadata.hasPartitionKey()) { @@ -449,6 +450,7 @@ VirtualColumnsDescription MergeTreeData::createVirtuals(const StorageInMemoryMet desc.addPersistent(RowExistsColumn::name, RowExistsColumn::type, nullptr, "Persisted mask created by lightweight delete that show whether row exists or is deleted"); desc.addPersistent(BlockNumberColumn::name, BlockNumberColumn::type, BlockNumberColumn::codec, "Persisted original number of block that was assigned at insert"); + desc.addPersistent(BlockOffsetColumn::name, BlockOffsetColumn::type, BlockOffsetColumn::codec, "Persisted original number of row in block that was assigned at insert"); return desc; } @@ -1026,7 +1028,7 @@ void MergeTreeData::MergingParams::check(const StorageInMemoryMetadata & metadat /// TODO Checks for Graphite mode. } -const Names MergeTreeData::virtuals_useful_for_filter = {"_part", "_partition_id", "_part_uuid", "_partition_value"}; +const Names MergeTreeData::virtuals_useful_for_filter = {"_part", "_partition_id", "_part_uuid", "_partition_value", "_part_data_version"}; Block MergeTreeData::getHeaderWithVirtualsForFilter() const { @@ -8219,6 +8221,11 @@ void MergeTreeData::updateObjectColumns(const DataPartPtr & part, const DataPart DB::updateObjectColumns(object_columns, columns, part->getColumns()); } +bool MergeTreeData::supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const +{ + return !hasLightweightDeletedMask(); +} + StorageSnapshotPtr MergeTreeData::getStorageSnapshot(const StorageMetadataPtr & metadata_snapshot, ContextPtr query_context) const { auto snapshot_data = std::make_unique(); diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 046376be474..0d56b902f1a 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -440,7 +440,7 @@ public: bool areAsynchronousInsertsEnabled() const override { return getSettings()->async_insert; } - bool supportsTrivialCountOptimization() const override { return !hasLightweightDeletedMask(); } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override; /// Snapshot for MergeTree contains the current set of data parts /// at the moment of the start of query. diff --git a/src/Storages/MergeTree/MergeTreeIndexReader.cpp b/src/Storages/MergeTree/MergeTreeIndexReader.cpp index 6012994b46d..e7ae1fc5c13 100644 --- a/src/Storages/MergeTree/MergeTreeIndexReader.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexReader.cpp @@ -31,6 +31,8 @@ std::unique_ptr makeIndexReader( load_marks_threadpool, /*num_columns_in_mark=*/ 1); + marks_loader->startAsyncLoad(); + return std::make_unique( part->getDataPartStoragePtr(), index->getFileName(), extension, marks_count, @@ -65,6 +67,7 @@ MergeTreeIndexReader::MergeTreeIndexReader( mark_cache, uncompressed_cache, std::move(settings)); + version = index_format.version; stream->adjustRightMark(getLastMark(all_mark_ranges_)); diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index a7a635f3b72..6798f97e494 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -64,6 +64,10 @@ MergeTreeMarksLoader::MergeTreeMarksLoader( , read_settings(read_settings_) , num_columns_in_mark(num_columns_in_mark_) , load_marks_threadpool(load_marks_threadpool_) +{ +} + +void MergeTreeMarksLoader::startAsyncLoad() { if (load_marks_threadpool) future = loadMarksAsync(); @@ -102,6 +106,8 @@ MergeTreeMarksGetterPtr MergeTreeMarksLoader::loadMarks() MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() { + LOG_TEST(getLogger("MergeTreeMarksLoader"), "Loading marks from path {}", mrk_path); + /// Memory for marks must not be accounted as memory usage for query, because they are stored in shared cache. MemoryTrackerBlockerInThread temporarily_disable_memory_tracker; @@ -218,7 +224,9 @@ MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksSync() } } else + { loaded_marks = loadMarksImpl(); + } if (!loaded_marks) { diff --git a/src/Storages/MergeTree/MergeTreeMarksLoader.h b/src/Storages/MergeTree/MergeTreeMarksLoader.h index 73dd462f2fa..2aa4474e1c5 100644 --- a/src/Storages/MergeTree/MergeTreeMarksLoader.h +++ b/src/Storages/MergeTree/MergeTreeMarksLoader.h @@ -50,6 +50,7 @@ public: ~MergeTreeMarksLoader(); + void startAsyncLoad(); MergeTreeMarksGetterPtr loadMarks(); size_t getNumColumns() const { return num_columns_in_mark; } diff --git a/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/src/Storages/MergeTree/MergeTreeRangeReader.cpp index 6932762f58b..eb757e1d8c7 100644 --- a/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -871,6 +872,8 @@ size_t MergeTreeRangeReader::currentMark() const return stream.currentMark(); } +const NameSet MergeTreeRangeReader::virtuals_to_fill = {"_part_offset", "_block_offset"}; + size_t MergeTreeRangeReader::Stream::numPendingRows() const { size_t rows_between_marks = index_granularity->getRowsCountInRange(current_mark, last_mark); @@ -1143,17 +1146,37 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::startReadingChain(size_t if (!result.rows_per_granule.empty()) result.adjustLastGranule(); - if (read_sample_block.has("_part_offset")) - { - size_t pos = read_sample_block.getPositionByName("_part_offset"); - chassert(pos < result.columns.size()); - chassert(result.columns[pos] == nullptr); - result.columns[pos] = createPartOffsetColumn(result, leading_begin_part_offset, leading_end_part_offset); - } - + fillVirtualColumns(result, leading_begin_part_offset, leading_end_part_offset); return result; } +void MergeTreeRangeReader::fillVirtualColumns(ReadResult & result, UInt64 leading_begin_part_offset, UInt64 leading_end_part_offset) +{ + ColumnPtr part_offset_column; + + auto add_offset_column = [&](const auto & column_name) + { + size_t pos = read_sample_block.getPositionByName(column_name); + chassert(pos < result.columns.size()); + + /// Column may be persisted in part. + if (result.columns[pos]) + return; + + if (!part_offset_column) + part_offset_column = createPartOffsetColumn(result, leading_begin_part_offset, leading_end_part_offset); + + result.columns[pos] = part_offset_column; + }; + + if (read_sample_block.has("_part_offset")) + add_offset_column("_part_offset"); + + /// Column _block_offset is the same as _part_offset if it's not persisted in part. + if (read_sample_block.has(BlockOffsetColumn::name)) + add_offset_column(BlockOffsetColumn::name); +} + ColumnPtr MergeTreeRangeReader::createPartOffsetColumn(ReadResult & result, UInt64 leading_begin_part_offset, UInt64 leading_end_part_offset) { size_t num_rows = result.numReadRows(); diff --git a/src/Storages/MergeTree/MergeTreeRangeReader.h b/src/Storages/MergeTree/MergeTreeRangeReader.h index 688a6b0922b..b282ada6038 100644 --- a/src/Storages/MergeTree/MergeTreeRangeReader.h +++ b/src/Storages/MergeTree/MergeTreeRangeReader.h @@ -115,6 +115,9 @@ public: bool isCurrentRangeFinished() const; bool isInitialized() const { return is_initialized; } + /// Names of virtual columns that are filled in RangeReader. + static const NameSet virtuals_to_fill; + private: /// Accumulates sequential read() requests to perform a large read instead of multiple small reads class DelayedStream @@ -308,6 +311,8 @@ private: ReadResult startReadingChain(size_t max_rows, MarkRanges & ranges); Columns continueReadingChain(const ReadResult & result, size_t & num_rows); void executePrewhereActionsAndFilterColumns(ReadResult & result) const; + + void fillVirtualColumns(ReadResult & result, UInt64 leading_begin_part_offset, UInt64 leading_end_part_offset); ColumnPtr createPartOffsetColumn(ReadResult & result, UInt64 leading_begin_part_offset, UInt64 leading_end_part_offset); IMergeTreeReader * merge_tree_reader = nullptr; diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index a22bff6b8d2..8810491b62e 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -48,6 +48,7 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( , profile_callback(profile_callback_) , clock_type(clock_type_) { + marks_loader->startAsyncLoad(); } void MergeTreeReaderCompact::fillColumnPositions() diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/src/Storages/MergeTree/MergeTreeReaderStream.cpp index 40a16176c69..15ef02440cb 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -13,6 +13,7 @@ namespace ErrorCodes { extern const int ARGUMENT_OUT_OF_BOUND; extern const int CANNOT_READ_ALL_DATA; + extern const int LOGICAL_ERROR; } MergeTreeReaderStream::MergeTreeReaderStream( @@ -41,14 +42,17 @@ MergeTreeReaderStream::MergeTreeReaderStream( { } +void MergeTreeReaderStream::loadMarks() +{ + if (!marks_getter) + marks_getter = marks_loader->loadMarks(); +} + void MergeTreeReaderStream::init() { if (initialized) return; - initialized = true; - marks_getter = marks_loader->loadMarks(); - /// Compute the size of the buffer. auto [max_mark_range_bytes, sum_mark_range_bytes] = estimateMarkRangeBytes(all_mark_ranges); @@ -110,11 +114,15 @@ void MergeTreeReaderStream::init() data_buffer = non_cached_buffer.get(); compressed_data_buffer = non_cached_buffer.get(); } + + initialized = true; } void MergeTreeReaderStream::seekToMarkAndColumn(size_t row_index, size_t column_position) { init(); + loadMarks(); + const auto & mark = marks_getter->getMark(row_index, column_position); try @@ -193,7 +201,7 @@ CompressedReadBufferBase * MergeTreeReaderStream::getCompressedDataBuffer() return compressed_data_buffer; } -size_t MergeTreeReaderStreamSingleColumn::getRightOffset(size_t right_mark) const +size_t MergeTreeReaderStreamSingleColumn::getRightOffset(size_t right_mark) { /// NOTE: if we are reading the whole file, then right_mark == marks_count /// and we will use max_read_buffer_size for buffer size, thus avoiding the need to load marks. @@ -202,7 +210,8 @@ size_t MergeTreeReaderStreamSingleColumn::getRightOffset(size_t right_mark) cons if (marks_count == 0) return 0; - assert(right_mark <= marks_count); + chassert(right_mark <= marks_count); + loadMarks(); if (right_mark == 0) return marks_getter->getMark(right_mark, 0).offset_in_compressed_file; @@ -281,9 +290,9 @@ size_t MergeTreeReaderStreamSingleColumn::getRightOffset(size_t right_mark) cons return file_size; } -std::pair MergeTreeReaderStreamSingleColumn::estimateMarkRangeBytes(const MarkRanges & mark_ranges) const +std::pair MergeTreeReaderStreamSingleColumn::estimateMarkRangeBytes(const MarkRanges & mark_ranges) { - assert(marks_getter != nullptr); + loadMarks(); size_t max_range_bytes = 0; size_t sum_range_bytes = 0; @@ -302,7 +311,34 @@ std::pair MergeTreeReaderStreamSingleColumn::estimateMarkRangeBy return {max_range_bytes, sum_range_bytes}; } -size_t MergeTreeReaderStreamMultipleColumns::getRightOffsetOneColumn(size_t right_mark_non_included, size_t column_position) const +size_t MergeTreeReaderStreamSingleColumnWholePart::getRightOffset(size_t right_mark) +{ + if (right_mark != marks_count) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Expected one right mark: {}, got: {}", + marks_count, right_mark); + } + return file_size; +} + +std::pair MergeTreeReaderStreamSingleColumnWholePart::estimateMarkRangeBytes(const MarkRanges & mark_ranges) +{ + if (!mark_ranges.isOneRangeForWholePart(marks_count)) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Expected one mark range that covers the whole part, got: {}", + mark_ranges.describe()); + } + return {file_size, file_size}; +} + +void MergeTreeReaderStreamSingleColumnWholePart::seekToMark(size_t) +{ + throw Exception(ErrorCodes::LOGICAL_ERROR, "MergeTreeReaderStreamSingleColumnWholePart cannot seek to marks"); +} + +size_t MergeTreeReaderStreamMultipleColumns::getRightOffsetOneColumn(size_t right_mark_non_included, size_t column_position) { /// NOTE: if we are reading the whole file, then right_mark == marks_count /// and we will use max_read_buffer_size for buffer size, thus avoiding the need to load marks. @@ -311,7 +347,8 @@ size_t MergeTreeReaderStreamMultipleColumns::getRightOffsetOneColumn(size_t righ if (marks_count == 0) return 0; - assert(right_mark_non_included <= marks_count); + chassert(right_mark_non_included <= marks_count); + loadMarks(); if (right_mark_non_included == 0) return marks_getter->getMark(right_mark_non_included, column_position).offset_in_compressed_file; @@ -347,9 +384,9 @@ size_t MergeTreeReaderStreamMultipleColumns::getRightOffsetOneColumn(size_t righ } std::pair -MergeTreeReaderStreamMultipleColumns::estimateMarkRangeBytesOneColumn(const MarkRanges & mark_ranges, size_t column_position) const +MergeTreeReaderStreamMultipleColumns::estimateMarkRangeBytesOneColumn(const MarkRanges & mark_ranges, size_t column_position) { - assert(marks_getter != nullptr); + loadMarks(); /// As a maximal range we return the maximal size of a whole stripe. size_t max_range_bytes = 0; @@ -386,8 +423,9 @@ MergeTreeReaderStreamMultipleColumns::estimateMarkRangeBytesOneColumn(const Mark return {max_range_bytes, sum_range_bytes}; } -MarkInCompressedFile MergeTreeReaderStreamMultipleColumns::getStartOfNextStripeMark(size_t row_index, size_t column_position) const +MarkInCompressedFile MergeTreeReaderStreamMultipleColumns::getStartOfNextStripeMark(size_t row_index, size_t column_position) { + loadMarks(); const auto & current_mark = marks_getter->getMark(row_index, column_position); if (marks_getter->getNumColumns() == 1) @@ -434,27 +472,27 @@ MarkInCompressedFile MergeTreeReaderStreamMultipleColumns::getStartOfNextStripeM return marks_getter->getMark(mark_index + 1, column_position + 1); } -size_t MergeTreeReaderStreamOneOfMultipleColumns::getRightOffset(size_t right_mark_non_included) const +size_t MergeTreeReaderStreamOneOfMultipleColumns::getRightOffset(size_t right_mark_non_included) { return getRightOffsetOneColumn(right_mark_non_included, column_position); } -std::pair MergeTreeReaderStreamOneOfMultipleColumns::estimateMarkRangeBytes(const MarkRanges & mark_ranges) const +std::pair MergeTreeReaderStreamOneOfMultipleColumns::estimateMarkRangeBytes(const MarkRanges & mark_ranges) { return estimateMarkRangeBytesOneColumn(mark_ranges, column_position); } -size_t MergeTreeReaderStreamAllOfMultipleColumns::getRightOffset(size_t right_mark_non_included) const +size_t MergeTreeReaderStreamAllOfMultipleColumns::getRightOffset(size_t right_mark_non_included) { return getRightOffsetOneColumn(right_mark_non_included, marks_loader->getNumColumns() - 1); } -std::pair MergeTreeReaderStreamAllOfMultipleColumns::estimateMarkRangeBytes(const MarkRanges & mark_ranges) const +std::pair MergeTreeReaderStreamAllOfMultipleColumns::estimateMarkRangeBytes(const MarkRanges & mark_ranges) { size_t max_range_bytes = 0; size_t sum_range_bytes = 0; - for (size_t i = 0; i < marks_getter->getNumColumns(); ++i) + for (size_t i = 0; i < marks_loader->getNumColumns(); ++i) { auto [current_max, current_sum] = estimateMarkRangeBytesOneColumn(mark_ranges, i); diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.h b/src/Storages/MergeTree/MergeTreeReaderStream.h index f3ca6953ceb..05341cd8acc 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.h +++ b/src/Storages/MergeTree/MergeTreeReaderStream.h @@ -40,6 +40,7 @@ public: /// Seeks to exact mark in file. void seekToMarkAndColumn(size_t row_index, size_t column_position); + /// Seeks to the start of the file. void seekToStart(); /** @@ -53,11 +54,11 @@ public: private: /// Returns offset in file up to which it's needed to read file to read all rows up to @right_mark mark. - virtual size_t getRightOffset(size_t right_mark) const = 0; + virtual size_t getRightOffset(size_t right_mark) = 0; /// Returns estimated max amount of bytes to read among mark ranges (which is used as size for read buffer) /// and total amount of bytes to read in all mark ranges. - virtual std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) const = 0; + virtual std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) = 0; const ReadBufferFromFileBase::ProfileCallback profile_callback; const clockid_t clock_type; @@ -80,6 +81,7 @@ private: protected: void init(); + void loadMarks(); const MergeTreeReaderSettings settings; const size_t marks_count; @@ -100,11 +102,25 @@ public: { } - size_t getRightOffset(size_t right_mark_non_included) const override; - std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) const override; + size_t getRightOffset(size_t right_mark_non_included) override; + std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) override; void seekToMark(size_t row_index) override { seekToMarkAndColumn(row_index, 0); } }; +class MergeTreeReaderStreamSingleColumnWholePart : public MergeTreeReaderStream +{ +public: + template + explicit MergeTreeReaderStreamSingleColumnWholePart(Args &&... args) + : MergeTreeReaderStream{std::forward(args)...} + { + } + + size_t getRightOffset(size_t right_mark_non_included) override; + std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) override; + void seekToMark(size_t row_index) override; +}; + /// Base class for reading from file that contains multiple columns. /// It is used to read from compact parts. /// See more details about data layout in MergeTreeDataPartCompact.h. @@ -118,9 +134,9 @@ public: } protected: - size_t getRightOffsetOneColumn(size_t right_mark_non_included, size_t column_position) const; - std::pair estimateMarkRangeBytesOneColumn(const MarkRanges & mark_ranges, size_t column_position) const; - MarkInCompressedFile getStartOfNextStripeMark(size_t row_index, size_t column_position) const; + size_t getRightOffsetOneColumn(size_t right_mark_non_included, size_t column_position); + std::pair estimateMarkRangeBytesOneColumn(const MarkRanges & mark_ranges, size_t column_position); + MarkInCompressedFile getStartOfNextStripeMark(size_t row_index, size_t column_position); }; /// Class for reading a single column from file that contains multiple columns @@ -135,8 +151,8 @@ public: { } - size_t getRightOffset(size_t right_mark_non_included) const override; - std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) const override; + size_t getRightOffset(size_t right_mark_non_included) override; + std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) override; void seekToMark(size_t row_index) override { seekToMarkAndColumn(row_index, column_position); } private: @@ -154,8 +170,8 @@ public: { } - size_t getRightOffset(size_t right_mark_non_included) const override; - std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) const override; + size_t getRightOffset(size_t right_mark_non_included) override; + std::pair estimateMarkRangeBytes(const MarkRanges & mark_ranges) override; void seekToMark(size_t row_index) override { seekToMarkAndColumn(row_index, 0); } }; diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 394a22835f1..d398668d5c8 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -43,6 +43,7 @@ MergeTreeReaderWide::MergeTreeReaderWide( mark_ranges_, settings_, avg_value_size_hints_) + , read_whole_part(all_mark_ranges.isOneRangeForWholePart(data_part_info_for_read->getMarksCount())) { try { @@ -227,12 +228,13 @@ void MergeTreeReaderWide::addStreams( auto context = data_part_info_for_read->getContext(); auto * load_marks_threadpool = settings.read_settings.load_marks_asynchronously ? &context->getLoadMarksThreadpool() : nullptr; + size_t num_marks_in_part = data_part_info_for_read->getMarksCount(); auto marks_loader = std::make_shared( data_part_info_for_read, mark_cache, data_part_info_for_read->getIndexGranularityInfo().getMarksFilePath(*stream_name), - data_part_info_for_read->getMarksCount(), + num_marks_in_part, data_part_info_for_read->getIndexGranularityInfo(), settings.save_marks_in_cache, settings.read_settings, @@ -243,11 +245,24 @@ void MergeTreeReaderWide::addStreams( auto stream_settings = settings; stream_settings.is_low_cardinality_dictionary = substream_path.size() > 1 && substream_path[substream_path.size() - 2].type == ISerialization::Substream::Type::DictionaryKeys; - streams.emplace(*stream_name, std::make_unique( - data_part_info_for_read->getDataPartStorage(), *stream_name, DATA_FILE_EXTENSION, - data_part_info_for_read->getMarksCount(), all_mark_ranges, stream_settings, - uncompressed_cache, data_part_info_for_read->getFileSizeOrZero(*stream_name + DATA_FILE_EXTENSION), - std::move(marks_loader), profile_callback, clock_type)); + auto create_stream = [&]() + { + return std::make_unique( + data_part_info_for_read->getDataPartStorage(), *stream_name, DATA_FILE_EXTENSION, + num_marks_in_part, all_mark_ranges, stream_settings, + uncompressed_cache, data_part_info_for_read->getFileSizeOrZero(*stream_name + DATA_FILE_EXTENSION), + std::move(marks_loader), profile_callback, clock_type); + }; + + if (read_whole_part) + { + streams.emplace(*stream_name, create_stream.operator()()); + } + else + { + marks_loader->startAsyncLoad(); + streams.emplace(*stream_name, create_stream.operator()()); + } }; serialization->enumerateStreams(callback); @@ -325,7 +340,8 @@ void MergeTreeReaderWide::prefetchForColumn( if (stream_name && !prefetched_streams.contains(*stream_name)) { - bool seek_to_mark = !continue_reading; + bool seek_to_mark = !continue_reading && !read_whole_part; + if (ReadBuffer * buf = getStream(false, substream_path, data_part_info_for_read->getChecksums(), streams, name_and_type, from_mark, seek_to_mark, current_task_last_mark, cache)) { buf->prefetch(priority); @@ -349,7 +365,7 @@ void MergeTreeReaderWide::readData( deserialize_settings.getter = [&](const ISerialization::SubstreamPath & substream_path) { - bool seek_to_mark = !was_prefetched && !continue_reading; + bool seek_to_mark = !was_prefetched && !continue_reading && !read_whole_part; return getStream( /* seek_to_start = */false, substream_path, diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.h b/src/Storages/MergeTree/MergeTreeReaderWide.h index a9a5526dd65..7ffe565d262 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.h +++ b/src/Storages/MergeTree/MergeTreeReaderWide.h @@ -73,6 +73,7 @@ private: std::unordered_map caches; std::unordered_set prefetched_streams; ssize_t prefetched_from_mark = -1; + bool read_whole_part = false; }; } diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.h b/src/Storages/MergeTree/MergeTreeSelectProcessor.h index 01bb3851e04..6b663e0fd36 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.h @@ -66,15 +66,6 @@ public: void addPartLevelToChunk(bool add_part_level_) { add_part_level = add_part_level_; } private: - /// This struct allow to return block with no columns but with non-zero number of rows similar to Chunk - struct BlockAndProgress - { - Block block; - size_t row_count = 0; - size_t num_read_rows = 0; - size_t num_read_bytes = 0; - }; - /// Sets up range readers corresponding to data readers void initializeRangeReaders(); diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp index 81eb166b300..47661a3ff93 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp @@ -184,18 +184,19 @@ MergeTreeSequentialSource::MergeTreeSequentialSource( storage_snapshot, *mark_ranges, /*virtual_fields=*/ {}, - /*uncompressed_cache=*/{}, + /*uncompressed_cache=*/ {}, mark_cache.get(), alter_conversions, reader_settings, - {}, - {}); + /*avg_value_size_hints=*/ {}, + /*profile_callback=*/ {}); } static void fillBlockNumberColumns( Columns & res_columns, const NamesAndTypesList & columns_list, UInt64 block_number, + UInt64 block_offset, UInt64 num_rows) { chassert(res_columns.size() == columns_list.size()); @@ -210,6 +211,16 @@ static void fillBlockNumberColumns( { res_columns[i] = BlockNumberColumn::type->createColumnConst(num_rows, block_number)->convertToFullColumnIfConst(); } + else if (it->name == BlockOffsetColumn::name) + { + auto column = BlockOffsetColumn::type->createColumn(); + auto & block_offset_data = assert_cast(*column).getData(); + + block_offset_data.resize(num_rows); + std::iota(block_offset_data.begin(), block_offset_data.end(), block_offset); + + res_columns[i] = std::move(column); + } } } @@ -219,6 +230,7 @@ try const auto & header = getPort().getHeader(); /// Part level is useful for next step for merging non-merge tree table bool add_part_level = storage.merging_params.mode != MergeTreeData::MergingParams::Ordinary; + size_t num_marks_in_part = data_part->getMarksCount(); if (!isCancelled() && current_row < data_part->rows_count) { @@ -227,11 +239,11 @@ try const auto & sample = reader->getColumns(); Columns columns(sample.size()); - size_t rows_read = reader->readRows(current_mark, data_part->getMarksCount(), continue_reading, rows_to_read, columns); + size_t rows_read = reader->readRows(current_mark, num_marks_in_part, continue_reading, rows_to_read, columns); if (rows_read) { - fillBlockNumberColumns(columns, sample, data_part->info.min_block, rows_read); + fillBlockNumberColumns(columns, sample, data_part->info.min_block, current_row, rows_read); reader->fillVirtualColumns(columns, rows_read); current_row += rows_read; diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 9c67a86997b..a00508fd1c1 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -186,6 +186,8 @@ struct Settings; M(Bool, disable_freeze_partition_for_zero_copy_replication, true, "Disable FREEZE PARTITION query for zero copy replication.", 0) \ M(Bool, disable_detach_partition_for_zero_copy_replication, true, "Disable DETACH PARTITION query for zero copy replication.", 0) \ M(Bool, disable_fetch_partition_for_zero_copy_replication, true, "Disable FETCH PARTITION query for zero copy replication.", 0) \ + M(Bool, enable_block_number_column, false, "Enable persisting column _block_number for each row.", 0) ALIAS(allow_experimental_block_number_column) \ + M(Bool, enable_block_offset_column, false, "Enable persisting column _block_offset for each row.", 0) \ \ /** Experimental/work in progress feature. Unsafe for production. */ \ M(UInt64, part_moves_between_shards_enable, 0, "Experimental/Incomplete feature to move parts between shards. Does not take into account sharding expressions.", 0) \ @@ -195,7 +197,6 @@ struct Settings; M(Bool, remote_fs_zero_copy_path_compatible_mode, false, "Run zero-copy in compatible mode during conversion process.", 0) \ M(Bool, cache_populated_by_fetch, false, "Only available in ClickHouse Cloud", 0) \ M(Bool, force_read_through_cache_for_merges, false, "Force read-through filesystem cache for merges", 0) \ - M(Bool, allow_experimental_block_number_column, false, "Enable persisting column _block_number for each row.", 0) \ M(Bool, allow_experimental_replacing_merge_with_cleanup, false, "Allow experimental CLEANUP merges for ReplacingMergeTree with is_deleted column.", 0) \ \ /** Compress marks and primary key. */ \ diff --git a/src/Storages/MergeTree/MergeTreeVirtualColumns.cpp b/src/Storages/MergeTree/MergeTreeVirtualColumns.cpp index b87dccc2b18..821724a3cfb 100644 --- a/src/Storages/MergeTree/MergeTreeVirtualColumns.cpp +++ b/src/Storages/MergeTree/MergeTreeVirtualColumns.cpp @@ -26,6 +26,10 @@ const String BlockNumberColumn::name = "_block_number"; const DataTypePtr BlockNumberColumn::type = std::make_shared(); const ASTPtr BlockNumberColumn::codec = getCompressionCodecDeltaLZ4(); +const String BlockOffsetColumn::name = "_block_offset"; +const DataTypePtr BlockOffsetColumn::type = std::make_shared(); +const ASTPtr BlockOffsetColumn::codec = getCompressionCodecDeltaLZ4(); + Field getFieldForConstVirtualColumn(const String & column_name, const IMergeTreeDataPart & part) { if (column_name == RowExistsColumn::name) @@ -43,6 +47,9 @@ Field getFieldForConstVirtualColumn(const String & column_name, const IMergeTree if (column_name == "_partition_id") return part.info.partition_id; + if (column_name == "_part_data_version") + return part.info.getDataVersion(); + if (column_name == "_partition_value") return Tuple(part.partition.value.begin(), part.partition.value.end()); diff --git a/src/Storages/MergeTree/MergeTreeVirtualColumns.h b/src/Storages/MergeTree/MergeTreeVirtualColumns.h index 24721bf1ad1..cd9fe544ed8 100644 --- a/src/Storages/MergeTree/MergeTreeVirtualColumns.h +++ b/src/Storages/MergeTree/MergeTreeVirtualColumns.h @@ -21,6 +21,13 @@ struct BlockNumberColumn static const ASTPtr codec; }; +struct BlockOffsetColumn +{ + static const String name; + static const DataTypePtr type; + static const ASTPtr codec; +}; + Field getFieldForConstVirtualColumn(const String & column_name, const IMergeTreeDataPart & part); } diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index f67e9484598..90e1cb0606e 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -342,21 +342,23 @@ getColumnsForNewDataPart( } } - if (!storage_columns_set.contains(RowExistsColumn::name)) - { - if (deleted_mask_updated || (part_columns.has(RowExistsColumn::name) && !has_delete_command)) - { - storage_columns.emplace_back(RowExistsColumn::name, RowExistsColumn::type); - storage_columns_set.insert(RowExistsColumn::name); - } - } + auto persistent_virtuals = source_part->storage.getVirtualsPtr()->getNamesAndTypesList(VirtualsKind::Persistent); - if (!storage_columns_set.contains(BlockNumberColumn::name)) + for (const auto & [name, type] : persistent_virtuals) { - if (source_part->tryGetSerialization(BlockNumberColumn::name) != nullptr) + if (storage_columns_set.contains(name)) + continue; + + bool need_column = false; + if (name == RowExistsColumn::name) + need_column = deleted_mask_updated || (part_columns.has(name) && !has_delete_command); + else + need_column = part_columns.has(name); + + if (need_column) { - storage_columns.push_back({BlockNumberColumn::name, BlockNumberColumn::type}); - storage_columns_set.insert(BlockNumberColumn::name); + storage_columns.emplace_back(name, type); + storage_columns_set.insert(name); } } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index bc0b4f73a31..181f54688f9 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -386,12 +386,12 @@ ReplicatedCheckResult ReplicatedMergeTreePartCheckThread::checkPartImpl(const St throw; PreformattedMessage message; - if (is_broken_projection) + if (is_broken_projection && throw_on_broken_projection) { WriteBufferFromOwnString wb; message = PreformattedMessage::create( "Part {} has a broken projections. It will be ignored. Broken projections info: {}", - part_name, getCurrentExceptionMessage(false)); + part_name, getCurrentExceptionMessage(true)); LOG_DEBUG(log, message); result.action = ReplicatedCheckResult::DoNothing; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index b79418da791..35f355d1d9b 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,11 @@ namespace ErrorCodes extern const int REPLICA_IS_ALREADY_ACTIVE; } +namespace FailPoints +{ + extern const char finish_clean_quorum_failed_parts[]; +}; + /// Used to check whether it's us who set node `is_active`, or not. static String generateActiveNodeIdentifier() { @@ -241,6 +247,7 @@ void ReplicatedMergeTreeRestartingThread::removeFailedQuorumParts() storage.queue.removeFailedQuorumPart(part->info); } } + FailPointInjection::disableFailPoint(FailPoints::finish_clean_quorum_failed_parts); } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index b43d47bf5f4..7fcf6b971bb 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -30,6 +30,7 @@ namespace FailPoints extern const char replicated_merge_tree_commit_zk_fail_after_op[]; extern const char replicated_merge_tree_insert_quorum_fail_0[]; extern const char replicated_merge_tree_commit_zk_fail_when_recovering_from_hw_fault[]; + extern const char replicated_merge_tree_insert_retry_pause[]; } namespace ErrorCodes @@ -940,14 +941,27 @@ std::pair, bool> ReplicatedMergeTreeSinkImpl:: }); bool node_exists = false; + bool quorum_fail_exists = false; /// The loop will be executed at least once new_retry_controller.retryLoop([&] { fiu_do_on(FailPoints::replicated_merge_tree_commit_zk_fail_when_recovering_from_hw_fault, { zookeeper->forceFailureBeforeOperation(); }); + FailPointInjection::pauseFailPoint(FailPoints::replicated_merge_tree_insert_retry_pause); zookeeper->setKeeper(storage.getZooKeeper()); node_exists = zookeeper->exists(fs::path(storage.replica_path) / "parts" / part->name); + if (isQuorumEnabled()) + quorum_fail_exists = zookeeper->exists(fs::path(storage.zookeeper_path) / "quorum" / "failed_parts" / part->name); }); + /// if it has quorum fail node, the restarting thread will clean the garbage. + if (quorum_fail_exists) + { + LOG_INFO(log, "Part {} fails to commit and will not retry or clean garbage. Restarting Thread will do everything.", part->name); + transaction.clear(); + /// `quorum/failed_parts/part_name` exists because table is read only for a while, So we return table is read only. + throw Exception(ErrorCodes::TABLE_IS_READ_ONLY, "Table is in readonly mode due to shutdown: replica_path={}", storage.replica_path); + } + if (node_exists) { LOG_DEBUG(log, "Insert of part {} recovered from keeper successfully. It will be committed", part->name); diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp index 0a1057916d6..d64568e0c3e 100644 --- a/src/Storages/MergeTree/checkDataPart.cpp +++ b/src/Storages/MergeTree/checkDataPart.cpp @@ -285,11 +285,6 @@ static IMergeTreeDataPart::Checksums checkDataPart( return {}; auto projection_file = name + ".proj"; - if (!throw_on_broken_projection && projection->is_broken) - { - projections_on_disk.erase(projection_file); - checksums_txt.remove(projection_file); - } IMergeTreeDataPart::Checksums projection_checksums; try @@ -306,13 +301,19 @@ static IMergeTreeDataPart::Checksums checkDataPart( if (isRetryableException(std::current_exception())) throw; + is_broken_projection = true; + projections_on_disk.erase(projection_file); + checksums_txt.remove(projection_file); + + const auto exception_message = getCurrentExceptionMessage(true); + if (!projection->is_broken) { - LOG_TEST(log, "Marking projection {} as broken ({})", name, projection_file); - projection->setBrokenReason(getCurrentExceptionMessage(false), getCurrentExceptionCode()); + LOG_WARNING(log, "Marking projection {} as broken ({}). Reason: {}", + name, projection_file, exception_message); + projection->setBrokenReason(exception_message, getCurrentExceptionCode()); } - is_broken_projection = true; if (throw_on_broken_projection) { if (!broken_projections_message.empty()) @@ -320,12 +321,10 @@ static IMergeTreeDataPart::Checksums checkDataPart( broken_projections_message += fmt::format( "Part {} has a broken projection {} (error: {})", - data_part->name, name, getCurrentExceptionMessage(false)); - continue; + data_part->name, name, exception_message); } - projections_on_disk.erase(projection_file); - checksums_txt.remove(projection_file); + continue; } checksums_data.files[projection_file] = IMergeTreeDataPart::Checksums::Checksum( diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index e89547952d0..9a3c17923d8 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -588,7 +588,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) FunctionNameNormalizer().visit(partition_key.get()); auto primary_key_asts = metadata.primary_key.expression_list_ast->children; metadata.minmax_count_projection.emplace(ProjectionDescription::getMinMaxCountProjection( - args.columns, partition_key, minmax_columns, primary_key_asts, context)); + columns, partition_key, minmax_columns, primary_key_asts, context)); if (args.storage_def->sample_by) metadata.sampling_key = KeyDescription::getKeyFromAST(args.storage_def->sample_by->ptr(), metadata.columns, context); @@ -697,7 +697,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) FunctionNameNormalizer().visit(partition_key.get()); auto primary_key_asts = metadata.primary_key.expression_list_ast->children; metadata.minmax_count_projection.emplace(ProjectionDescription::getMinMaxCountProjection( - args.columns, partition_key, minmax_columns, primary_key_asts, context)); + columns, partition_key, minmax_columns, primary_key_asts, context)); const auto * ast = engine_args[arg_num]->as(); if (ast && ast->value.getType() == Field::Types::UInt64) diff --git a/src/Storages/NATS/NATSHandler.cpp b/src/Storages/NATS/NATSHandler.cpp index 03f1fc1a495..f0554a7f1f0 100644 --- a/src/Storages/NATS/NATSHandler.cpp +++ b/src/Storages/NATS/NATSHandler.cpp @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/src/Storages/RocksDB/StorageEmbeddedRocksDB.h b/src/Storages/RocksDB/StorageEmbeddedRocksDB.h index 8525108735b..230464a161f 100644 --- a/src/Storages/RocksDB/StorageEmbeddedRocksDB.h +++ b/src/Storages/RocksDB/StorageEmbeddedRocksDB.h @@ -93,7 +93,7 @@ public: bool supportsDelete() const override { return true; } /// To turn on the optimization optimize_trivial_approximate_count_query=1 should be set for a query. - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } std::optional totalRows(const Settings & settings) const override; diff --git a/src/Storages/S3Queue/S3QueueFilesMetadata.cpp b/src/Storages/S3Queue/S3QueueFilesMetadata.cpp index 7b4438e1387..ed2f8d2ec1b 100644 --- a/src/Storages/S3Queue/S3QueueFilesMetadata.cpp +++ b/src/Storages/S3Queue/S3QueueFilesMetadata.cpp @@ -1,9 +1,5 @@ #include "config.h" -#include -#include -#include -#include #include #include #include @@ -12,6 +8,12 @@ #include #include #include +#include +#include +#include +#include +#include + #include #include #include diff --git a/src/Storages/StorageAzureBlob.h b/src/Storages/StorageAzureBlob.h index 27ac7a5c368..3f1ba33f636 100644 --- a/src/Storages/StorageAzureBlob.h +++ b/src/Storages/StorageAzureBlob.h @@ -100,7 +100,7 @@ public: bool supportsSubsetOfColumns(const ContextPtr & context) const; - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } bool prefersLargeBlocks() const override; diff --git a/src/Storages/StorageAzureBlobCluster.h b/src/Storages/StorageAzureBlobCluster.h index 545e568a772..eff4d70f1bd 100644 --- a/src/Storages/StorageAzureBlobCluster.h +++ b/src/Storages/StorageAzureBlobCluster.h @@ -35,7 +35,7 @@ public: bool supportsSubcolumns() const override { return true; } - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } private: void updateBeforeRead(const ContextPtr & /*context*/) override {} diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 1ee7c6fc6a5..12c2ad331ad 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -334,6 +334,7 @@ StorageDistributed::StorageDistributed( , remote_database(remote_database_) , remote_table(remote_table_) , remote_table_function_ptr(remote_table_function_ptr_) + , remote_storage(remote_table_function_ptr ? StorageID::createEmpty() : StorageID{remote_database, remote_table}) , log(getLogger("StorageDistributed (" + id_.table_name + ")")) , owned_cluster(std::move(owned_cluster_)) , cluster_name(getContext()->getMacros()->expand(cluster_name_)) @@ -896,10 +897,6 @@ void StorageDistributed::read( return; } - StorageID main_table = StorageID::createEmpty(); - if (!remote_table_function_ptr) - main_table = StorageID{remote_database, remote_table}; - const auto & snapshot_data = assert_cast(*storage_snapshot->data); ClusterProxy::SelectStreamFactory select_stream_factory = ClusterProxy::SelectStreamFactory( @@ -932,7 +929,7 @@ void StorageDistributed::read( query_plan, header, processed_stage, - main_table, + remote_storage, remote_table_function_ptr, select_stream_factory, log, @@ -978,10 +975,8 @@ SinkToStoragePtr StorageDistributed::write(const ASTPtr &, const StorageMetadata else columns_to_send = metadata_snapshot->getSampleBlockNonMaterialized().getNames(); - /// DistributedSink will not own cluster - return std::make_shared( - local_context, *this, metadata_snapshot, cluster, insert_sync, timeout, - StorageID{remote_database, remote_table}, columns_to_send); + /// DistributedSink will not own cluster, but will own ConnectionPools of the cluster + return std::make_shared(local_context, *this, metadata_snapshot, cluster, insert_sync, timeout, columns_to_send); } diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index 6709b1a2d8c..3a7e63aef50 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -241,6 +241,7 @@ private: String remote_database; String remote_table; ASTPtr remote_table_function_ptr; + StorageID remote_storage; LoggerPtr log; @@ -275,7 +276,7 @@ private: struct ClusterNodeData { std::shared_ptr directory_queue; - ConnectionPoolPtr connection_pool; + ConnectionPoolWithFailoverPtr connection_pool; Cluster::Addresses addresses; size_t clusters_version; }; diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index 93c263008a6..588429284f0 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -134,7 +134,7 @@ public: const ContextPtr & context, size_t & total_bytes_to_read); - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } protected: friend class StorageFileSource; diff --git a/src/Storages/StorageFileCluster.h b/src/Storages/StorageFileCluster.h index 3acbc71ba7e..973d595bbf0 100644 --- a/src/Storages/StorageFileCluster.h +++ b/src/Storages/StorageFileCluster.h @@ -32,7 +32,7 @@ public: bool supportsSubcolumns() const override { return true; } - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } private: void updateQueryToSendIfNeeded(ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const ContextPtr & context) override; diff --git a/src/Storages/StorageJoin.h b/src/Storages/StorageJoin.h index dc68c68a21b..c76df0cb452 100644 --- a/src/Storages/StorageJoin.h +++ b/src/Storages/StorageJoin.h @@ -85,7 +85,7 @@ public: const Names & getKeyNames() const { return key_names; } - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } private: Block sample_block; diff --git a/src/Storages/StorageMaterializedMySQL.h b/src/Storages/StorageMaterializedMySQL.h index 9f5d157ce3b..3e0cf3e6925 100644 --- a/src/Storages/StorageMaterializedMySQL.h +++ b/src/Storages/StorageMaterializedMySQL.h @@ -40,7 +40,7 @@ public: void drop() override { nested_storage->drop(); } - bool supportsTrivialCountOptimization() const override { return false; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return false; } IndexSizeByName getSecondaryIndexSizes() const override { diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index caec03c95b3..5eceddfe06d 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -583,7 +583,7 @@ std::vector ReadFromMerge::createChildrenPlans(SelectQ database_name, table_name, RowPolicyFilterType::SELECT_FILTER); - if (row_policy_filter_ptr) + if (row_policy_filter_ptr && !row_policy_filter_ptr->empty()) { row_policy_data_opt = RowPolicyData(row_policy_filter_ptr, storage, modified_context); row_policy_data_opt->extendNames(real_column_names); @@ -1675,9 +1675,9 @@ std::tuple StorageMerge::evaluateDatabaseName(cons return {false, ast}; } -bool StorageMerge::supportsTrivialCountOptimization() const +bool StorageMerge::supportsTrivialCountOptimization(const StorageSnapshotPtr & storage_snapshot, ContextPtr ctx) const { - return getFirstTable([&](const auto & table) { return !table->supportsTrivialCountOptimization(); }) == nullptr; + return getFirstTable([&](const auto & table) { return !table->supportsTrivialCountOptimization(storage_snapshot, ctx); }) == nullptr; } std::optional StorageMerge::totalRows(const Settings & settings) const diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index c049d50f3b4..a63ea1e32ef 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -76,7 +76,7 @@ public: /// Evaluate database name or regexp for StorageMerge and TableFunction merge static std::tuple evaluateDatabaseName(const ASTPtr & node, ContextPtr context); - bool supportsTrivialCountOptimization() const override; + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override; std::optional totalRows(const Settings & settings) const override; std::optional totalBytes(const Settings & settings) const override; diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 905473302ba..8ca061db4ec 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -141,6 +141,7 @@ namespace FailPoints { extern const char replicated_queue_fail_next_entry[]; extern const char replicated_queue_unfail_entries[]; + extern const char finish_set_quorum_failed_parts[]; } namespace ErrorCodes @@ -2221,6 +2222,7 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry, bool need_to_che if (code == Coordination::Error::ZOK) { LOG_DEBUG(log, "Marked quorum for part {} as failed.", entry.new_part_name); + FailPointInjection::disableFailPoint(FailPoints::finish_set_quorum_failed_parts); queue.removeFailedQuorumPart(part_info); return true; } diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index d8aefdf5b4c..c472c11e7f8 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -351,7 +351,7 @@ public: bool canUseZeroCopyReplication() const; - bool isTableReadOnly () { return is_readonly; } + bool isTableReadOnly () { return is_readonly || isStaticStorage(); } std::optional hasMetadataInZooKeeper () { return has_metadata_in_zookeeper; } diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 5b6ee90a1bf..2d3aef312bf 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -725,12 +725,12 @@ std::unique_ptr StorageS3Source::createAsyncS3ReadBuffer( auto context = getContext(); auto read_buffer_creator = [this, read_settings, object_size] - (bool restricted_seek, const std::string & path) -> std::unique_ptr + (bool restricted_seek, const StoredObject & object) -> std::unique_ptr { return std::make_unique( client, bucket, - path, + object.remote_path, version_id, request_settings, read_settings, diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index d1f15edfd6d..19cbfaa6f08 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -352,7 +352,7 @@ public: using KeysWithInfo = StorageS3Source::KeysWithInfo; - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } protected: virtual Configuration updateConfigurationAndGetCopy(const ContextPtr & local_context); diff --git a/src/Storages/StorageS3Cluster.h b/src/Storages/StorageS3Cluster.h index 6a5b03e682f..802fd3f9139 100644 --- a/src/Storages/StorageS3Cluster.h +++ b/src/Storages/StorageS3Cluster.h @@ -32,7 +32,7 @@ public: bool supportsSubcolumns() const override { return true; } - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } protected: void updateConfigurationIfChanged(ContextPtr local_context); diff --git a/src/Storages/StorageSet.cpp b/src/Storages/StorageSet.cpp index 7d7f3113cdb..54218351cf1 100644 --- a/src/Storages/StorageSet.cpp +++ b/src/Storages/StorageSet.cpp @@ -247,6 +247,8 @@ void StorageSetOrJoinBase::restore() static const char * file_suffix = ".bin"; static const auto file_suffix_size = strlen(".bin"); + using FilePriority = std::pair; + std::priority_queue, std::greater<>> backup_files; for (auto dir_it{disk->iterateDirectory(path)}; dir_it->isValid(); dir_it->next()) { const auto & name = dir_it->name(); @@ -261,9 +263,18 @@ void StorageSetOrJoinBase::restore() if (file_num > increment) increment = file_num; - restoreFromFile(dir_it->path()); + backup_files.push({file_num, file_path}); } } + + /// Restore in the same order as blocks were written + /// It may be important for storage Join, user expect to get the first row (unless `join_any_take_last_row` setting is set) + /// but after restart we may have different order of blocks in memory. + while (!backup_files.empty()) + { + restoreFromFile(backup_files.top().second); + backup_files.pop(); + } } diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index 842cfd5b627..5aca3df1513 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -127,7 +127,7 @@ protected: bool parallelizeOutputAfterReading(ContextPtr context) const override; - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } private: static std::pair getTableStructureAndFormatFromDataImpl( diff --git a/src/Storages/StorageURLCluster.h b/src/Storages/StorageURLCluster.h index dce2e0106ea..c80cdec74a2 100644 --- a/src/Storages/StorageURLCluster.h +++ b/src/Storages/StorageURLCluster.h @@ -35,7 +35,7 @@ public: bool supportsSubcolumns() const override { return true; } - bool supportsTrivialCountOptimization() const override { return true; } + bool supportsTrivialCountOptimization(const StorageSnapshotPtr &, ContextPtr) const override { return true; } private: void updateQueryToSendIfNeeded(ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const ContextPtr & context) override; diff --git a/src/Storages/System/StorageSystemContributors.generated.cpp b/src/Storages/System/StorageSystemContributors.generated.cpp index 0ad22082863..6a66d8a7e19 100644 --- a/src/Storages/System/StorageSystemContributors.generated.cpp +++ b/src/Storages/System/StorageSystemContributors.generated.cpp @@ -245,6 +245,7 @@ const char * auto_contributors[] { "Brendan Cox", "Brett Hoerner", "Brian Hunter", + "Brokenice0415", "Bulat Gaifullin", "Camden Cheek", "Camilo Sierra", @@ -286,6 +287,7 @@ const char * auto_contributors[] { "Dale Mcdiarmid", "Dalitso Banda", "Dan Roscigno", + "Dan Wu", "DanRoscigno", "Dani Pozo", "Daniel Bershatsky", @@ -294,6 +296,7 @@ const char * auto_contributors[] { "Daniel Kutenin", "Daniel Pozo Escalona", "Daniel Qin", + "Daniil Ivanik", "Daniil Rubin", "Danila Kutenin", "Daniël van Eeden", @@ -634,6 +637,7 @@ const char * auto_contributors[] { "LiuCong", "LiuNeng", "LiuYangkuan", + "LiuYuan", "Lloyd-Pottiger", "Lopatin Konstantin", "Lorenzo Mangani", @@ -668,6 +672,7 @@ const char * auto_contributors[] { "Marek Vavruša", "Marek Vavruša", "Mariano Benítez Mulet", + "Marina Fathouat", "Mark Andreev", "Mark Frost", "Mark Needham", @@ -767,6 +772,7 @@ const char * auto_contributors[] { "N. Kolotov", "NIKITA MIKHAILOV", "Narek Galstyan", + "Nataly Merezhuk", "Natalya Chizhonkova", "Natasha Murashkina", "NeZeD [Mac Pro]", @@ -787,6 +793,7 @@ const char * auto_contributors[] { "Nikhil Raman", "Nikifor Seriakov", "Nikita", + "Nikita Fomichev", "Nikita Keba", "Nikita Lapkov", "Nikita Mikhailov", @@ -804,10 +811,12 @@ const char * auto_contributors[] { "Nikolay Degterinsky", "Nikolay Edigaryev", "Nikolay Kirsh", + "Nikolay Monkov", "Nikolay Semyachkin", "Nikolay Shcheglov", "Nikolay Vasiliev", "Nikolay Volosatov", + "Nikolay Yankin", "Nir Peled", "Nityananda Gohain", "Niu Zhaojie", @@ -831,11 +840,13 @@ const char * auto_contributors[] { "Orkhan Zeynalli", "Oskar Wojciski", "OuO", + "Oxide Computer Company", "PHO", "Pablo Alegre", "Pablo Marcos", "Pablo Musa", "Palash Goel", + "PapaToemmsn", "Paramtamtam", "Patrick Zippenfenig", "Paul Loyd", @@ -859,7 +870,9 @@ const char * auto_contributors[] { "Persiyanov Dmitriy Andreevich", "Pervakov Grigorii", "Pervakov Grigory", + "Peter", "Petr Vasilev", + "Pham Anh Tuan", "Philip Hallstrom", "Philippe Ombredanne", "PigInCloud", @@ -973,11 +986,14 @@ const char * auto_contributors[] { "SevaCode", "Seyed Mehrshad Hosseini", "Shane Andrade", + "Shanfeng Pang", "Shani Elharrar", "Shaun Struwig", "Sherry Wang", "Shoh Jahon", "Shri Bodas", + "Shuai li", + "Shubham Ranjan", "Sichen Zhao", "SiderZhang", "Sidorov Pavel", @@ -1139,6 +1155,7 @@ const char * auto_contributors[] { "Wangyang Guo", "Waterkin", "Weiqing Xu", + "William Schoeffel", "William Shallum", "Winter Zhang", "Xbitz29", @@ -1252,6 +1269,7 @@ const char * auto_contributors[] { "awesomeleo", "bakam412", "bbkas", + "beetelbrox", "benamazing", "benbiti", "bgranvea", @@ -1261,6 +1279,7 @@ const char * auto_contributors[] { "bkuschel", "blazerer", "bluebirddm", + "bluikko", "bo zeng", "bobrovskij artemij", "booknouse", @@ -1309,6 +1328,7 @@ const char * auto_contributors[] { "d.v.semenov", "dalei2019", "damozhaeva", + "danila-ermakov", "dankondr", "daoready", "darkkeks", @@ -1324,6 +1344,7 @@ const char * auto_contributors[] { "dheerajathrey", "dimarub2000", "dinosaur", + "divanik", "divanorama", "dkxiaohei", "dmi-feo", @@ -1454,6 +1475,7 @@ const char * auto_contributors[] { "joelynch", "johanngan", "johnnymatthews", + "josh-hildred", "jsc0218", "jthmath", "jun won", @@ -1595,6 +1617,7 @@ const char * auto_contributors[] { "nautaa", "ndchikin", "nellicus", + "nemonlou", "neng.liu", "never lee", "ni1l", @@ -1637,6 +1660,7 @@ const char * auto_contributors[] { "pufit", "pyos", "pzhdfy", + "qaziqarta", "qianlixiang", "qianmoQ", "qieqieplus", @@ -1684,8 +1708,10 @@ const char * auto_contributors[] { "sev7e0", "sevirov", "sfod", + "shabroo", "shangshujie", "shedx", + "shuai-xu", "shuchaome", "shuyang", "sichenzhao", @@ -1710,6 +1736,7 @@ const char * auto_contributors[] { "sundy-li", "sundyli", "sunlisheng", + "sunny", "sunny19930321", "svladykin", "tai", @@ -1733,6 +1760,7 @@ const char * auto_contributors[] { "tiger.yan", "timfursov", "tison", + "tomershafir", "tomtana", "topvisor", "tpanetti", @@ -1740,6 +1768,7 @@ const char * auto_contributors[] { "tyrionhuang", "ubuntu", "una", + "unashi", "unbyte", "unegare", "unknown", @@ -1882,6 +1911,7 @@ const char * auto_contributors[] { "董海镔", "袁焊忠", "谢磊", + "豪肥肥", "贾顺名(Jarvis)", "郭小龙", "陈小玉", diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index 2acb5ebe221..b1ea2dd3f2b 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -24,6 +24,14 @@ #include #include +namespace +{ +constexpr auto * database_column_name = "database"; +constexpr auto * table_column_name = "table"; +constexpr auto * engine_column_name = "engine"; +constexpr auto * active_column_name = "active"; +constexpr auto * storage_uuid_column_name = "storage_uuid"; +} namespace DB { @@ -112,7 +120,7 @@ StoragesInfoStream::StoragesInfoStream(const ActionsDAGPtr & filter_by_database, database_column_mut->insert(database.first); } block_to_filter.insert(ColumnWithTypeAndName( - std::move(database_column_mut), std::make_shared(), "database")); + std::move(database_column_mut), std::make_shared(), database_column_name)); /// Filter block_to_filter with column 'database'. if (filter_by_database) @@ -120,7 +128,7 @@ StoragesInfoStream::StoragesInfoStream(const ActionsDAGPtr & filter_by_database, rows = block_to_filter.rows(); /// Block contains new columns, update database_column. - ColumnPtr database_column_for_filter = block_to_filter.getByName("database").column; + ColumnPtr database_column_for_filter = block_to_filter.getByName(database_column_name).column; if (rows) { @@ -187,10 +195,10 @@ StoragesInfoStream::StoragesInfoStream(const ActionsDAGPtr & filter_by_database, } } - block_to_filter.insert(ColumnWithTypeAndName(std::move(table_column_mut), std::make_shared(), "table")); - block_to_filter.insert(ColumnWithTypeAndName(std::move(engine_column_mut), std::make_shared(), "engine")); - block_to_filter.insert(ColumnWithTypeAndName(std::move(active_column_mut), std::make_shared(), "active")); - block_to_filter.insert(ColumnWithTypeAndName(std::move(storage_uuid_column_mut), std::make_shared(), "uuid")); + block_to_filter.insert(ColumnWithTypeAndName(std::move(table_column_mut), std::make_shared(), table_column_name)); + block_to_filter.insert(ColumnWithTypeAndName(std::move(engine_column_mut), std::make_shared(), engine_column_name)); + block_to_filter.insert(ColumnWithTypeAndName(std::move(active_column_mut), std::make_shared(), active_column_name)); + block_to_filter.insert(ColumnWithTypeAndName(std::move(storage_uuid_column_mut), std::make_shared(), storage_uuid_column_name)); if (rows) { @@ -200,10 +208,10 @@ StoragesInfoStream::StoragesInfoStream(const ActionsDAGPtr & filter_by_database, rows = block_to_filter.rows(); } - database_column = block_to_filter.getByName("database").column; - table_column = block_to_filter.getByName("table").column; - active_column = block_to_filter.getByName("active").column; - storage_uuid_column = block_to_filter.getByName("uuid").column; + database_column = block_to_filter.getByName(database_column_name).column; + table_column = block_to_filter.getByName(table_column_name).column; + active_column = block_to_filter.getByName(active_column_name).column; + storage_uuid_column = block_to_filter.getByName(storage_uuid_column_name).column; } class ReadFromSystemPartsBase : public SourceStepWithFilter @@ -261,16 +269,16 @@ void ReadFromSystemPartsBase::applyFilters(ActionDAGNodes added_filter_nodes) const auto * predicate = filter_actions_dag->getOutputs().at(0); Block block; - block.insert(ColumnWithTypeAndName({}, std::make_shared(), "database")); + block.insert(ColumnWithTypeAndName({}, std::make_shared(), database_column_name)); filter_by_database = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &block); if (filter_by_database) VirtualColumnUtils::buildSetsForDAG(filter_by_database, context); - block.insert(ColumnWithTypeAndName({}, std::make_shared(), "table")); - block.insert(ColumnWithTypeAndName({}, std::make_shared(), "engine")); - block.insert(ColumnWithTypeAndName({}, std::make_shared(), "active")); - block.insert(ColumnWithTypeAndName({}, std::make_shared(), "uuid")); + block.insert(ColumnWithTypeAndName({}, std::make_shared(), table_column_name)); + block.insert(ColumnWithTypeAndName({}, std::make_shared(), engine_column_name)); + block.insert(ColumnWithTypeAndName({}, std::make_shared(), active_column_name)); + block.insert(ColumnWithTypeAndName({}, std::make_shared(), storage_uuid_column_name)); filter_by_other_columns = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &block); if (filter_by_other_columns) diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp index 35550de11cb..5045dec3682 100644 --- a/src/Storages/System/StorageSystemReplicas.cpp +++ b/src/Storages/System/StorageSystemReplicas.cpp @@ -235,8 +235,8 @@ StorageSystemReplicas::StorageSystemReplicas(const StorageID & table_id_) "If log_pointer is much smaller than log_max_index, something is wrong."}, { "last_queue_update", std::make_shared(), "When the queue was updated last time."}, { "absolute_delay", std::make_shared(), "How big lag in seconds the current replica has."}, - { "total_replicas", std::make_shared(), "The total number of known replicas of this table."}, - { "active_replicas", std::make_shared(), "The number of replicas of this table that have a session in ClickHouse Keeper (i.e., the number of functioning replicas)."}, + { "total_replicas", std::make_shared(), "The total number of known replicas of this table."}, + { "active_replicas", std::make_shared(), "The number of replicas of this table that have a session in ClickHouse Keeper (i.e., the number of functioning replicas)."}, { "lost_part_count", std::make_shared(), "The number of data parts lost in the table by all replicas in total since table creation. Value is persisted in ClickHouse Keeper and can only increase."}, { "last_queue_update_exception", std::make_shared(), "When the queue contains broken entries. Especially important when ClickHouse breaks backward compatibility between versions and log entries written by newer versions aren't parseable by old versions."}, { "zookeeper_exception", std::make_shared(), "The last exception message, got if the error happened when fetching the info from ClickHouse Keeper."}, @@ -261,6 +261,7 @@ public: Block sample_block, std::map> replicated_tables_, bool with_zk_fields_, + size_t max_block_size_, std::shared_ptr impl_) : SourceStepWithFilter( DataStream{.header = std::move(sample_block)}, @@ -270,6 +271,7 @@ public: context_) , replicated_tables(std::move(replicated_tables_)) , with_zk_fields(with_zk_fields_) + , max_block_size(max_block_size_) , impl(std::move(impl_)) { } @@ -279,6 +281,7 @@ public: private: std::map> replicated_tables; const bool with_zk_fields; + const size_t max_block_size; std::shared_ptr impl; const ActionsDAG::Node * predicate = nullptr; }; @@ -297,7 +300,7 @@ void StorageSystemReplicas::read( SelectQueryInfo & query_info, ContextPtr context, QueryProcessingStage::Enum /*processed_stage*/, - const size_t /*max_block_size*/, + const size_t max_block_size, const size_t /*num_streams*/) { storage_snapshot->check(column_names); @@ -348,11 +351,51 @@ void StorageSystemReplicas::read( auto header = storage_snapshot->metadata->getSampleBlock(); auto reading = std::make_unique( column_names, query_info, storage_snapshot, - std::move(context), std::move(header), std::move(replicated_tables), with_zk_fields, impl); // /*std::move(this_ptr),*/ std::move(columns_mask), max_block_size); + std::move(context), std::move(header), std::move(replicated_tables), with_zk_fields, max_block_size, impl); query_plan.addStep(std::move(reading)); } +class SystemReplicasSource : public ISource +{ +public: + SystemReplicasSource( + Block header_, + size_t max_block_size_, + ColumnPtr col_database_, + ColumnPtr col_table_, + ColumnPtr col_engine_, + std::vector> futures_, + ContextPtr context_) + : ISource(header_) + , max_block_size(max_block_size_) + , col_database(std::move(col_database_)) + , col_table(std::move(col_table_)) + , col_engine(std::move(col_engine_)) + , futures(std::move(futures_)) + , context(std::move(context_)) + { + } + + String getName() const override { return "SystemReplicas"; } + +protected: + Chunk generate() override; + +private: + const size_t max_block_size; + /// Columns with table metadata. + ColumnPtr col_database; + ColumnPtr col_table; + ColumnPtr col_engine; + /// Futures for the status of each table. + std::vector> futures; + ContextPtr context; + /// Index (row number) of the next table to process. + size_t i = 0; +}; + + void ReadFromSystemReplicas::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { auto header = getOutputStream().header; @@ -398,8 +441,6 @@ void ReadFromSystemReplicas::initializePipeline(QueryPipelineBuilder & pipeline, col_engine = filtered_block.getByName("engine").column; } - MutableColumns res_columns = storage_snapshot->metadata->getSampleBlock().cloneEmptyColumns(); - size_t tables_size = col_database->size(); /// Use separate queues for requests with and without ZooKeeper fields. @@ -426,11 +467,46 @@ void ReadFromSystemReplicas::initializePipeline(QueryPipelineBuilder & pipeline, /// If there are more requests, they will be scheduled by the query that needs them. get_status_requests.scheduleRequests(max_request_id, query_status); - for (size_t i = 0; i < tables_size; ++i) + pipeline.init(Pipe(std::make_shared(header, max_block_size, col_database, col_table, col_engine, std::move(futures), context))); +} + +Chunk SystemReplicasSource::generate() +{ + if (i == futures.size()) + return {}; + + QueryStatusPtr query_status = context ? context->getProcessListElement() : nullptr; + + MutableColumns res_columns = getPort().getHeader().cloneEmptyColumns(); + + bool rows_added = false; + + for (; i < futures.size(); ++i) { if (query_status) query_status->checkTimeLimit(); + if (rows_added) + { + /// Return current chunk if the next future is not ready yet + if (futures[i].wait_for(std::chrono::seconds(0)) != std::future_status::ready) + break; + + if (max_block_size != 0) + { + size_t total_size = 0; + for (const auto & column : res_columns) + total_size += column->byteSize(); + /// If the block size exceeds the maximum, return the current block + if (total_size >= max_block_size) + break; + } + } + + res_columns[0]->insert((*col_database)[i]); + res_columns[1]->insert((*col_table)[i]); + res_columns[2]->insert((*col_engine)[i]); + const auto & status = futures[i].get(); size_t col_num = 3; res_columns[col_num++]->insert(status.is_leader); @@ -476,23 +552,12 @@ void ReadFromSystemReplicas::initializePipeline(QueryPipelineBuilder & pipeline, } res_columns[col_num++]->insert(std::move(replica_is_active_values)); + + rows_added = true; } - Columns fin_columns; - fin_columns.reserve(res_columns.size()); - - for (auto & col : res_columns) - fin_columns.emplace_back(std::move(col)); - - fin_columns[0] = std::move(col_database); - fin_columns[1] = std::move(col_table); - fin_columns[2] = std::move(col_engine); - - UInt64 num_rows = fin_columns.at(0)->size(); - Chunk chunk(std::move(fin_columns), num_rows); - - pipeline.init(Pipe(std::make_shared(header, std::move(chunk)))); + UInt64 num_rows = res_columns.at(0)->size(); + return Chunk(std::move(res_columns), num_rows); } - } diff --git a/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/src/Storages/tests/gtest_transform_query_for_external_database.cpp index 6490498d717..7e2d393c3d1 100644 --- a/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -306,8 +306,7 @@ TEST(TransformQueryForExternalDatabase, Aliases) check(state, 1, {"field"}, "SELECT field AS value, field AS display FROM table WHERE field NOT IN ('') AND display LIKE '%test%'", - R"(SELECT "field" FROM "test"."table" WHERE ("field" NOT IN ('')) AND ("field" LIKE '%test%'))", - R"(SELECT "field" FROM "test"."table" WHERE ("field" != '') AND ("field" LIKE '%test%'))"); + R"(SELECT "field" FROM "test"."table" WHERE ("field" NOT IN ('')) AND ("field" LIKE '%test%'))"); } TEST(TransformQueryForExternalDatabase, ForeignColumnInWhere) @@ -409,6 +408,5 @@ TEST(TransformQueryForExternalDatabase, Analyzer) check(state, 1, {"column", "apply_id", "apply_type", "apply_status", "create_time", "field", "value", "a", "b", "foo"}, "SELECT * FROM table WHERE (column) IN (1)", - R"(SELECT "column", "apply_id", "apply_type", "apply_status", "create_time", "field", "value", "a", "b", "foo" FROM "test"."table" WHERE "column" IN (1))", - R"(SELECT "column", "apply_id", "apply_type", "apply_status", "create_time", "field", "value", "a", "b", "foo" FROM "test"."table" WHERE "column" = 1)"); + R"(SELECT "column", "apply_id", "apply_type", "apply_status", "create_time", "field", "value", "a", "b", "foo" FROM "test"."table" WHERE "column" IN (1))"); } diff --git a/tests/analyzer_tech_debt.txt b/tests/analyzer_tech_debt.txt index 16f153253d6..5f798158a41 100644 --- a/tests/analyzer_tech_debt.txt +++ b/tests/analyzer_tech_debt.txt @@ -2,3 +2,10 @@ 01624_soft_constraints 02354_vector_search_queries 02901_parallel_replicas_rollup +02999_scalar_subqueries_bug_2 +# Flaky list +01825_type_json_in_array +01414_mutations_and_errors_zookeeper +01287_max_execution_speed +# Check after ConstantNode refactoring +02154_parser_backtracking diff --git a/tests/ci/ci.py b/tests/ci/ci.py index a3cea281a56..36e9b183805 100644 --- a/tests/ci/ci.py +++ b/tests/ci/ci.py @@ -12,7 +12,7 @@ from copy import deepcopy from dataclasses import asdict, dataclass from enum import Enum from pathlib import Path -from typing import Any, Dict, List, Optional, Sequence, Set, Union +from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Union import docker_images_helper import upload_result_helper @@ -733,6 +733,233 @@ class CiCache: return await_finished +@dataclass +class CiOptions: + # job will be included in the run if any keyword from the list matches job name + include_keywords: Optional[List[str]] = None + # job will be excluded in the run if any keyword from the list matches job name + exclude_keywords: Optional[List[str]] = None + + # list of specified preconfigured ci sets to run + ci_sets: Optional[List[str]] = None + # list of specified jobs to run + ci_jobs: Optional[List[str]] = None + + # btaches to run for all multi-batch jobs + job_batches: Optional[List[int]] = None + + do_not_test: bool = False + no_ci_cache: bool = False + no_merge_commit: bool = False + + def as_dict(self) -> Dict[str, Any]: + return asdict(self) + + @staticmethod + def create_from_run_config(run_config: Dict[str, Any]) -> "CiOptions": + return CiOptions(**run_config["ci_options"]) + + @staticmethod + def create_from_pr_message( + debug_message: Optional[str], update_from_api: bool + ) -> "CiOptions": + """ + Creates CiOptions instance based on tags found in PR body and/or commit message + @commit_message - may be provided directly for debugging purposes, otherwise it will be retrieved from git. + """ + res = CiOptions() + pr_info = PRInfo() + if ( + not pr_info.is_pr() and not debug_message + ): # if commit_message is provided it's test/debug scenario - do not return + # CI options can be configured in PRs only + return res + message = debug_message or GitRunner(set_cwd_to_git_root=True).run( + f"{GIT_PREFIX} log {pr_info.sha} --format=%B -n 1" + ) + + pattern = r"(#|- \[x\] + Integration tests +- [ ] Integration tests (arm64) +- [x] Integration tests +- [x] Integration tests +- [ ] Integration tests +- [x] some invalid mask - should be skipped +- [x] Integration tests +- [ ] Integration tests + +#### CI options: +- [ ] do not test (only style check) +- [x] disable merge-commit (no merge from master before tests) +- [ ] disable CI cache (job reuse) + +#### Only specified batches in multi-batch jobs: +- [x] 1 +- [ ] 2 +""" + +_TEST_BODY_2 = """ +- [x] MUST include integration tests +- [x] MUST include stateless tests +- [x] no action must be applied +- [ ] no action must be applied +- [x] MUST exclude tsan +- [x] MUST exclude aarch64 +- [x] MUST exclude test with analazer +- [ ] no action applied +- [x] Must exclude statless test with s3 storage +- [x] Must exclude tests on coverage build +""" + +_TEST_BODY_3 = """ +- [x] Must include all tests for analyzer +""" + + +class TestCIOptions(unittest.TestCase): + def test_pr_body_parsing(self): + ci_options = CiOptions.create_from_pr_message( + _TEST_BODY_1, update_from_api=False + ) + self.assertFalse(ci_options.do_not_test) + self.assertFalse(ci_options.no_ci_cache) + self.assertTrue(ci_options.no_merge_commit) + self.assertEqual(ci_options.ci_sets, ["ci_set_integration"]) + self.assertCountEqual(ci_options.include_keywords, ["foo", "foo_bar"]) + self.assertCountEqual(ci_options.exclude_keywords, ["foo", "foo_bar"]) + + def test_options_applied(self): + self.maxDiff = None + ci_options = CiOptions.create_from_pr_message( + _TEST_BODY_2, update_from_api=False + ) + self.assertCountEqual( + ci_options.include_keywords, ["integration", "foo_bar", "stateless"] + ) + self.assertCountEqual( + ci_options.exclude_keywords, + ["tsan", "aarch64", "analyzer", "s3_storage", "coverage"], + ) + jobs_to_do = list(JobNames) + jobs_to_skip = [] + job_params = {} + jobs_to_do, jobs_to_skip, job_params = ci_options.apply( + jobs_to_do, jobs_to_skip, job_params + ) + self.assertCountEqual( + jobs_to_do, + [ + "Style check", + "package_release", + "package_asan", + "package_ubsan", + "package_debug", + "package_msan", + "Stateless tests (asan)", + "Stateless tests flaky check (asan)", + "Stateless tests (msan)", + "Stateless tests (ubsan)", + "Stateless tests (debug)", + "Stateless tests (release)", + "Integration tests (release)", + "Integration tests (asan)", + "Integration tests flaky check (asan)", + ], + ) + + def test_options_applied_2(self): + self.maxDiff = None + ci_options = CiOptions.create_from_pr_message( + _TEST_BODY_3, update_from_api=False + ) + self.assertCountEqual(ci_options.include_keywords, ["analyzer"]) + self.assertIsNone(ci_options.exclude_keywords) + jobs_to_do = list(JobNames) + jobs_to_skip = [] + job_params = {} + jobs_to_do, jobs_to_skip, job_params = ci_options.apply( + jobs_to_do, jobs_to_skip, job_params + ) + self.assertCountEqual( + jobs_to_do, + [ + "Style check", + "Integration tests (asan, old analyzer)", + "package_release", + "Stateless tests (release, old analyzer, s3, DatabaseReplicated)", + "package_asan", + ], + ) diff --git a/tests/ci/upload_result_helper.py b/tests/ci/upload_result_helper.py index 9dca3fae1dc..cb745131e0d 100644 --- a/tests/ci/upload_result_helper.py +++ b/tests/ci/upload_result_helper.py @@ -92,13 +92,21 @@ def upload_results( else: raw_log_url = GITHUB_JOB_URL() + try: + job_url = GITHUB_JOB_URL() + except Exception: + print( + "ERROR: Failed to get job URL from GH API, job report will use run URL instead." + ) + job_url = GITHUB_RUN_URL + if test_results or not ready_report_url: html_report = create_test_html_report( check_name, test_results, raw_log_url, GITHUB_RUN_URL, - GITHUB_JOB_URL(), + job_url, branch_url, branch_name, commit_url, diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 10851d23481..624512058bc 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -69,6 +69,8 @@ TEST_FILE_EXTENSIONS = [".sql", ".sql.j2", ".sh", ".py", ".expect"] VERSION_PATTERN = r"^((\d+\.)?(\d+\.)?(\d+\.)?\d+)$" +TEST_MAX_RUN_TIME_IN_SECONDS = 120 + class SharedEngineReplacer: ENGINES_NON_REPLICATED_REGEXP = r"[ =]((Collapsing|VersionedCollapsing|Summing|Replacing|Aggregating|)MergeTree\(?\)?)" @@ -451,8 +453,7 @@ def get_processlist_size(args): """ SELECT count() - FROM - FROM system.processes + FROM clusterAllReplicas('test_cluster_database_replicated', system.processes) WHERE query NOT LIKE '%system.processes%' """, ).strip() @@ -682,7 +683,9 @@ class FailureReason(enum.Enum): STDERR = "having stderror: " EXCEPTION = "having exception in stdout: " RESULT_DIFF = "result differs with reference: " - TOO_LONG = "Test runs too long (> 60s). Make it faster." + TOO_LONG = ( + f"Test runs too long (> {TEST_MAX_RUN_TIME_IN_SECONDS}s). Make it faster." + ) INTERNAL_QUERY_FAIL = "Internal query (CREATE/DROP DATABASE) failed:" # SKIPPED reasons @@ -1421,7 +1424,7 @@ class TestCase: if ( self.testcase_args.test_runs > 1 - and total_time > 120 + and total_time > TEST_MAX_RUN_TIME_IN_SECONDS and "long" not in self.tags ): if debug_log: diff --git a/tests/config/config.d/clusters.xml b/tests/config/config.d/clusters.xml index 9490f98d6d8..af26565d7e6 100644 --- a/tests/config/config.d/clusters.xml +++ b/tests/config/config.d/clusters.xml @@ -67,6 +67,21 @@ + + + true + + shard_0 + localhost + 9000 + + + shard_1 + localhost + 9000 + + + 123457 diff --git a/tests/config/install.sh b/tests/config/install.sh index 467636cfa40..652d25a0a35 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -92,7 +92,7 @@ ln -sf $SRC_PATH/users.d/nonconst_timezone.xml $DEST_SERVER_PATH/users.d/ ln -sf $SRC_PATH/users.d/allow_introspection_functions.yaml $DEST_SERVER_PATH/users.d/ ln -sf $SRC_PATH/users.d/replicated_ddl_entry.xml $DEST_SERVER_PATH/users.d/ -if [[ -n "$USE_NEW_ANALYZER" ]] && [[ "$USE_NEW_ANALYZER" -eq 1 ]]; then +if [[ -n "$USE_OLD_ANALYZER" ]] && [[ "$USE_OLD_ANALYZER" -eq 1 ]]; then ln -sf $SRC_PATH/users.d/analyzer.xml $DEST_SERVER_PATH/users.d/ fi diff --git a/tests/config/users.d/analyzer.xml b/tests/config/users.d/analyzer.xml index aa374364ef0..4b9764526fa 100644 --- a/tests/config/users.d/analyzer.xml +++ b/tests/config/users.d/analyzer.xml @@ -1,7 +1,7 @@ - 1 + 0 diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index a4b18ff523a..f4be31cc532 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -13,6 +13,34 @@ from helpers.network import _NetworkManager logging.raiseExceptions = False +@pytest.fixture(scope="session", autouse=True) +def pdb_history(request): + """ + Fixture loads and saves pdb history to file, so it can be preserved between runs + """ + if request.config.getoption("--pdb"): + import readline # pylint:disable=import-outside-toplevel + import pdb # pylint:disable=import-outside-toplevel + + def save_history(): + readline.write_history_file(".pdb_history") + + def load_history(): + try: + readline.read_history_file(".pdb_history") + except FileNotFoundError: + pass + + load_history() + pdb.Pdb.use_rawinput = True + + yield + + save_history() + else: + yield + + @pytest.fixture(autouse=True, scope="session") def tune_local_port_range(): # Lots of services uses non privileged ports: diff --git a/tests/integration/helpers/0_common_enable_analyzer.xml b/tests/integration/helpers/0_common_enable_analyzer.xml index aa374364ef0..4b9764526fa 100644 --- a/tests/integration/helpers/0_common_enable_analyzer.xml +++ b/tests/integration/helpers/0_common_enable_analyzer.xml @@ -1,7 +1,7 @@ - 1 + 0 diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 281d7d8476c..1f29dfe8eee 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -862,12 +862,12 @@ class ClickHouseCluster: def get_docker_handle(self, docker_id): exception = None - for i in range(5): + for i in range(20): try: return self.docker_client.containers.get(docker_id) except Exception as ex: print("Got exception getting docker handle", str(ex)) - time.sleep(i * 2) + time.sleep(0.5) exception = ex raise exception @@ -1601,7 +1601,7 @@ class ClickHouseCluster: with_jdbc_bridge=False, with_hive=False, with_coredns=False, - allow_analyzer=True, + use_old_analyzer=False, hostname=None, env_variables=None, instance_env_variables=False, @@ -1700,7 +1700,7 @@ class ClickHouseCluster: with_coredns=with_coredns, with_cassandra=with_cassandra, with_ldap=with_ldap, - allow_analyzer=allow_analyzer, + use_old_analyzer=use_old_analyzer, server_bin_path=self.server_bin_path, odbc_bridge_bin_path=self.odbc_bridge_bin_path, library_bridge_bin_path=self.library_bridge_bin_path, @@ -3262,7 +3262,7 @@ class ClickHouseInstance: with_coredns, with_cassandra, with_ldap, - allow_analyzer, + use_old_analyzer, server_bin_path, odbc_bridge_bin_path, library_bridge_bin_path, @@ -3356,7 +3356,7 @@ class ClickHouseInstance: self.with_hive = with_hive self.with_coredns = with_coredns self.coredns_config_dir = p.abspath(p.join(base_path, "coredns_config")) - self.allow_analyzer = allow_analyzer + self.use_old_analyzer = use_old_analyzer self.main_config_name = main_config_name self.users_config_name = users_config_name @@ -4405,10 +4405,7 @@ class ClickHouseInstance: ) write_embedded_config("0_common_instance_users.xml", users_d_dir) - if ( - os.environ.get("CLICKHOUSE_USE_NEW_ANALYZER") is not None - and self.allow_analyzer - ): + if self.use_old_analyzer: write_embedded_config("0_common_enable_analyzer.xml", users_d_dir) if len(self.custom_dictionaries_paths): diff --git a/tests/integration/runner b/tests/integration/runner index 2834eaf311b..97d06c2b78c 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -387,7 +387,7 @@ if __name__ == "__main__": use_analyzer = "" if args.analyzer: - use_analyzer = "-e CLICKHOUSE_USE_NEW_ANALYZER=1" + use_analyzer = "-e CLICKHOUSE_USE_OLD_ANALYZER=1" # NOTE: since pytest options is in the argument value already we need to additionally escape '"' pytest_opts = " ".join( diff --git a/tests/integration/test_async_load_databases/test.py b/tests/integration/test_async_load_databases/test.py index 050b529a227..d06897b1045 100644 --- a/tests/integration/test_async_load_databases/test.py +++ b/tests/integration/test_async_load_databases/test.py @@ -122,6 +122,9 @@ def test_dependent_tables(started_cluster): ) query("create table system.join (n int, m int) engine=Join(any, left, n)") query("insert into system.join values (1, 1)") + for i in range(2, 100): + query(f"insert into system.join values (1, {i})") + query( "create table src (n int, m default joinGet('system.join', 'm', 1::int)," "t default dictGetOrNull('a.d', 'm', toUInt64(3))," diff --git a/tests/integration/test_backup_restore_on_cluster/test.py b/tests/integration/test_backup_restore_on_cluster/test.py index c76e9718640..2c60b096428 100644 --- a/tests/integration/test_backup_restore_on_cluster/test.py +++ b/tests/integration/test_backup_restore_on_cluster/test.py @@ -1,4 +1,3 @@ -from time import sleep import pytest import re import os.path @@ -164,8 +163,15 @@ def test_replicated_database(): node2.query("INSERT INTO mydb.tbl VALUES (2, 'count')") node1.query("INSERT INTO mydb.tbl VALUES (3, 'your')") node2.query("INSERT INTO mydb.tbl VALUES (4, 'chickens')") + node1.query("OPTIMIZE TABLE mydb.tbl ON CLUSTER 'cluster' FINAL") + node1.query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster' mydb.tbl") + # check data in sync + expect = TSV([[1, "Don\\'t"], [2, "count"], [3, "your"], [4, "chickens"]]) + assert node1.query("SELECT * FROM mydb.tbl ORDER BY x") == expect + assert node2.query("SELECT * FROM mydb.tbl ORDER BY x") == expect + # Make backup. backup_name = new_backup_name() node1.query( @@ -179,14 +185,63 @@ def test_replicated_database(): node1.query(f"RESTORE DATABASE mydb ON CLUSTER 'cluster' FROM {backup_name}") node1.query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster' mydb.tbl") - assert node1.query("SELECT * FROM mydb.tbl ORDER BY x") == TSV( - [[1, "Don\\'t"], [2, "count"], [3, "your"], [4, "chickens"]] + assert node1.query("SELECT * FROM mydb.tbl ORDER BY x") == expect + assert node2.query("SELECT * FROM mydb.tbl ORDER BY x") == expect + + +def test_replicated_database_compare_parts(): + """ + stop merges and fetches then write data to two nodes and + compare that parts are restored from single node (second) after backup + replica is selected by settings replica_num=2, replica_num_in_backup=2 + """ + node1.query( + "CREATE DATABASE mydb ON CLUSTER 'cluster' ENGINE=Replicated('/clickhouse/path/','{shard}','{replica}')" ) - assert node2.query("SELECT * FROM mydb.tbl ORDER BY x") == TSV( - [[1, "Don\\'t"], [2, "count"], [3, "your"], [4, "chickens"]] + node1.query( + "CREATE TABLE mydb.tbl(x UInt8, y String) ENGINE=ReplicatedMergeTree ORDER BY x" ) + node2.query("SYSTEM SYNC DATABASE REPLICA mydb") + + node1.query("SYSTEM STOP MERGES mydb.tbl") + node2.query("SYSTEM STOP MERGES mydb.tbl") + + node1.query("SYSTEM STOP FETCHES mydb.tbl") + node2.query("SYSTEM STOP FETCHES mydb.tbl") + + node1.query("INSERT INTO mydb.tbl VALUES (1, 'a')") + node1.query("INSERT INTO mydb.tbl VALUES (2, 'b')") + + node2.query("INSERT INTO mydb.tbl VALUES (3, 'x')") + node2.query("INSERT INTO mydb.tbl VALUES (4, 'y')") + + p2 = node2.query("SELECT * FROM mydb.tbl ORDER BY x") + + # Make backup. + backup_name = new_backup_name() + node1.query( + f"BACKUP DATABASE mydb ON CLUSTER 'cluster' TO {backup_name} SETTINGS replica_num=2" + ) + + # Drop table on both nodes. + node1.query("DROP DATABASE mydb ON CLUSTER 'cluster' SYNC") + + # Restore from backup on node2. + node1.query( + f"RESTORE DATABASE mydb ON CLUSTER 'cluster' FROM {backup_name} SETTINGS replica_num_in_backup=2" + ) + node1.query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster' mydb.tbl") + + # compare parts + p1_ = node1.query("SELECT _part, * FROM mydb.tbl ORDER BY x") + p2_ = node2.query("SELECT _part, * FROM mydb.tbl ORDER BY x") + assert p1_ == p2_ + + # compare data + assert p2 == node2.query("SELECT * FROM mydb.tbl ORDER BY x") + def test_different_tables_on_nodes(): node1.query( @@ -427,7 +482,12 @@ def test_replicated_database_async(): node1.query("INSERT INTO mydb.tbl VALUES (22)") node2.query("INSERT INTO mydb.tbl2 VALUES ('a')") node2.query("INSERT INTO mydb.tbl2 VALUES ('bb')") + + node1.query("OPTIMIZE TABLE mydb.tbl ON CLUSTER 'cluster' FINAL") + node1.query("OPTIMIZE TABLE mydb.tbl2 ON CLUSTER 'cluster' FINAL") + node1.query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster' mydb.tbl") + node1.query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster' mydb.tbl2") backup_name = new_backup_name() [id, status] = node1.query( diff --git a/tests/integration/test_backward_compatibility/test.py b/tests/integration/test_backward_compatibility/test.py index 098fc8c1025..7de5f51921b 100644 --- a/tests/integration/test_backward_compatibility/test.py +++ b/tests/integration/test_backward_compatibility/test.py @@ -10,13 +10,12 @@ node1 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, ) node2 = cluster.add_instance( "node2", main_configs=["configs/wide_parts_only.xml", "configs/no_compress_marks.xml"], with_zookeeper=True, - allow_analyzer=False, + use_old_analyzer=True, ) diff --git a/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py b/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py index b0c0f5d17c7..6b385bf8402 100644 --- a/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py +++ b/tests/integration/test_backward_compatibility/test_aggregate_fixed_key.py @@ -9,10 +9,9 @@ node1 = cluster.add_instance( image="clickhouse/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, - allow_analyzer=False, ) -node2 = cluster.add_instance("node2", with_zookeeper=True, allow_analyzer=False) -node3 = cluster.add_instance("node3", with_zookeeper=True, allow_analyzer=False) +node2 = cluster.add_instance("node2", with_zookeeper=True, use_old_analyzer=True) +node3 = cluster.add_instance("node3", with_zookeeper=True, use_old_analyzer=True) @pytest.fixture(scope="module") diff --git a/tests/integration/test_backward_compatibility/test_aggregate_function_state.py b/tests/integration/test_backward_compatibility/test_aggregate_function_state.py index 5972f57b928..9878c1ed70e 100644 --- a/tests/integration/test_backward_compatibility/test_aggregate_function_state.py +++ b/tests/integration/test_backward_compatibility/test_aggregate_function_state.py @@ -10,7 +10,6 @@ node1 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, ) node2 = cluster.add_instance( "node2", @@ -19,10 +18,9 @@ node2 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, ) -node3 = cluster.add_instance("node3", with_zookeeper=False, allow_analyzer=False) -node4 = cluster.add_instance("node4", with_zookeeper=False, allow_analyzer=False) +node3 = cluster.add_instance("node3", with_zookeeper=False, use_old_analyzer=True) +node4 = cluster.add_instance("node4", with_zookeeper=False, use_old_analyzer=True) @pytest.fixture(scope="module") diff --git a/tests/integration/test_backward_compatibility/test_convert_ordinary.py b/tests/integration/test_backward_compatibility/test_convert_ordinary.py index 8e7d773ad2c..b8db4e005a4 100644 --- a/tests/integration/test_backward_compatibility/test_convert_ordinary.py +++ b/tests/integration/test_backward_compatibility/test_convert_ordinary.py @@ -9,7 +9,6 @@ node = cluster.add_instance( stay_alive=True, with_zookeeper=True, with_installed_binary=True, - allow_analyzer=False, ) diff --git a/tests/integration/test_backward_compatibility/test_cte_distributed.py b/tests/integration/test_backward_compatibility/test_cte_distributed.py index e612bf2989a..e0be009e874 100644 --- a/tests/integration/test_backward_compatibility/test_cte_distributed.py +++ b/tests/integration/test_backward_compatibility/test_cte_distributed.py @@ -3,7 +3,7 @@ import pytest from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance("node1", with_zookeeper=False, allow_analyzer=False) +node1 = cluster.add_instance("node1", with_zookeeper=False, use_old_analyzer=True) node2 = cluster.add_instance( "node2", with_zookeeper=False, @@ -11,7 +11,6 @@ node2 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, ) diff --git a/tests/integration/test_backward_compatibility/test_functions.py b/tests/integration/test_backward_compatibility/test_functions.py index 0217c46a660..1cf5c3deb81 100644 --- a/tests/integration/test_backward_compatibility/test_functions.py +++ b/tests/integration/test_backward_compatibility/test_functions.py @@ -9,13 +9,12 @@ from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION from helpers.client import QueryRuntimeException cluster = ClickHouseCluster(__file__) -upstream = cluster.add_instance("upstream", allow_analyzer=False) +upstream = cluster.add_instance("upstream", use_old_analyzer=True) backward = cluster.add_instance( "backward", image="clickhouse/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, - allow_analyzer=False, ) diff --git a/tests/integration/test_backward_compatibility/test_insert_profile_events.py b/tests/integration/test_backward_compatibility/test_insert_profile_events.py index 30958127ee5..a90453d045b 100644 --- a/tests/integration/test_backward_compatibility/test_insert_profile_events.py +++ b/tests/integration/test_backward_compatibility/test_insert_profile_events.py @@ -7,13 +7,12 @@ import pytest from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION cluster = ClickHouseCluster(__file__) -upstream_node = cluster.add_instance("upstream_node", allow_analyzer=False) +upstream_node = cluster.add_instance("upstream_node", use_old_analyzer=True) old_node = cluster.add_instance( "old_node", image="clickhouse/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, - allow_analyzer=False, ) diff --git a/tests/integration/test_backward_compatibility/test_ip_types_binary_compatibility.py b/tests/integration/test_backward_compatibility/test_ip_types_binary_compatibility.py index 02c81ddbd52..4752a589a44 100644 --- a/tests/integration/test_backward_compatibility/test_ip_types_binary_compatibility.py +++ b/tests/integration/test_backward_compatibility/test_ip_types_binary_compatibility.py @@ -10,7 +10,6 @@ node = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, ) diff --git a/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py b/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py index 337a967e309..b13e6c975e8 100644 --- a/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py +++ b/tests/integration/test_backward_compatibility/test_memory_bound_aggregation.py @@ -10,7 +10,6 @@ node1 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, ) node2 = cluster.add_instance( "node2", @@ -19,9 +18,8 @@ node2 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, ) -node3 = cluster.add_instance("node3", with_zookeeper=False, allow_analyzer=False) +node3 = cluster.add_instance("node3", with_zookeeper=False, use_old_analyzer=True) @pytest.fixture(scope="module") diff --git a/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py b/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py index 7f6c3fc92e1..83be0e4c5a3 100644 --- a/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py +++ b/tests/integration/test_backward_compatibility/test_normalized_count_comparison.py @@ -3,7 +3,7 @@ import pytest from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance("node1", with_zookeeper=False, allow_analyzer=False) +node1 = cluster.add_instance("node1", with_zookeeper=False, use_old_analyzer=True) node2 = cluster.add_instance( "node2", with_zookeeper=False, @@ -11,7 +11,6 @@ node2 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, ) diff --git a/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py b/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py index be161df0640..cbe147dc07b 100644 --- a/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py +++ b/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py @@ -3,7 +3,7 @@ import pytest from helpers.cluster import ClickHouseCluster, CLICKHOUSE_CI_MIN_TESTED_VERSION cluster = ClickHouseCluster(__file__) -node1 = cluster.add_instance("node1", with_zookeeper=False, allow_analyzer=False) +node1 = cluster.add_instance("node1", with_zookeeper=False, use_old_analyzer=True) node2 = cluster.add_instance( "node2", with_zookeeper=False, @@ -11,7 +11,6 @@ node2 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, ) diff --git a/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py b/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py index 5d0e5e24af5..60375196366 100644 --- a/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py +++ b/tests/integration/test_backward_compatibility/test_short_strings_aggregation.py @@ -10,7 +10,6 @@ node1 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, ) node2 = cluster.add_instance( "node2", @@ -19,9 +18,8 @@ node2 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, ) -node3 = cluster.add_instance("node3", with_zookeeper=False, allow_analyzer=False) +node3 = cluster.add_instance("node3", with_zookeeper=False, use_old_analyzer=True) @pytest.fixture(scope="module") diff --git a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py index d3730357989..e36c3310e4a 100644 --- a/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py +++ b/tests/integration/test_backward_compatibility/test_vertical_merges_from_compact_parts.py @@ -11,7 +11,6 @@ node_old = cluster.add_instance( stay_alive=True, with_installed_binary=True, with_zookeeper=True, - allow_analyzer=False, ) node_new = cluster.add_instance( "node2", @@ -21,7 +20,7 @@ node_new = cluster.add_instance( ], with_zookeeper=True, stay_alive=True, - allow_analyzer=False, + use_old_analyzer=True, ) diff --git a/tests/integration/test_broken_projections/test.py b/tests/integration/test_broken_projections/test.py index 4a4690a5d0a..e198f98e4c5 100644 --- a/tests/integration/test_broken_projections/test.py +++ b/tests/integration/test_broken_projections/test.py @@ -148,7 +148,7 @@ def break_part(node, table, part): bash(node, f"rm '{part_path}/columns.txt'") -def get_broken_projections_info(node, table): +def get_broken_projections_info(node, table, active=True): return node.query( f""" SELECT parent_name, name, errors.name FROM @@ -158,6 +158,7 @@ def get_broken_projections_info(node, table): WHERE table='{table}' AND database=currentDatabase() AND is_broken = 1 + AND active = {active} ) AS parts_info INNER JOIN system.errors AS errors ON parts_info.exception_code = errors.code @@ -214,23 +215,33 @@ def random_str(length=6): return "".join(random.SystemRandom().choice(alphabet) for _ in range(length)) -def check(node, table, check_result, expect_broken_part="", expected_error=""): +def check( + node, + table, + check_result, + expect_broken_part="", + expected_error="", + do_check_command=True, +): if expect_broken_part == "proj1": assert expected_error in node.query_and_get_error( - f"SELECT c FROM '{table}' WHERE d == 12 ORDER BY c" + f"SELECT c FROM '{table}' WHERE d == 12 ORDER BY c SETTINGS force_optimize_projection_name = 'proj1'" ) else: query_id = node.query( - f"SELECT queryID() FROM (SELECT c FROM '{table}' WHERE d == 12 ORDER BY c)" + f"SELECT queryID() FROM (SELECT c FROM '{table}' WHERE d == 12 ORDER BY c SETTINGS force_optimize_projection_name = 'proj1')" ).strip() - node.query("SYSTEM FLUSH LOGS") - res = node.query( - f""" - SELECT query, splitByChar('.', arrayJoin(projections))[-1] - FROM system.query_log - WHERE query_id='{query_id}' AND type='QueryFinish' - """ - ) + for _ in range(10): + node.query("SYSTEM FLUSH LOGS") + res = node.query( + f""" + SELECT query, splitByChar('.', arrayJoin(projections))[-1] + FROM system.query_log + WHERE query_id='{query_id}' AND type='QueryFinish' + """ + ) + if res != "": + break if res == "": res = node.query( """ @@ -238,26 +249,29 @@ def check(node, table, check_result, expect_broken_part="", expected_error=""): FROM system.query_log ORDER BY query_start_time_microseconds DESC """ ) - print(f"LOG: {res}") + print(f"Looked for query id {query_id}, but to no avail: {res}") assert False assert "proj1" in res if expect_broken_part == "proj2": assert expected_error in node.query_and_get_error( - f"SELECT d FROM '{table}' WHERE c == 12 ORDER BY d" + f"SELECT d FROM '{table}' WHERE c == 12 ORDER BY d SETTINGS force_optimize_projection_name = 'proj2'" ) else: query_id = node.query( - f"SELECT queryID() FROM (SELECT d FROM '{table}' WHERE c == 12 ORDER BY d)" + f"SELECT queryID() FROM (SELECT d FROM '{table}' WHERE c == 12 ORDER BY d SETTINGS force_optimize_projection_name = 'proj2')" ).strip() - node.query("SYSTEM FLUSH LOGS") - res = node.query( - f""" - SELECT query, splitByChar('.', arrayJoin(projections))[-1] - FROM system.query_log - WHERE query_id='{query_id}' AND type='QueryFinish' - """ - ) + for _ in range(10): + node.query("SYSTEM FLUSH LOGS") + res = node.query( + f""" + SELECT query, splitByChar('.', arrayJoin(projections))[-1] + FROM system.query_log + WHERE query_id='{query_id}' AND type='QueryFinish' + """ + ) + if res != "": + break if res == "": res = node.query( """ @@ -265,11 +279,12 @@ def check(node, table, check_result, expect_broken_part="", expected_error=""): FROM system.query_log ORDER BY query_start_time_microseconds DESC """ ) - print(f"LOG: {res}") + print(f"Looked for query id {query_id}, but to no avail: {res}") assert False assert "proj2" in res - assert check_result == int(node.query(f"CHECK TABLE {table}")) + if do_check_command: + assert check_result == int(node.query(f"CHECK TABLE {table}")) def test_broken_ignored(cluster): @@ -352,7 +367,14 @@ def test_broken_ignored(cluster): # """) # ) - assert "all_3_3_0" in get_broken_projections_info(node, table_name) + assert ["all_0_0_0", "all_1_1_0", "all_2_2_0", "all_3_5_1"] == get_parts( + node, table_name + ) + + assert "all_3_3_0" in get_broken_projections_info(node, table_name, active=False) + assert "all_2_2_0" in get_broken_projections_info(node, table_name, active=True) + + # 0 because of all_2_2_0 check(node, table_name, 0) @@ -574,3 +596,168 @@ def test_broken_projections_in_backups_3(cluster): assert "all_1_1_0\tproj1\tNO_FILE_IN_DATA_PART" == get_broken_projections_info( node, table_name ) + + +def test_check_part_thread(cluster): + node = cluster.instances["node"] + + table_name = "check_part_thread_test1" + create_table(node, table_name, 1) + + insert(node, table_name, 0, 5) + insert(node, table_name, 5, 5) + insert(node, table_name, 10, 5) + insert(node, table_name, 15, 5) + + assert ["all_0_0_0", "all_1_1_0", "all_2_2_0", "all_3_3_0"] == get_parts( + node, table_name + ) + + # Break data file of projection 'proj2' for part all_2_2_0 + break_projection(node, table_name, "proj2", "all_2_2_0", "data") + + # It will not yet appear in broken projections info. + assert "proj2" not in get_broken_projections_info(node, table_name) + + # Select now fails with error "File doesn't exist" + check(node, table_name, 0, "proj2", "FILE_DOESNT_EXIST", do_check_command=False) + + good = False + for _ in range(10): + # We marked projection as broken, checkPartThread must not complain about the part. + good = node.contains_in_log( + f"{table_name} (ReplicatedMergeTreePartCheckThread): Part all_2_2_0 looks good" + ) + if good: + break + time.sleep(1) + + assert good + + +def test_broken_on_start(cluster): + node = cluster.instances["node"] + + table_name = "test1" + create_table(node, table_name, 1) + + insert(node, table_name, 0, 5) + insert(node, table_name, 5, 5) + insert(node, table_name, 10, 5) + insert(node, table_name, 15, 5) + + assert ["all_0_0_0", "all_1_1_0", "all_2_2_0", "all_3_3_0"] == get_parts( + node, table_name + ) + + # Break data file of projection 'proj2' for part all_2_2_0 + break_projection(node, table_name, "proj2", "all_2_2_0", "data") + + # It will not yet appear in broken projections info. + assert "proj2" not in get_broken_projections_info(node, table_name) + + # Select now fails with error "File doesn't exist" + # We will mark projection as broken. + check(node, table_name, 0, "proj2", "FILE_DOESNT_EXIST") + + # Projection 'proj2' from part all_2_2_0 will now appear in broken parts info. + assert "all_2_2_0\tproj2\tNO_FILE_IN_DATA_PART" in get_broken_projections_info( + node, table_name + ) + + # Second select works, because projection is now marked as broken. + check(node, table_name, 0) + + node.restart_clickhouse() + + # It will not yet appear in broken projections info. + assert "proj2" in get_broken_projections_info(node, table_name) + + # Select works + check(node, table_name, 0) + + +def test_mutation_with_broken_projection(cluster): + node = cluster.instances["node"] + + table_name = "test1" + create_table(node, table_name, 1) + + insert(node, table_name, 0, 5) + insert(node, table_name, 5, 5) + insert(node, table_name, 10, 5) + insert(node, table_name, 15, 5) + + assert ["all_0_0_0", "all_1_1_0", "all_2_2_0", "all_3_3_0"] == get_parts( + node, table_name + ) + + check(node, table_name, 1) + + node.query( + f"ALTER TABLE {table_name} DELETE WHERE c == 11 SETTINGS mutations_sync = 1" + ) + + assert ["all_0_0_0_4", "all_1_1_0_4", "all_2_2_0_4", "all_3_3_0_4"] == get_parts( + node, table_name + ) + + assert "" == get_broken_projections_info(node, table_name) + + check(node, table_name, 1) + + # Break data file of projection 'proj2' for part all_2_2_0_4 + break_projection(node, table_name, "proj2", "all_2_2_0_4", "data") + + # It will not yet appear in broken projections info. + assert "proj2" not in get_broken_projections_info(node, table_name) + + # Select now fails with error "File doesn't exist" + # We will mark projection as broken. + check(node, table_name, 0, "proj2", "FILE_DOESNT_EXIST") + + # Projection 'proj2' from part all_2_2_0_4 will now appear in broken parts info. + assert "all_2_2_0_4\tproj2\tNO_FILE_IN_DATA_PART" in get_broken_projections_info( + node, table_name + ) + + # Second select works, because projection is now marked as broken. + check(node, table_name, 0) + + assert "all_2_2_0_4" in get_broken_projections_info(node, table_name) + + node.query( + f"ALTER TABLE {table_name} DELETE WHERE _part == 'all_0_0_0_4' SETTINGS mutations_sync = 1" + ) + + # All parts changes because this is how alter delete works, + # but all parts apart from the first have only hardlinks to files in previous part. + assert ["all_0_0_0_5", "all_1_1_0_5", "all_2_2_0_5", "all_3_3_0_5"] == get_parts( + node, table_name + ) or ["all_1_1_0_5", "all_2_2_0_5", "all_3_3_0_5"] == get_parts(node, table_name) + + # Still broken because it was hardlinked. + broken = get_broken_projections_info(node, table_name) + assert ( + "all_2_2_0_5" in broken or "" == broken + ) # second could be because of a merge. + + if "" == broken: + check(node, table_name, 1) + else: + check(node, table_name, 0) + + node.query( + f"ALTER TABLE {table_name} DELETE WHERE c == 13 SETTINGS mutations_sync = 1" + ) + + assert ["all_1_1_0_6", "all_2_2_0_6", "all_3_3_0_6"] == get_parts( + node, table_name + ) or ["all_0_0_0_6", "all_1_1_0_6", "all_2_2_0_6", "all_3_3_0_6"] == get_parts( + node, table_name + ) + + # Not broken anymore. + assert "" == get_broken_projections_info(node, table_name) + + check(node, table_name, 1) diff --git a/tests/integration/test_dictionaries_update_and_reload/test.py b/tests/integration/test_dictionaries_update_and_reload/test.py index 648ea847afb..db1c8e47467 100644 --- a/tests/integration/test_dictionaries_update_and_reload/test.py +++ b/tests/integration/test_dictionaries_update_and_reload/test.py @@ -37,6 +37,16 @@ def get_status(dictionary_name): ).rstrip("\n") +def get_status_retry(dictionary_name, expect, retry_count=10, sleep_time=0.5): + for _ in range(retry_count): + res = get_status(dictionary_name) + if res == expect: + return res + time.sleep(sleep_time) + + raise Exception(f'Expected result "{expect}" did not occur') + + def get_last_exception(dictionary_name): return ( instance.query( @@ -250,6 +260,15 @@ def test_reload_after_fail_by_timer(started_cluster): assert expected_error in instance.query_and_get_error( "SELECT dictGetInt32('no_file_2', 'a', toUInt64(9))" ) + + # on sanitizers builds it can return 'FAILED_AND_RELOADING' which is not quite right + # add retry for these builds + if ( + instance.is_built_with_sanitizer() + and get_status("no_file_2") == "FAILED_AND_RELOADING" + ): + get_status_retry("no_file_2", expect="FAILED") + assert get_status("no_file_2") == "FAILED" # Creating the file source makes the dictionary able to load. diff --git a/tests/integration/test_disk_over_web_server/configs/storage_conf_web.xml b/tests/integration/test_disk_over_web_server/configs/storage_conf_web.xml index 56ee3e532c4..482aa78e611 100644 --- a/tests/integration/test_disk_over_web_server/configs/storage_conf_web.xml +++ b/tests/integration/test_disk_over_web_server/configs/storage_conf_web.xml @@ -29,4 +29,6 @@ + + diff --git a/tests/integration/test_disk_over_web_server/test.py b/tests/integration/test_disk_over_web_server/test.py index dbcd7cc3c21..fba51949ef0 100644 --- a/tests/integration/test_disk_over_web_server/test.py +++ b/tests/integration/test_disk_over_web_server/test.py @@ -13,7 +13,7 @@ def cluster(): "node1", main_configs=["configs/storage_conf.xml"], with_nginx=True, - allow_analyzer=False, + use_old_analyzer=True, ) cluster.add_instance( "node2", @@ -21,14 +21,14 @@ def cluster(): with_nginx=True, stay_alive=True, with_zookeeper=True, - allow_analyzer=False, + use_old_analyzer=True, ) cluster.add_instance( "node3", main_configs=["configs/storage_conf_web.xml"], with_nginx=True, with_zookeeper=True, - allow_analyzer=False, + use_old_analyzer=True, ) cluster.add_instance( @@ -39,7 +39,6 @@ def cluster(): with_installed_binary=True, image="clickhouse/clickhouse-server", tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, - allow_analyzer=False, ) cluster.start() @@ -322,3 +321,74 @@ def test_replicated_database(cluster): node1.query("DROP DATABASE rdb SYNC") node2.query("DROP DATABASE rdb SYNC") + + +def test_page_cache(cluster): + node = cluster.instances["node2"] + global uuids + assert len(uuids) == 3 + for i in range(3): + node.query( + """ + CREATE TABLE test{} UUID '{}' + (id Int32) ENGINE = MergeTree() ORDER BY id + SETTINGS storage_policy = 'web'; + """.format( + i, uuids[i], i, i + ) + ) + + result1 = node.query( + f"SELECT sum(cityHash64(*)) FROM test{i} SETTINGS use_page_cache_for_disks_without_file_cache=1 -- test cold cache" + ) + result2 = node.query( + f"SELECT sum(cityHash64(*)) FROM test{i} SETTINGS use_page_cache_for_disks_without_file_cache=1 -- test warm cache" + ) + result3 = node.query( + f"SELECT sum(cityHash64(*)) FROM test{i} SETTINGS use_page_cache_for_disks_without_file_cache=0 -- test no cache" + ) + + assert result1 == result3 + assert result2 == result3 + + node.query("SYSTEM FLUSH LOGS") + + def get_profile_events(query_name): + print(f"asdqwe {query_name}") + text = node.query( + f"SELECT ProfileEvents.Names, ProfileEvents.Values FROM system.query_log ARRAY JOIN ProfileEvents WHERE query LIKE '% -- {query_name}' AND type = 'QueryFinish'" + ) + res = {} + for line in text.split("\n"): + if line == "": + continue + name, value = line.split("\t") + print(f"asdqwe {name} = {int(value)}") + res[name] = int(value) + return res + + ev1 = get_profile_events("test cold cache") + assert ev1.get("PageCacheChunkMisses", 0) > 0 + assert ( + ev1.get("DiskConnectionsCreated", 0) + ev1.get("DiskConnectionsReused", 0) + > 0 + ) + + ev2 = get_profile_events("test warm cache") + assert ev2.get("PageCacheChunkDataHits", 0) > 0 + assert ev2.get("PageCacheChunkMisses", 0) == 0 + assert ( + ev2.get("DiskConnectionsCreated", 0) + ev2.get("DiskConnectionsReused", 0) + == 0 + ) + + ev3 = get_profile_events("test no cache") + assert ev3.get("PageCacheChunkDataHits", 0) == 0 + assert ev3.get("PageCacheChunkMisses", 0) == 0 + assert ( + ev3.get("DiskConnectionsCreated", 0) + ev3.get("DiskConnectionsReused", 0) + > 0 + ) + + node.query("DROP TABLE test{} SYNC".format(i)) + print(f"Ok {i}") diff --git a/tests/integration/test_distributed_insert_backward_compatibility/test.py b/tests/integration/test_distributed_insert_backward_compatibility/test.py index 839e1008df1..9e794555d49 100644 --- a/tests/integration/test_distributed_insert_backward_compatibility/test.py +++ b/tests/integration/test_distributed_insert_backward_compatibility/test.py @@ -14,7 +14,6 @@ node_dist = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, ) diff --git a/tests/integration/test_global_overcommit_tracker/configs/global_overcommit_tracker.xml b/tests/integration/test_global_overcommit_tracker/configs/global_overcommit_tracker.xml index a51009542a3..db0926a19ad 100644 --- a/tests/integration/test_global_overcommit_tracker/configs/global_overcommit_tracker.xml +++ b/tests/integration/test_global_overcommit_tracker/configs/global_overcommit_tracker.xml @@ -1,4 +1,4 @@ - 2000000000 + 1500000000 false - \ No newline at end of file + diff --git a/tests/integration/test_global_overcommit_tracker/test.py b/tests/integration/test_global_overcommit_tracker/test.py index 4c60ee8783c..bab95f4706c 100644 --- a/tests/integration/test_global_overcommit_tracker/test.py +++ b/tests/integration/test_global_overcommit_tracker/test.py @@ -22,7 +22,7 @@ def start_cluster(): cluster.shutdown() -GLOBAL_TEST_QUERY_A = "SELECT groupArray(number) FROM numbers(2500000) SETTINGS memory_overcommit_ratio_denominator_for_user=1" +GLOBAL_TEST_QUERY_A = "SELECT groupArray(number) FROM numbers(5000000) SETTINGS memory_overcommit_ratio_denominator_for_user=1" GLOBAL_TEST_QUERY_B = "SELECT groupArray(number) FROM numbers(2500000) SETTINGS memory_overcommit_ratio_denominator_for_user=80000000" @@ -42,11 +42,9 @@ def test_global_overcommit(): responses_A = list() responses_B = list() - for i in range(100): - if i % 2 == 0: - responses_A.append(node.get_query_request(GLOBAL_TEST_QUERY_A, user="A")) - else: - responses_B.append(node.get_query_request(GLOBAL_TEST_QUERY_B, user="B")) + for i in range(50): + responses_A.append(node.get_query_request(GLOBAL_TEST_QUERY_A, user="A")) + responses_B.append(node.get_query_request(GLOBAL_TEST_QUERY_B, user="B")) overcommited_killed = False for response in responses_A: diff --git a/tests/integration/test_groupBitmapAnd_on_distributed/test.py b/tests/integration/test_groupBitmapAnd_on_distributed/test.py index 5119a4e0e36..118cd78105e 100644 --- a/tests/integration/test_groupBitmapAnd_on_distributed/test.py +++ b/tests/integration/test_groupBitmapAnd_on_distributed/test.py @@ -8,13 +8,13 @@ node1 = cluster.add_instance( "node1", main_configs=["configs/clusters.xml"], with_zookeeper=True, - allow_analyzer=False, + use_old_analyzer=True, ) node2 = cluster.add_instance( "node2", main_configs=["configs/clusters.xml"], with_zookeeper=True, - allow_analyzer=False, + use_old_analyzer=True, ) diff --git a/tests/integration/test_groupBitmapAnd_on_distributed/test_groupBitmapAndState_on_distributed_table.py b/tests/integration/test_groupBitmapAnd_on_distributed/test_groupBitmapAndState_on_distributed_table.py index 237acf6b9e0..20ba02e7a80 100644 --- a/tests/integration/test_groupBitmapAnd_on_distributed/test_groupBitmapAndState_on_distributed_table.py +++ b/tests/integration/test_groupBitmapAnd_on_distributed/test_groupBitmapAndState_on_distributed_table.py @@ -8,13 +8,13 @@ node1 = cluster.add_instance( "node1", main_configs=["configs/clusters.xml"], with_zookeeper=True, - allow_analyzer=False, + use_old_analyzer=True, ) node2 = cluster.add_instance( "node2", main_configs=["configs/clusters.xml"], with_zookeeper=True, - allow_analyzer=False, + use_old_analyzer=True, ) diff --git a/tests/integration/test_join_set_family_s3/test.py b/tests/integration/test_join_set_family_s3/test.py index 38b56b7b15b..f0e1480d867 100644 --- a/tests/integration/test_join_set_family_s3/test.py +++ b/tests/integration/test_join_set_family_s3/test.py @@ -93,10 +93,18 @@ def test_join_s3(cluster): "CREATE TABLE testLocalJoin(`id` UInt64, `val` String) ENGINE = Join(ANY, LEFT, id)" ) node.query( - "CREATE TABLE testS3Join(`id` UInt64, `val` String) ENGINE = Join(ANY, LEFT, id) SETTINGS disk='s3'" + "CREATE TABLE testS3Join(`id` UInt64, `val` String) ENGINE = Join(ANY, LEFT, id) SETTINGS disk='s3', join_any_take_last_row = 1" ) node.query("INSERT INTO testLocalJoin VALUES (1, 'a')") + for i in range(1, 10): + c = chr(ord("a") + i) + node.query(f"INSERT INTO testLocalJoin VALUES (1, '{c}')") + + # because of `join_any_take_last_row = 1` we expect the last row with 'a' value + for i in range(1, 10): + c = chr(ord("a") + i) + node.query(f"INSERT INTO testS3Join VALUES (1, '{c}')") node.query("INSERT INTO testS3Join VALUES (1, 'a')") assert ( @@ -105,7 +113,7 @@ def test_join_s3(cluster): ) == "\t\na\ta\n\t\n" ) - assert_objects_count(cluster, 1) + assert_objects_count(cluster, 10) node.query("INSERT INTO testLocalJoin VALUES (2, 'b')") node.query("INSERT INTO testS3Join VALUES (2, 'b')") @@ -116,7 +124,7 @@ def test_join_s3(cluster): ) == "\t\na\ta\nb\tb\n" ) - assert_objects_count(cluster, 2) + assert_objects_count(cluster, 11) node.restart_clickhouse() assert ( diff --git a/tests/integration/test_merge_tree_load_marks/__init__.py b/tests/integration/test_merge_tree_load_marks/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_merge_tree_load_marks/configs/config.xml b/tests/integration/test_merge_tree_load_marks/configs/config.xml new file mode 100644 index 00000000000..1c9ee8d698f --- /dev/null +++ b/tests/integration/test_merge_tree_load_marks/configs/config.xml @@ -0,0 +1,12 @@ + + + system + text_log
+ 7500 + 1048576 + 8192 + 524288 + false + test + + diff --git a/tests/integration/test_merge_tree_load_marks/test.py b/tests/integration/test_merge_tree_load_marks/test.py new file mode 100644 index 00000000000..b066b2a6ec0 --- /dev/null +++ b/tests/integration/test_merge_tree_load_marks/test.py @@ -0,0 +1,62 @@ +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node = cluster.add_instance( + "node", + main_configs=["configs/config.xml"], +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +# This test is bad and it should be a functional test but S3 metrics +# are accounted incorrectly for merges in part_log and query_log. +# Also we have text_log with level 'trace' in functional tests +# but this test requeires text_log with level 'test'. + + +@pytest.mark.parametrize("min_bytes_for_wide_part", [0, 1000000000]) +def test_merge_load_marks(started_cluster, min_bytes_for_wide_part): + node.query( + f""" + DROP TABLE IF EXISTS t_load_marks; + + CREATE TABLE t_load_marks (a UInt64, b UInt64) + ENGINE = MergeTree ORDER BY a + SETTINGS min_bytes_for_wide_part = {min_bytes_for_wide_part}; + + INSERT INTO t_load_marks SELECT number, number FROM numbers(1000); + INSERT INTO t_load_marks SELECT number, number FROM numbers(1000); + + OPTIMIZE TABLE t_load_marks FINAL; + SYSTEM FLUSH LOGS; + """ + ) + + uuid = node.query( + "SELECT uuid FROM system.tables WHERE table = 't_prewarm_merge'" + ).strip() + + result = node.query( + f""" + SELECT count() + FROM system.text_log + WHERE (query_id LIKE '%{uuid}::all_1_2_1%') AND (message LIKE '%Loading marks%') + """ + ).strip() + + result = int(result) + + is_wide = min_bytes_for_wide_part == 0 + not_loaded = result == 0 + + assert is_wide == not_loaded diff --git a/tests/integration/test_merges_memory_limit/test.py b/tests/integration/test_merges_memory_limit/test.py index ca2b0bc5429..fb592dd8a06 100644 --- a/tests/integration/test_merges_memory_limit/test.py +++ b/tests/integration/test_merges_memory_limit/test.py @@ -17,18 +17,24 @@ def start_cluster(): def test_memory_limit_success(): + if node.is_built_with_thread_sanitizer(): + pytest.skip( + "tsan build is skipped because it slowly merges the parts, " + "rather than failing over the memory limit" + ) + node.query( - "CREATE TABLE test_merge_oom ENGINE=AggregatingMergeTree ORDER BY id EMPTY AS SELECT number%1024 AS id, arrayReduce( 'groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(20000)" + "CREATE TABLE test_merge_oom ENGINE=AggregatingMergeTree ORDER BY id EMPTY AS SELECT number%1024 AS id, arrayReduce('groupArrayState', arrayMap(x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(20000)" ) node.query("SYSTEM STOP MERGES test_merge_oom") node.query( - "INSERT INTO test_merge_oom SELECT number%1024 AS id, arrayReduce( 'groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(3000)" + "INSERT INTO test_merge_oom SELECT number%1024 AS id, arrayReduce('groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(3000)" ) node.query( - "INSERT INTO test_merge_oom SELECT number%1024 AS id, arrayReduce( 'groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(3000)" + "INSERT INTO test_merge_oom SELECT number%1024 AS id, arrayReduce('groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(3000)" ) node.query( - "INSERT INTO test_merge_oom SELECT number%1024 AS id, arrayReduce( 'groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(3000)" + "INSERT INTO test_merge_oom SELECT number%1024 AS id, arrayReduce('groupArrayState', arrayMap( x-> randomPrintableASCII(100), range(8192))) fat_state FROM numbers(3000)" ) _, error = node.query_and_get_answer_with_error( diff --git a/tests/integration/test_old_versions/test.py b/tests/integration/test_old_versions/test.py index 43f91b7d265..a5e62a380bf 100644 --- a/tests/integration/test_old_versions/test.py +++ b/tests/integration/test_old_versions/test.py @@ -10,7 +10,6 @@ node_oldest = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, main_configs=["configs/config.d/test_cluster.xml"], - allow_analyzer=False, ) old_nodes = [node_oldest] new_node = cluster.add_instance("node_new") diff --git a/tests/integration/test_polymorphic_parts/test.py b/tests/integration/test_polymorphic_parts/test.py index b91a72c5534..2b30170b203 100644 --- a/tests/integration/test_polymorphic_parts/test.py +++ b/tests/integration/test_polymorphic_parts/test.py @@ -369,7 +369,6 @@ node7 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, ) node8 = cluster.add_instance( "node8", diff --git a/tests/integration/test_quorum_inserts/test.py b/tests/integration/test_quorum_inserts/test.py index 1276a6079f0..b842a54741e 100644 --- a/tests/integration/test_quorum_inserts/test.py +++ b/tests/integration/test_quorum_inserts/test.py @@ -1,7 +1,9 @@ +import concurrent import time import pytest from helpers.cluster import ClickHouseCluster +from helpers.network import PartitionManager from helpers.test_tools import TSV cluster = ClickHouseCluster(__file__) @@ -361,3 +363,81 @@ def test_insert_quorum_with_ttl(started_cluster): ) zero.query("DROP TABLE IF EXISTS test_insert_quorum_with_ttl ON CLUSTER cluster") + + +def test_insert_quorum_with_keeper_loss_connection(): + zero.query( + "DROP TABLE IF EXISTS test_insert_quorum_with_keeper_fail ON CLUSTER cluster" + ) + create_query = ( + "CREATE TABLE test_insert_quorum_with_keeper_loss" + "(a Int8, d Date) " + "Engine = ReplicatedMergeTree('/clickhouse/tables/{table}', '{replica}') " + "ORDER BY a " + ) + + zero.query(create_query) + first.query(create_query) + + first.query("SYSTEM STOP FETCHES test_insert_quorum_with_keeper_loss") + + zero.query("SYSTEM ENABLE FAILPOINT replicated_merge_tree_commit_zk_fail_after_op") + zero.query("SYSTEM ENABLE FAILPOINT replicated_merge_tree_insert_retry_pause") + + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + insert_future = executor.submit( + lambda: zero.query( + "INSERT INTO test_insert_quorum_with_keeper_loss(a,d) VALUES(1, '2011-01-01')", + settings={"insert_quorum_timeout": 150000}, + ) + ) + + pm = PartitionManager() + pm.drop_instance_zk_connections(zero) + + retries = 0 + zk = cluster.get_kazoo_client("zoo1") + while True: + if ( + zk.exists( + "/clickhouse/tables/test_insert_quorum_with_keeper_loss/replicas/zero/is_active" + ) + is None + ): + break + print("replica is still active") + time.sleep(1) + retries += 1 + if retries == 120: + raise Exception("Can not wait cluster replica inactive") + + first.query("SYSTEM ENABLE FAILPOINT finish_set_quorum_failed_parts") + quorum_fail_future = executor.submit( + lambda: first.query( + "SYSTEM WAIT FAILPOINT finish_set_quorum_failed_parts", timeout=300 + ) + ) + first.query("SYSTEM START FETCHES test_insert_quorum_with_keeper_loss") + + concurrent.futures.wait([quorum_fail_future]) + + assert quorum_fail_future.exception() is None + + zero.query("SYSTEM ENABLE FAILPOINT finish_clean_quorum_failed_parts") + clean_quorum_fail_parts_future = executor.submit( + lambda: first.query( + "SYSTEM WAIT FAILPOINT finish_clean_quorum_failed_parts", timeout=300 + ) + ) + pm.restore_instance_zk_connections(zero) + concurrent.futures.wait([clean_quorum_fail_parts_future]) + + assert clean_quorum_fail_parts_future.exception() is None + + zero.query("SYSTEM DISABLE FAILPOINT replicated_merge_tree_insert_retry_pause") + concurrent.futures.wait([insert_future]) + assert insert_future.exception() is not None + assert not zero.contains_in_log("LOGICAL_ERROR") + assert zero.contains_in_log( + "fails to commit and will not retry or clean garbage" + ) diff --git a/tests/integration/test_reload_certificate/configs/WithChain.crt b/tests/integration/test_reload_certificate/configs/WithChain.crt new file mode 100644 index 00000000000..c75ee533f26 --- /dev/null +++ b/tests/integration/test_reload_certificate/configs/WithChain.crt @@ -0,0 +1,43 @@ +-----BEGIN CERTIFICATE----- +MIIDkzCCAnugAwIBAgIUSChEeHqJus9jzKmD/L3Tw0x4OwcwDQYJKoZIhvcNAQEL +BQAwQTEaMBgGA1UEAwwRZGVtby5tbG9wc2h1Yi5jb20xCzAJBgNVBAYTAlVTMRYw +FAYDVQQHDA1TYW4gRnJhbnNpc2NvMCAXDTI0MDMyMDE5MTE1OVoYDzIxMjQwMjI1 +MTkxMTU5WjBxMQswCQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQG +A1UEBwwNU2FuIEZyYW5zaXNjbzEPMA0GA1UECgwGRm9vQmFyMRAwDgYDVQQLDAdG +b28gQmFyMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IB +DwAwggEKAoIBAQDM//0s6AX988J86HzxX95irGa9cJIfY24UDBXVkO3gREiwj1Uf +bpvpxCcwADMuFdggsJlppWa3q+PNJ/eoVwdl3gG0WXaZp1rcuv6ltxdQAUtgfMAb +5p7HwsO7rCTGJBwa62Jg+E79j+V8rZWfaJRfNtY0p7eauWIrqLA0Gyse+lRayPHI +hsR9+0qedF+qziFpbNxpW8DHrpIrLb8LEao1BCYQ44koBXjkrXeR6OidXw/gek8+ +9M2GLxy6ubQ7hrcYwVWpFOKLLZLmyYDgescM6AIU904o1bN0yJ5rM7a1+f150qp6 +ttZlya0sJH0lm3gzEsqb6Fbh+Dw/9Lsp66sDAgMBAAGjUTBPMB8GA1UdIwQYMBaA +FMjyDUyXujACeQa2G+4I8ic5HVBiMAkGA1UdEwQCMAAwCwYDVR0PBAQDAgTwMBQG +A1UdEQQNMAuCCWxvY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEABEWEaVDqv9Za +bFpD/JEMIFVqe5ANKAnbbff0/vFJQ7yFmsL8/G4bbdd8uidRgE4WgoeNUscCnfk+ +kjb1vFjvY4/cBYITn41Pz5I7lQMH+BaR5gHb1oJVlJavQ2vsaeMuyDJaNmumejmU +YnoKZRwwb6SCXujS2MKgKl+jL5OkZk60i+nJhIXfxwMNmlvtqADSU5Z3VMagq8hj +DnEsxTz8PptuVaLVT4kcZm9gZpDEW2KPMZhNCv/g7EzQv8r3WnFGqumMGBO82ZE0 +mUh/Chrhss/meVK0FqTTBjOlex7R0GiJBCDfZGYTWIVdND4ICdZ1OpGWid5CXbfQ +sWBrbBaEyw== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDYzCCAkugAwIBAgIUR0PldYYSk3DAifgb600kvqjaCAIwDQYJKoZIhvcNAQEL +BQAwQTEaMBgGA1UEAwwRZGVtby5tbG9wc2h1Yi5jb20xCzAJBgNVBAYTAlVTMRYw +FAYDVQQHDA1TYW4gRnJhbnNpc2NvMB4XDTI0MDMyMDE5MTEzMVoXDTI1MDMxMTE5 +MTEzMVowQTEaMBgGA1UEAwwRZGVtby5tbG9wc2h1Yi5jb20xCzAJBgNVBAYTAlVT +MRYwFAYDVQQHDA1TYW4gRnJhbnNpc2NvMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAqL3k6Pexu1KR8fc84fRmu0hYon7+xOPmtFSzspeN+DJNe5oDO6x0 +RzTQkgtDoxTcq32O290r3uURnDmnvNubz5yTpM1Zcz/kuSNpHLJh4yyZsXRsB21v +lb3bhjqyn6rkfoQzIMekt7clPQS0dWdU2T+lwn6XBVShOyB/W7ysP309ofQGXV+T +VFyU+lgZc2WjK6611QDCpTXgRc/UKUfU5460BnTCylP6jzBOWBZb8FX6dYBzS4U2 +yISvOXagxJVruoWjscc35ln6HBQ8bu/fI8Q0n1/ROlm785Bsd/LpVw465kklwQwS +FY3FQkiedD1fyszXO4Yq5PARw54AGKbAyQIDAQABo1MwUTAdBgNVHQ4EFgQUyPIN +TJe6MAJ5BrYb7gjyJzkdUGIwHwYDVR0jBBgwFoAUyPINTJe6MAJ5BrYb7gjyJzkd +UGIwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAYyl27sxwrjz5 +qcqwbLYv2eFIR6VZzvHfknSK1ht1jzMkXxvAOjETbYwqwWquWwMHOD2X4q5d57nu +qYAE9YE27HFzknPQkDdzwJ0u4dKi28PK8tM6dqDK46LSal/MEUxXGuzW3TRyJXrl +lPi+Wh6gZRRiANJ+giEwvQ+8k6eURHrhtL7yZqT+swi+jP4h6S4mmHmsaOj4VoP/ +NCFoRZud5SCd7RZV+fzNfxhLHI9I2c2gFycBDZOEdlrIZHM6EoaDb3i9kDVbnZqG +Zj/+k/NwCKg5UiDap6Z7Xj7w0chSppg3DMcsxGeQ9vQcMtydNu5fSK4CozNqxObb +hGBJrQylAw== +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_certificate/configs/WithChain.key b/tests/integration/test_reload_certificate/configs/WithChain.key new file mode 100644 index 00000000000..eb55458b143 --- /dev/null +++ b/tests/integration/test_reload_certificate/configs/WithChain.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpQIBAAKCAQEAzP/9LOgF/fPCfOh88V/eYqxmvXCSH2NuFAwV1ZDt4ERIsI9V +H26b6cQnMAAzLhXYILCZaaVmt6vjzSf3qFcHZd4BtFl2mada3Lr+pbcXUAFLYHzA +G+aex8LDu6wkxiQcGutiYPhO/Y/lfK2Vn2iUXzbWNKe3mrliK6iwNBsrHvpUWsjx +yIbEfftKnnRfqs4haWzcaVvAx66SKy2/CxGqNQQmEOOJKAV45K13kejonV8P4HpP +PvTNhi8curm0O4a3GMFVqRTiiy2S5smA4HrHDOgCFPdOKNWzdMieazO2tfn9edKq +erbWZcmtLCR9JZt4MxLKm+hW4fg8P/S7KeurAwIDAQABAoIBAQDIT6LtzFJ+kT+L +mgjsOocs19UUavj9XsTjJeP36UPnDhIRJB1DN2t2Astf5fIcqA+l8aoWRx6Vfop7 +nwAqJ8/8w+/acEipX5Qzdyz4dG19Kaajw4JcQP8JptBng4/zZLlpvHNG2ZslvQO6 +zksTlrbUStsIXJHVyCubCZwTbjC2dJlc97705DZAk/8L7FkljmyJws2xwQZlxdQG +FZ+8IrAqpWJC55EPfKo6+QsKEc9hh4N/MQW483V5yCuw9dLShcEMDGuxcHGFHif8 +BrwImih0rIwj9tTDY9pw6aJ5+80tVStNLDk+1eQRME6Fy/c7RG/sm/lj+P0YOI7F +jH4wyXVRAoGBAOc+rFNpM/CgRZD4VXQ+gV1+PLdaqUUU3UXAPfNWkrMIr0u4+tmg +OMMkXaM7B/ps7o5+Rcj+tO1VAvl6m7uM376szC3Ex8JA0XBDmJrJKfEQCHkzvkdf +wywVBeAR4f3D2+9Meh1XSNRqGU+Anb48neTyVYzPNIoK8ZmtZID49GfVAoGBAOLy +EX1TT1xwE/VwDy5BeJQzyZ+xwevie/960RIYooLeb31NLhOcX142b8U7XRMtiqdd +wfsT5SbjnrATBponKZELO7LwE+Z4djo2+O6JZjYB5/t/Z6r7qfOaTTlJEl8VJKo4 +F+qAsqKo0Q9EpkRUeNdcOjDzkuEikw9IlhS0VEt3AoGAWLHoRQH4AxZmOGmX1UNY +OTT/MtCaVj3fdS58VIZjNDpjiibESI601txu8fnlYH9BrPPv7l0LpnBR+MC3VON+ +ulLq6a8tc2uLKYUz1kLMTIL6zQo0tImdgZ36p+wUA1KJXCq4N+LPs3GSjbTmTB5R +7Yuplp2vKDd0XZ5tCy7yB5UCgYEA3ppoE1DTSC1bNmSLT2jCuEOv4ic+unw1+lti +lWh6hvldzD8XEf9RAB1PNtvKqNQD67SoX/mczK956OVQlYYSXIXzMcoRCwBfnyxq +sbct/Y2TGXpXmjwt8JcKZkVJcuBPTXOl6cwA7FHAdkR0/hMJUNzS608PZCtAqj4d +kANtp3MCgYEA3plv6RYYDZsUdJmIcdIvr/LxWTSqYf37LQM//k+OXo+zWMxgnUNv +AEqDlNWP+bw3yJlU1bQS2o1Z+hKzDgqDZtaVVVDgdVsUaDSW0EsJpedWXoGqJdrw +yxhB7RYi1tQsXHbR1iyT5hH0ZlV7s0XIKRU4U8MP05av099+++YKhks= +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/test_reload_certificate/test.py b/tests/integration/test_reload_certificate/test.py index 1718e440629..f0efc4e0bbd 100644 --- a/tests/integration/test_reload_certificate/test.py +++ b/tests/integration/test_reload_certificate/test.py @@ -13,6 +13,8 @@ node = cluster.add_instance( "configs/second.key", "configs/ECcert.crt", "configs/ECcert.key", + "configs/WithChain.crt", + "configs/WithChain.key", "configs/WithPassPhrase.crt", "configs/WithPassPhrase.key", "configs/cert.xml", @@ -158,3 +160,18 @@ def test_cert_with_pass_phrase(): check_certificate_switch( "first", "WithPassPhrase", pass_phrase_second=pass_phrase_for_cert ) + + +def test_chain_reload(): + """Check cert chain reload""" + check_certificate_switch("first", "WithChain") + assert ( + node.exec_in_container( + [ + "bash", + "-c", + "openssl s_client -showcerts -servername localhost -connect localhost:8443 /dev/null | grep 'BEGIN CERTIFICATE' | wc -l", + ] + ) + == "2\n" + ) diff --git a/tests/integration/test_replicated_merge_tree_compatibility/test.py b/tests/integration/test_replicated_merge_tree_compatibility/test.py index 25cf3caa50c..a70f3234c1e 100644 --- a/tests/integration/test_replicated_merge_tree_compatibility/test.py +++ b/tests/integration/test_replicated_merge_tree_compatibility/test.py @@ -9,7 +9,6 @@ node1 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, ) node2 = cluster.add_instance( "node2", @@ -18,7 +17,6 @@ node2 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, stay_alive=True, with_installed_binary=True, - allow_analyzer=False, ) diff --git a/tests/integration/test_runtime_configurable_cache_size/test.py b/tests/integration/test_runtime_configurable_cache_size/test.py index 844105efd23..f761005f297 100644 --- a/tests/integration/test_runtime_configurable_cache_size/test.py +++ b/tests/integration/test_runtime_configurable_cache_size/test.py @@ -1,7 +1,6 @@ import os -import pytest -import shutil import time +import pytest from helpers.cluster import ClickHouseCluster # Tests that sizes of in-memory caches (mark / uncompressed / index mark / index uncompressed / mmapped file / query cache) can be changed @@ -101,9 +100,10 @@ def test_query_cache_size_is_runtime_configurable(start_cluster): node.query("SELECT 2 SETTINGS use_query_cache = 1, query_cache_ttl = 1") node.query("SELECT 3 SETTINGS use_query_cache = 1, query_cache_ttl = 1") - res = node.query_with_retry( + time.sleep(2) + node.query("SYSTEM RELOAD ASYNCHRONOUS METRICS") + res = node.query( "SELECT value FROM system.asynchronous_metrics WHERE metric = 'QueryCacheEntries'", - check_callback=lambda result: result == "2\n", ) assert res == "2\n" @@ -116,9 +116,10 @@ def test_query_cache_size_is_runtime_configurable(start_cluster): node.query("SYSTEM RELOAD CONFIG") # check that eviction worked as expected - res = node.query_with_retry( + time.sleep(2) + node.query("SYSTEM RELOAD ASYNCHRONOUS METRICS") + res = node.query( "SELECT value FROM system.asynchronous_metrics WHERE metric = 'QueryCacheEntries'", - check_callback=lambda result: result == "2\n", ) assert ( res == "2\n" @@ -132,9 +133,10 @@ def test_query_cache_size_is_runtime_configurable(start_cluster): node.query("SELECT 4 SETTINGS use_query_cache = 1, query_cache_ttl = 1") node.query("SELECT 5 SETTINGS use_query_cache = 1, query_cache_ttl = 1") - res = node.query_with_retry( + time.sleep(2) + node.query("SYSTEM RELOAD ASYNCHRONOUS METRICS") + res = node.query( "SELECT value FROM system.asynchronous_metrics WHERE metric = 'QueryCacheEntries'", - check_callback=lambda result: result == "1\n", ) assert res == "1\n" diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py index a7abd840834..c7893c3e643 100644 --- a/tests/integration/test_storage_s3_queue/test.py +++ b/tests/integration/test_storage_s3_queue/test.py @@ -108,7 +108,7 @@ def started_cluster(): tag="23.12", stay_alive=True, with_installed_binary=True, - allow_analyzer=False, + use_old_analyzer=True, ) logging.info("Starting cluster...") diff --git a/tests/integration/test_storage_url_http_headers/http_headers_echo_server.py b/tests/integration/test_storage_url_http_headers/http_headers_echo_server.py index b1a3f6777b1..3c62112a7d3 100644 --- a/tests/integration/test_storage_url_http_headers/http_headers_echo_server.py +++ b/tests/integration/test_storage_url_http_headers/http_headers_echo_server.py @@ -1,6 +1,8 @@ import http.server +import sys +import json -RESULT_PATH = "/headers.txt" +RESULT_PATH = "/echo_server_headers.txt" class RequestHandler(http.server.BaseHTTPRequestHandler): @@ -8,6 +10,28 @@ class RequestHandler(http.server.BaseHTTPRequestHandler): with open(RESULT_PATH, "w") as f: f.write(self.headers.as_string()) + def do_GET(self): + if self.path == "/": + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write(b'{"status":"ok"}') + if self.path == "/sample-data": + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.end_headers() + sample_data = [ + { + "title": "ClickHouse Newsletter June 2022: Materialized, but still real-time", + "theme": "Newsletter", + }, + { + "title": "ClickHouse Over the Years with Benchmarks", + "theme": "ClickHouse Journey", + }, + ] + self.wfile.write(bytes(json.dumps(sample_data), "UTF-8")) + def do_POST(self): self.rfile.read1() self.send_response(200) @@ -16,15 +40,16 @@ class RequestHandler(http.server.BaseHTTPRequestHandler): if __name__ == "__main__": - with open(RESULT_PATH, "w") as f: - f.write("") - httpd = http.server.HTTPServer( + host = sys.argv[1] + port = int(sys.argv[2]) + httpd = http.server.ThreadingHTTPServer( ( - "localhost", - 8000, + host, + port, ), RequestHandler, ) + try: httpd.serve_forever() finally: diff --git a/tests/integration/test_storage_url_http_headers/redirect_server.py b/tests/integration/test_storage_url_http_headers/redirect_server.py new file mode 100644 index 00000000000..b1d92d0cd4e --- /dev/null +++ b/tests/integration/test_storage_url_http_headers/redirect_server.py @@ -0,0 +1,46 @@ +import http.server +import sys + +REDIRECT_HOST = "" +REDIRECT_PORT = 0 + +RESULT_PATH = "/redirect_server_headers.txt" + + +class RequestHandler(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): + with open(RESULT_PATH, "w") as f: + f.write(self.headers.as_string()) + + def do_GET(self): + if self.path == "/": + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write(b'{"status":"ok"}') + else: + global REDIRECT_HOST, REDIRECT_PORT + self.send_response(302) + target_location = f"http://{REDIRECT_HOST}:{REDIRECT_PORT}{self.path}" + self.send_header("Location", target_location) + self.end_headers() + self.wfile.write(b'{"status":"redirected"}') + + +if __name__ == "__main__": + host = sys.argv[1] + port = int(sys.argv[2]) + REDIRECT_HOST = sys.argv[3] + REDIRECT_PORT = int(sys.argv[4]) + httpd = http.server.ThreadingHTTPServer( + ( + host, + port, + ), + RequestHandler, + ) + + try: + httpd.serve_forever() + finally: + httpd.server_close() diff --git a/tests/integration/test_storage_url_http_headers/test.py b/tests/integration/test_storage_url_http_headers/test.py index 3bbf5ec81c9..8fc08ec5c9d 100644 --- a/tests/integration/test_storage_url_http_headers/test.py +++ b/tests/integration/test_storage_url_http_headers/test.py @@ -1,8 +1,7 @@ import pytest import os -import time - from . import http_headers_echo_server +from . import redirect_server from helpers.cluster import ClickHouseCluster @@ -10,31 +9,37 @@ cluster = ClickHouseCluster(__file__) server = cluster.add_instance("node") -def run_echo_server(): +def run_server(container_id, file_name, hostname, port, *args): script_dir = os.path.dirname(os.path.realpath(__file__)) - server.copy_file_to_container( - os.path.join(script_dir, "http_headers_echo_server.py"), - "/http_headers_echo_server.py", + cluster.copy_file_to_container( + container_id, + os.path.join(script_dir, file_name), + f"/{file_name}", ) - server.exec_in_container( + cmd_args = [hostname, port] + list(args) + cmd_args_val = " ".join([str(x) for x in cmd_args]) + + cluster.exec_in_container( + container_id, [ "bash", "-c", - "python3 /http_headers_echo_server.py > /http_headers_echo.server.log 2>&1", + f"python3 /{file_name} {cmd_args_val} > {file_name}.log 2>&1", ], detach=True, user="root", ) for _ in range(0, 10): - ping_response = server.exec_in_container( - ["curl", "-s", f"http://localhost:8000/"], + ping_response = cluster.exec_in_container( + container_id, + ["curl", "-s", f"http://{hostname}:{port}/"], nothrow=True, ) - if "html" in ping_response: + if '{"status":"ok"}' in ping_response: return print(ping_response) @@ -42,11 +47,23 @@ def run_echo_server(): raise Exception("Echo server is not responding") +def run_echo_server(): + container_id = cluster.get_container_id("node") + run_server(container_id, "http_headers_echo_server.py", "localhost", 8000) + + +def run_redirect_server(): + container_id = cluster.get_container_id("node") + run_server(container_id, "redirect_server.py", "localhost", 8080, "localhost", 8000) + + @pytest.fixture(scope="module") def started_cluster(): try: cluster.start() + run_redirect_server() run_echo_server() + yield cluster finally: cluster.shutdown() @@ -64,3 +81,35 @@ def test_storage_url_http_headers(started_cluster): print(result) assert "X-My-Custom-Header: test-header" in result + + +def test_storage_url_redirected_headers(started_cluster): + query = """ + SELECT + title::String as title, + theme::String as theme + FROM + url('http://127.0.0.1:8080/sample-data', 'JSONEachRow', 'title String, theme String') + SETTINGS http_max_tries=2, max_http_get_redirects=2 + """ + + result = server.query(query) + assert 2 == len(result.strip().split("\n")) + + result_redirect = server.exec_in_container( + ["cat", redirect_server.RESULT_PATH], user="root" + ) + + print(result_redirect) + + assert "Host: 127.0.0.1" in result_redirect + assert "Host: localhost" not in result_redirect + + result = server.exec_in_container( + ["cat", http_headers_echo_server.RESULT_PATH], user="root" + ) + + print(result) + + assert "Host: 127.0.0.1" not in result + assert "Host: localhost" in result diff --git a/tests/integration/test_ttl_replicated/test.py b/tests/integration/test_ttl_replicated/test.py index b20b761ef47..f944adbea41 100644 --- a/tests/integration/test_ttl_replicated/test.py +++ b/tests/integration/test_ttl_replicated/test.py @@ -23,7 +23,6 @@ node4 = cluster.add_instance( main_configs=[ "configs/compat.xml", ], - allow_analyzer=False, ) node5 = cluster.add_instance( @@ -36,7 +35,6 @@ node5 = cluster.add_instance( main_configs=[ "configs/compat.xml", ], - allow_analyzer=False, ) node6 = cluster.add_instance( "node6", @@ -48,7 +46,6 @@ node6 = cluster.add_instance( main_configs=[ "configs/compat.xml", ], - allow_analyzer=False, ) diff --git a/tests/integration/test_undrop_query/configs/with_delay_config.xml b/tests/integration/test_undrop_query/configs/with_delay_config.xml index 5461003991d..5e6de6b7567 100644 --- a/tests/integration/test_undrop_query/configs/with_delay_config.xml +++ b/tests/integration/test_undrop_query/configs/with_delay_config.xml @@ -1,3 +1,3 @@ - 3 + 20 diff --git a/tests/integration/test_undrop_query/test.py b/tests/integration/test_undrop_query/test.py index 9eac01c5219..1e77fc8db04 100644 --- a/tests/integration/test_undrop_query/test.py +++ b/tests/integration/test_undrop_query/test.py @@ -1,5 +1,6 @@ import pytest import uuid +import logging import time from helpers.cluster import ClickHouseCluster @@ -20,29 +21,28 @@ def started_cluster(): def test_undrop_drop_and_undrop_loop(started_cluster): - # create, drop, undrop, drop, undrop table 5 times - for _ in range(5): - table_uuid = str(uuid.uuid1()) - table = f"test_undrop_loop" + uuid_list = [] + + for i in range(4): + table_uuid = uuid.uuid1().__str__() + uuid_list.append(table_uuid) + logging.info(f"table_uuid: {table_uuid}") + node.query( - f"CREATE TABLE {table} " - f"UUID '{table_uuid}' (id Int32) " - f"Engine=MergeTree() ORDER BY id" + f"CREATE TABLE test_undrop_{i} UUID '{table_uuid}' (id Int32) ENGINE = MergeTree() ORDER BY id;" ) - node.query(f"DROP TABLE {table}") - node.query(f"UNDROP TABLE {table} UUID '{table_uuid}'") + node.query(f"DROP TABLE test_undrop_{i};") - node.query(f"DROP TABLE {table}") - # database_atomic_delay_before_drop_table_sec=3 - time.sleep(6) - - """ - Expect two things: - 1. Table is dropped - UNKNOWN_TABLE in error - 2. Table in process of dropping - Return code: 60. - The drop task of table ... (uuid) is in progress, - has been dropped or the database engine doesn't support it - """ - error = node.query_and_get_error(f"UNDROP TABLE {table} UUID '{table_uuid}'") - assert "UNKNOWN_TABLE" in error or "The drop task of table" in error + for i in range(4): + if ( + i >= 3 + ): # First 3 tables are undropped after 0, 5 and 10 seconds. Fourth is undropped after 21 seconds + time.sleep(6) + error = node.query_and_get_error( + f"UNDROP TABLE test_undrop_loop_{i} UUID '{uuid_list[i]}';" + ) + assert "UNKNOWN_TABLE" in error + else: + node.query(f"UNDROP TABLE test_undrop_loop_{i} UUID '{uuid_list[i]}';") + time.sleep(5) diff --git a/tests/integration/test_version_update/test.py b/tests/integration/test_version_update/test.py index ab3eb1860f3..b386a79c932 100644 --- a/tests/integration/test_version_update/test.py +++ b/tests/integration/test_version_update/test.py @@ -14,7 +14,6 @@ node2 = cluster.add_instance( tag=CLICKHOUSE_CI_MIN_TESTED_VERSION, with_installed_binary=True, stay_alive=True, - allow_analyzer=False, ) diff --git a/tests/integration/test_version_update_after_mutation/test.py b/tests/integration/test_version_update_after_mutation/test.py index 90f8d283a6a..9365498f89d 100644 --- a/tests/integration/test_version_update_after_mutation/test.py +++ b/tests/integration/test_version_update_after_mutation/test.py @@ -16,7 +16,6 @@ node1 = cluster.add_instance( main_configs=[ "configs/compat.xml", ], - allow_analyzer=False, ) node2 = cluster.add_instance( "node2", @@ -28,7 +27,6 @@ node2 = cluster.add_instance( main_configs=[ "configs/compat.xml", ], - allow_analyzer=False, ) node3 = cluster.add_instance( "node3", @@ -40,7 +38,6 @@ node3 = cluster.add_instance( main_configs=[ "configs/compat.xml", ], - allow_analyzer=False, ) diff --git a/tests/performance/aggregating_merge_tree_simple_aggregate_function_string.xml b/tests/performance/aggregating_merge_tree_simple_aggregate_function_string.xml index 0c93b4745cf..5845d1dd0d2 100644 --- a/tests/performance/aggregating_merge_tree_simple_aggregate_function_string.xml +++ b/tests/performance/aggregating_merge_tree_simple_aggregate_function_string.xml @@ -1,4 +1,7 @@ + + 1 + CREATE TABLE bench ENGINE = AggregatingMergeTree() diff --git a/tests/performance/function_in.xml b/tests/performance/function_in.xml deleted file mode 100644 index af4f8737ba7..00000000000 --- a/tests/performance/function_in.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - 8 - 1 - - - - CREATE TABLE t_nullable - ( - key_string1 Nullable(String), - key_string2 Nullable(String), - key_string3 Nullable(String), - key_int64_1 Nullable(Int64), - key_int64_2 Nullable(Int64), - key_int64_3 Nullable(Int64), - key_int64_4 Nullable(Int64), - key_int64_5 Nullable(Int64), - m1 Int64, - m2 Int64 - ) - ENGINE = Memory - - insert into t_nullable select ['aaaaaa','bbaaaa','ccaaaa','ddaaaa'][number % 101 + 1], ['aa','bb','cc','dd'][number % 100 + 1], ['aa','bb','cc','dd'][number % 102 + 1], number%10+1, number%10+2, number%10+3, number%10+4,number%10+5, number%6000+1, number%5000+2 from numbers_mt(30000000) - select * from t_nullable where key_string1 in ('aaaaaa') format Null SETTINGS allow_experimental_analyzer=1 - select * from t_nullable where key_string2 in ('3') format Null SETTINGS allow_experimental_analyzer=1 - drop table if exists t_nullable - - diff --git a/tests/performance/function_tokens.xml b/tests/performance/function_tokens.xml new file mode 100644 index 00000000000..63b72f83df3 --- /dev/null +++ b/tests/performance/function_tokens.xml @@ -0,0 +1,3 @@ + + with 'Many years later as he faced the firing squad, Colonel Aureliano Buendia was to remember that distant afternoon when his father took him to discover ice.' as s select splitByChar(' ', materialize(s)) as w from numbers(1000000) + diff --git a/tests/performance/scripts/eqmed.sql b/tests/performance/scripts/eqmed.sql index 94e6733a3d7..372534f6f48 100644 --- a/tests/performance/scripts/eqmed.sql +++ b/tests/performance/scripts/eqmed.sql @@ -67,4 +67,5 @@ from select throwIf(uniq((test, query)) != 1) from table ) check_single_query -- this subselect checks that there is only one query in the input table; -- written this way so that it is not optimized away (#10523) +SETTINGS allow_experimental_analyzer = 0 ; diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql index dfa11dfcb13..74608360b9c 100644 --- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql +++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql @@ -203,7 +203,7 @@ select with 0.16 as threshold select 'noisy Trace messages', - greatest(coalesce(((select message_format_string, count() from logs where level = 'Trace' and message_format_string not in ('Access granted: {}{}', '{} -> {}') + greatest(coalesce(((select message_format_string, count() from logs where level = 'Trace' and message_format_string not in ('Access granted: {}{}', '{} -> {}', 'Query {} to stage {}{}', 'Query {} from stage {} to stage {}{}') group by message_format_string order by count() desc limit 1) as top_message).2, 0) / (select count() from logs), threshold) as r, r <= threshold ? '' : top_message.1; diff --git a/tests/queries/0_stateless/00300_csv.reference b/tests/queries/0_stateless/00300_csv.reference index 42cd22078c4..e7966a9e8d9 100644 --- a/tests/queries/0_stateless/00300_csv.reference +++ b/tests/queries/0_stateless/00300_csv.reference @@ -1,11 +1,11 @@ -"Hello, ""World""",123,"[1,2,3]",456,"['abc','def']","Newline +"Hello, ""World""",123,"[1,2,3]","(456,['abc','def'])","Newline here" "x","y","z","a","b" -"Hello, ""World""",123,"[1,2,3]",456,"['abc','def']","Newline +"Hello, ""World""",123,"[1,2,3]","(456,['abc','def'])","Newline here" "x","y","z","a","b" "String","UInt8","Array(UInt8)","Tuple(UInt16, Array(String))","String" -"Hello, ""World""",123,"[1,2,3]",456,"['abc','def']","Newline +"Hello, ""World""",123,"[1,2,3]","(456,['abc','def'])","Newline here" 0,"0","[]","2000-01-01","2000-01-01 00:00:00" 1,"1","[0]","2000-01-02","2000-01-01 00:00:01" diff --git a/tests/queries/0_stateless/00309_formats.reference b/tests/queries/0_stateless/00309_formats.reference index e637ee0363a..5c0ae4d2c3b 100644 Binary files a/tests/queries/0_stateless/00309_formats.reference and b/tests/queries/0_stateless/00309_formats.reference differ diff --git a/tests/queries/0_stateless/00309_formats_case_insensitive.sql b/tests/queries/0_stateless/00309_formats_case_insensitive.sql index b4037ed9861..0b45e9c22a9 100644 --- a/tests/queries/0_stateless/00309_formats_case_insensitive.sql +++ b/tests/queries/0_stateless/00309_formats_case_insensitive.sql @@ -1,3 +1,5 @@ +-- Tags: no-parallel + SELECT '-- test FORMAT clause --'; SET output_format_write_statistics = 0; SELECT number, 'Hello & world' FROM numbers(3) FORMAT Tsv; diff --git a/tests/queries/0_stateless/00538_datediff_plural_units.reference b/tests/queries/0_stateless/00538_datediff_plural_units.reference index ebe63974df8..885f8a1cfe2 100644 --- a/tests/queries/0_stateless/00538_datediff_plural_units.reference +++ b/tests/queries/0_stateless/00538_datediff_plural_units.reference @@ -8,3 +8,4 @@ -63072000 -63072000000 -63072000000000 +-63072000000000000 diff --git a/tests/queries/0_stateless/00538_datediff_plural_units.sql b/tests/queries/0_stateless/00538_datediff_plural_units.sql index d1234155a56..dd8395fc60f 100644 --- a/tests/queries/0_stateless/00538_datediff_plural_units.sql +++ b/tests/queries/0_stateless/00538_datediff_plural_units.sql @@ -8,3 +8,4 @@ SELECT dateDiff('minutes', toDateTime('2017-12-31', 'UTC'), toDateTime('2016-01- SELECT dateDiff('seconds', toDateTime('2017-12-31', 'UTC'), toDateTime('2016-01-01', 'UTC')); SELECT dateDiff('milliseconds', toDateTime('2017-12-31', 'UTC'), toDateTime('2016-01-01', 'UTC')); SELECT dateDiff('microseconds', toDateTime('2017-12-31', 'UTC'), toDateTime('2016-01-01', 'UTC')); +SELECT dateDiff('nanoseconds', toDateTime('2017-12-31', 'UTC'), toDateTime('2016-01-01', 'UTC')); diff --git a/tests/queries/0_stateless/00623_truncate_all_tables.reference b/tests/queries/0_stateless/00623_truncate_all_tables.reference new file mode 100644 index 00000000000..b1df4c9107d --- /dev/null +++ b/tests/queries/0_stateless/00623_truncate_all_tables.reference @@ -0,0 +1,16 @@ +======Before Truncate====== +1 +1 +1 +1 +1 +2000-01-01 1 +======After Truncate And Empty====== +0 +======After Truncate And Insert Data====== +1 +1 +1 +1 +1 +2000-01-01 1 diff --git a/tests/queries/0_stateless/00623_truncate_all_tables.sql b/tests/queries/0_stateless/00623_truncate_all_tables.sql new file mode 100644 index 00000000000..3a6f94d2907 --- /dev/null +++ b/tests/queries/0_stateless/00623_truncate_all_tables.sql @@ -0,0 +1,50 @@ +-- Tags: no-parallel + +DROP DATABASE IF EXISTS truncate_test; + +CREATE DATABASE IF NOT EXISTS truncate_test; +CREATE TABLE IF NOT EXISTS truncate_test.truncate_test_set(id UInt64) ENGINE = Set; +CREATE TABLE IF NOT EXISTS truncate_test.truncate_test_log(id UInt64) ENGINE = Log; +CREATE TABLE IF NOT EXISTS truncate_test.truncate_test_memory(id UInt64) ENGINE = Memory; +CREATE TABLE IF NOT EXISTS truncate_test.truncate_test_tiny_log(id UInt64) ENGINE = TinyLog; +CREATE TABLE IF NOT EXISTS truncate_test.truncate_test_stripe_log(id UInt64) ENGINE = StripeLog; +CREATE TABLE IF NOT EXISTS truncate_test.truncate_test_merge_tree(p Date, k UInt64) ENGINE = MergeTree ORDER BY p; + +SELECT '======Before Truncate======'; +INSERT INTO truncate_test.truncate_test_set VALUES(0); +INSERT INTO truncate_test.truncate_test_log VALUES(1); +INSERT INTO truncate_test.truncate_test_memory VALUES(1); +INSERT INTO truncate_test.truncate_test_tiny_log VALUES(1); +INSERT INTO truncate_test.truncate_test_stripe_log VALUES(1); +INSERT INTO truncate_test.truncate_test_merge_tree VALUES('2000-01-01', 1); +SELECT * FROM system.numbers WHERE number NOT IN truncate_test.truncate_test_set LIMIT 1; +SELECT * FROM truncate_test.truncate_test_log; +SELECT * FROM truncate_test.truncate_test_memory; +SELECT * FROM truncate_test.truncate_test_tiny_log; +SELECT * FROM truncate_test.truncate_test_stripe_log; +SELECT * FROM truncate_test.truncate_test_merge_tree; + +SELECT '======After Truncate And Empty======'; +TRUNCATE ALL TABLES IF EXISTS truncate_test; +SELECT * FROM system.numbers WHERE number NOT IN truncate_test.truncate_test_set LIMIT 1; +SELECT * FROM truncate_test.truncate_test_log; +SELECT * FROM truncate_test.truncate_test_memory; +SELECT * FROM truncate_test.truncate_test_tiny_log; +SELECT * FROM truncate_test.truncate_test_stripe_log; +SELECT * FROM truncate_test.truncate_test_merge_tree; + +SELECT '======After Truncate And Insert Data======'; +INSERT INTO truncate_test.truncate_test_set VALUES(0); +INSERT INTO truncate_test.truncate_test_log VALUES(1); +INSERT INTO truncate_test.truncate_test_memory VALUES(1); +INSERT INTO truncate_test.truncate_test_tiny_log VALUES(1); +INSERT INTO truncate_test.truncate_test_stripe_log VALUES(1); +INSERT INTO truncate_test.truncate_test_merge_tree VALUES('2000-01-01', 1); +SELECT * FROM system.numbers WHERE number NOT IN truncate_test.truncate_test_set LIMIT 1; +SELECT * FROM truncate_test.truncate_test_log; +SELECT * FROM truncate_test.truncate_test_memory; +SELECT * FROM truncate_test.truncate_test_tiny_log; +SELECT * FROM truncate_test.truncate_test_stripe_log; +SELECT * FROM truncate_test.truncate_test_merge_tree; + +DROP DATABASE IF EXISTS truncate_test; diff --git a/tests/queries/0_stateless/00918_json_functions.reference b/tests/queries/0_stateless/00918_json_functions.reference index 7b725111755..078348cd20f 100644 --- a/tests/queries/0_stateless/00918_json_functions.reference +++ b/tests/queries/0_stateless/00918_json_functions.reference @@ -69,7 +69,6 @@ hello (3333.6,'test') (3333.6333333333,'test') (3333.6333333333,'test') -\N 123456.1234 Decimal(20, 4) 123456.1234 Decimal(20, 4) 123456789012345.12 Decimal(30, 4) @@ -287,3 +286,9 @@ v --show error: type should be const string --show error: index type should be integer --show error: key of map type should be String +\N +\N +Hello +Hello +Hello +Hello diff --git a/tests/queries/0_stateless/00918_json_functions.sql b/tests/queries/0_stateless/00918_json_functions.sql index 61fcb21fcbd..3d30ce841ba 100644 --- a/tests/queries/0_stateless/00918_json_functions.sql +++ b/tests/queries/0_stateless/00918_json_functions.sql @@ -81,7 +81,6 @@ SELECT JSONExtract('{"a":3333.6333333333333333333333, "b":"test"}', 'Tuple(a Dec SELECT JSONExtract('{"a":"3333.6333333333333333333333", "b":"test"}', 'Tuple(a Decimal(10,1), b LowCardinality(String))'); SELECT JSONExtract('{"a":3333.6333333333333333333333, "b":"test"}', 'Tuple(a Decimal(20,10), b LowCardinality(String))'); SELECT JSONExtract('{"a":"3333.6333333333333333333333", "b":"test"}', 'Tuple(a Decimal(20,10), b LowCardinality(String))'); -SELECT JSONExtract(materialize('{"string_value":null}'), materialize('string_value'), 'LowCardinality(Nullable(String))'); SELECT JSONExtract('{"a":123456.123456}', 'a', 'Decimal(20, 4)') as a, toTypeName(a); SELECT JSONExtract('{"a":"123456.123456"}', 'a', 'Decimal(20, 4)') as a, toTypeName(a); SELECT JSONExtract('{"a":"123456789012345.12"}', 'a', 'Decimal(30, 4)') as a, toTypeName(a); @@ -327,4 +326,9 @@ SELECT JSONExtract('[]', JSONExtract('0', 'UInt256'), 'UInt256'); -- { serverErr SELECT '--show error: key of map type should be String'; SELECT JSONExtract('{"a": [100.0, 200], "b": [-100, 200.0, 300]}', 'Map(Int64, Array(Float64))'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } -SELECT JSONExtract(materialize(toLowCardinality('{"string_value":null}')), materialize('string_value'), 'LowCardinality(Nullable(String))'); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT JSONExtract(materialize(toLowCardinality('{"string_value":null}')), materialize('string_value'), 'LowCardinality(Nullable(String))'); +SELECT JSONExtract(materialize('{"string_value":null}'), materialize('string_value'), 'LowCardinality(Nullable(String))'); +SELECT JSONExtract(materialize('{"string_value":"Hello"}'), materialize('string_value'), 'LowCardinality(Nullable(String))') AS x; +SELECT JSONExtract(materialize(toLowCardinality('{"string_value":"Hello"}')), materialize('string_value'), 'LowCardinality(Nullable(String))') AS x; +SELECT JSONExtract(materialize('{"string_value":"Hello"}'), materialize(toLowCardinality('string_value')), 'LowCardinality(Nullable(String))') AS x; +SELECT JSONExtract(materialize(toLowCardinality('{"string_value":"Hello"}')), materialize(toLowCardinality('string_value')), 'LowCardinality(Nullable(String))') AS x; diff --git a/tests/queries/0_stateless/01016_input_null_as_default.sh b/tests/queries/0_stateless/01016_input_null_as_default.sh index 24d93b2703c..8d6a9a07435 100755 --- a/tests/queries/0_stateless/01016_input_null_as_default.sh +++ b/tests/queries/0_stateless/01016_input_null_as_default.sh @@ -11,8 +11,8 @@ $CLICKHOUSE_CLIENT --query="CREATE TABLE default_by_other_column (a Float32 DEFA echo 'CSV' echo '\N, 1, \N, "2019-07-22", "[10, 20, 30]", \N -1, world, 3, "2019-07-23", \N, tuple, 3.14 -2, \N, 123, \N, "[]", test, 2.71828 +1, world, 3, "2019-07-23", \N, "('\''tuple'\'', 3.14)" +2, \N, 123, \N, "[]", "('\''test'\'', 2.71828)" 3, \N, \N, \N, \N, \N' | $CLICKHOUSE_CLIENT --input_format_null_as_default=1 --query="INSERT INTO null_as_default FORMAT CSV"; $CLICKHOUSE_CLIENT --query="SELECT * FROM null_as_default ORDER BY i"; $CLICKHOUSE_CLIENT --query="TRUNCATE TABLE null_as_default"; diff --git a/tests/queries/0_stateless/01193_metadata_loading.reference b/tests/queries/0_stateless/01193_metadata_loading.reference deleted file mode 100644 index 9789cbf33ba..00000000000 --- a/tests/queries/0_stateless/01193_metadata_loading.reference +++ /dev/null @@ -1,5 +0,0 @@ -1000 0 2020-06-25 hello [1,2] [3,4] -1000 1 2020-06-26 word [10,20] [30,40] -ok -8000 0 2020-06-25 hello [1,2] [3,4] -8000 1 2020-06-26 word [10,20] [30,40] diff --git a/tests/queries/0_stateless/01193_metadata_loading.sh b/tests/queries/0_stateless/01193_metadata_loading.sh deleted file mode 100755 index 69178a93d42..00000000000 --- a/tests/queries/0_stateless/01193_metadata_loading.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -# Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug, no-parallel, no-fasttest, no-s3-storage, no-sanitize-coverage - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CURDIR"/../shell_config.sh - -# Check that attaching a database with a large number of tables is not too slow. -# it is the worst way of making performance test, nevertheless it can detect significant slowdown and some other issues, that usually found by stress test - -db="test_01193_$RANDOM_$RANDOM_$RANDOM_$RANDOM" -tables=1000 -threads=10 -count_multiplier=1 -max_time_ms=1500 - -create_tables() { - $CLICKHOUSE_CLIENT -q "WITH - 'CREATE TABLE $db.table_$1_' AS create1, - ' (i UInt64, d Date, s String, n Nested(i UInt8, f Float32)) ENGINE=' AS create2, - ['Memory', 'File(CSV)', 'Log', 'StripeLog', 'MergeTree ORDER BY i'] AS engines, - 'INSERT INTO $db.table_$1_' AS insert1, - ' VALUES (0, ''2020-06-25'', ''hello'', [1, 2], [3, 4]), (1, ''2020-06-26'', ''word'', [10, 20], [30, 40])' AS insert2 - SELECT arrayStringConcat( - groupArray( - create1 || toString(number) || create2 || engines[1 + number % length(engines)] || ';\n' || - insert1 || toString(number) || insert2 - ), ';\n') FROM numbers($tables) SETTINGS max_bytes_before_external_group_by = 0 FORMAT TSVRaw;" | $CLICKHOUSE_CLIENT -nm -} - -$CLICKHOUSE_CLIENT -q "CREATE DATABASE $db" - -for i in $(seq 1 $threads); do - create_tables "$i" & -done -wait - -$CLICKHOUSE_CLIENT -q "CREATE TABLE $db.table_merge (i UInt64, d Date, s String, n Nested(i UInt8, f Float32)) ENGINE=Merge('$db', '^table_')" -$CLICKHOUSE_CLIENT -q "SELECT count() * $count_multiplier, i, d, s, n.i, n.f FROM merge('$db', '^table_9') GROUP BY i, d, s, n.i, n.f ORDER BY i" - -for i in {1..50}; do - $CLICKHOUSE_CLIENT -q "DETACH DATABASE $db" - $CLICKHOUSE_CLIENT --query_profiler_real_time_period_ns=100000000 --query_profiler_cpu_time_period_ns=100000000 -q "ATTACH DATABASE $db" --query_id="$db-$i"; -done - -$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS" -durations=$($CLICKHOUSE_CLIENT -q "SELECT groupArray(query_duration_ms) FROM system.query_log WHERE current_database = currentDatabase() AND query_id LIKE '$db-%' AND type=2") -$CLICKHOUSE_CLIENT -q "SELECT 'durations', '$db', $durations FORMAT Null" -$CLICKHOUSE_CLIENT -q "SELECT if(quantile(0.5)(arrayJoin($durations)) < $max_time_ms, 'ok', toString($durations))" - -$CLICKHOUSE_CLIENT -q "SELECT count() * $count_multiplier, i, d, s, n.i, n.f FROM $db.table_merge GROUP BY i, d, s, n.i, n.f ORDER BY i" - -$CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS $db" diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index b18ae8a99be..b1237189cb3 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -155,7 +155,7 @@ SYSTEM FLUSH ASYNC INSERT QUEUE ['FLUSH ASYNC INSERT QUEUE'] GLOBAL SYSTEM FLUSH SYSTEM FLUSH [] \N SYSTEM SYSTEM THREAD FUZZER ['SYSTEM START THREAD FUZZER','SYSTEM STOP THREAD FUZZER','START THREAD FUZZER','STOP THREAD FUZZER'] GLOBAL SYSTEM SYSTEM UNFREEZE ['SYSTEM UNFREEZE'] GLOBAL SYSTEM -SYSTEM FAILPOINT ['SYSTEM ENABLE FAILPOINT','SYSTEM DISABLE FAILPOINT'] GLOBAL SYSTEM +SYSTEM FAILPOINT ['SYSTEM ENABLE FAILPOINT','SYSTEM DISABLE FAILPOINT','SYSTEM WAIT FAILPOINT'] GLOBAL SYSTEM SYSTEM LISTEN ['SYSTEM START LISTEN','SYSTEM STOP LISTEN'] GLOBAL SYSTEM SYSTEM JEMALLOC ['SYSTEM JEMALLOC PURGE','SYSTEM JEMALLOC ENABLE PROFILE','SYSTEM JEMALLOC DISABLE PROFILE','SYSTEM JEMALLOC FLUSH PROFILE'] GLOBAL SYSTEM SYSTEM [] \N ALL diff --git a/tests/queries/0_stateless/01287_max_execution_speed.sql b/tests/queries/0_stateless/01287_max_execution_speed.sql index 104c10a323d..6c749294975 100644 --- a/tests/queries/0_stateless/01287_max_execution_speed.sql +++ b/tests/queries/0_stateless/01287_max_execution_speed.sql @@ -2,13 +2,13 @@ SET min_execution_speed = 100000000000, timeout_before_checking_execution_speed = 0; SELECT count() FROM system.numbers; -- { serverError 160 } -SELECT 'Ok (1)'; SET min_execution_speed = 0; +SELECT 'Ok (1)'; SET min_execution_speed_bytes = 800000000000, timeout_before_checking_execution_speed = 0; SELECT count() FROM system.numbers; -- { serverError 160 } -SELECT 'Ok (2)'; SET min_execution_speed_bytes = 0; +SELECT 'Ok (2)'; SET max_execution_speed = 1000000; SET max_block_size = 100; diff --git a/tests/queries/0_stateless/01388_multi_if_optimization.reference b/tests/queries/0_stateless/01388_multi_if_optimization.reference index 6dbe0f0d96f..9c2990274fe 100644 --- a/tests/queries/0_stateless/01388_multi_if_optimization.reference +++ b/tests/queries/0_stateless/01388_multi_if_optimization.reference @@ -2,3 +2,5 @@ SELECT if(number = 1, \'hello\', if(number = 2, \'world\', \'xyz\')) FROM numbers(10) SELECT multiIf(number = 1, \'hello\', number = 2, \'world\', \'xyz\') FROM numbers(10) +\N +\N diff --git a/tests/queries/0_stateless/01388_multi_if_optimization.sql b/tests/queries/0_stateless/01388_multi_if_optimization.sql index 345fcfb6fcc..0ad6df9fdfa 100644 --- a/tests/queries/0_stateless/01388_multi_if_optimization.sql +++ b/tests/queries/0_stateless/01388_multi_if_optimization.sql @@ -3,3 +3,6 @@ SET optimize_if_chain_to_multiif = 0; EXPLAIN SYNTAX SELECT number = 1 ? 'hello' : (number = 2 ? 'world' : 'xyz') FROM numbers(10); SET optimize_if_chain_to_multiif = 1; EXPLAIN SYNTAX SELECT number = 1 ? 'hello' : (number = 2 ? 'world' : 'xyz') FROM numbers(10); + +-- fuzzed +SELECT now64(if(Null, NULL, if(Null, nan, toFloat64(number))), Null) FROM numbers(2); diff --git a/tests/queries/0_stateless/01600_detach_permanently.reference b/tests/queries/0_stateless/01600_detach_permanently.reference index 98ed3b6762d..4fda43df162 100644 --- a/tests/queries/0_stateless/01600_detach_permanently.reference +++ b/tests/queries/0_stateless/01600_detach_permanently.reference @@ -96,6 +96,7 @@ View can be reattached test for MV with inner table MV is working 1 +1 View can be reattached ################## DETACH DATABASE is not implemented (proper error) diff --git a/tests/queries/0_stateless/01600_detach_permanently.sh b/tests/queries/0_stateless/01600_detach_permanently.sh index 036706d2fe8..6721dbf3015 100755 --- a/tests/queries/0_stateless/01600_detach_permanently.sh +++ b/tests/queries/0_stateless/01600_detach_permanently.sh @@ -111,7 +111,9 @@ clickhouse_local "INSERT INTO db_ordinary.src SELECT * FROM numbers(10)" clickhouse_local "SELECT if(count() = 10, 'MV is working', 'MV failed') FROM db_ordinary.src_mv_with_inner" clickhouse_local "DETACH VIEW db_ordinary.src_mv_with_inner PERMANENTLY; INSERT INTO db_ordinary.src SELECT * FROM numbers(10)" --stacktrace -clickhouse_local "SELECT if(count() = 10, 'MV can be detached permanently', 'MV detach failed') FROM db_ordinary.src_mv_with_inner" 2>&1 | grep -c "db_ordinary.src_mv_with_inner does not exist" +clickhouse_local "SELECT if(count() = 10, 'MV can be detached permanently', 'MV detach failed') FROM db_ordinary.src_mv_with_inner SETTINGS allow_experimental_analyzer = 0" 2>&1 | grep -c "db_ordinary.src_mv_with_inner does not exist" +clickhouse_local "SELECT if(count() = 10, 'MV can be detached permanently', 'MV detach failed') FROM db_ordinary.src_mv_with_inner SETTINGS allow_experimental_analyzer = 1" 2>&1 | grep -c "Unknown table expression identifier 'db_ordinary.src_mv_with_inner'" + ## Quite silly: ATTACH MATERIALIZED VIEW don't work with short syntax (w/o select), but i can attach it using ATTACH TABLE ... clickhouse_local "ATTACH TABLE db_ordinary.src_mv_with_inner" clickhouse_local "INSERT INTO db_ordinary.src SELECT * FROM numbers(10)" diff --git a/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.sh b/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.sh index db62dedb5b4..ebd6490077e 100755 --- a/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.sh +++ b/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.sh @@ -21,6 +21,10 @@ function test_completion_word() # - here and below you should escape variables of the expect. # - you should not use "expect <<..." since in this case timeout/eof will # not work (I guess due to attached stdin) + + # TODO: get build sanitizer and debug/release info to dynamically change test + # like here timeout 120 seconds is too big for release build + # but ok for sanitizer builds cat > "$SCRIPT_PATH" << EOF # NOTE: log will be appended exp_internal -f $CLICKHOUSE_TMP/$(basename "${BASH_SOURCE[0]}").debuglog 0 @@ -30,7 +34,7 @@ exp_internal -f $CLICKHOUSE_TMP/$(basename "${BASH_SOURCE[0]}").debuglog 0 set stdout_channel [open "/dev/stdout" w] log_user 0 -set timeout 60 +set timeout 120 match_max 100000 expect_after { # Do not ignore eof from expect diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index c9638e62655..bdd0da7d166 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -881,8 +881,8 @@ CREATE TABLE system.replicas `log_pointer` UInt64, `last_queue_update` DateTime, `absolute_delay` UInt64, - `total_replicas` UInt8, - `active_replicas` UInt8, + `total_replicas` UInt32, + `active_replicas` UInt32, `lost_part_count` UInt64, `last_queue_update_exception` String, `zookeeper_exception` String, diff --git a/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh b/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh index acf0608acc5..eb4a91bd850 100755 --- a/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh +++ b/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh @@ -6,4 +6,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_LOCAL --query="SELECT n SETTINGS allow_experimental_analyzer = 1" 2>&1 | grep -q "Code: 47. DB::Exception:" && echo 'OK' || echo 'FAIL' ||: -$CLICKHOUSE_LOCAL --query="SELECT n -- { serverError 47 }" 2>&1 | grep -o 'Missing columns' +$CLICKHOUSE_LOCAL --query="SELECT n SETTINGS allow_experimental_analyzer = 0 -- { serverError 47 }" 2>&1 | grep -o 'Missing columns' diff --git a/tests/queries/0_stateless/02246_tsv_csv_best_effort_schema_inference.reference b/tests/queries/0_stateless/02246_tsv_csv_best_effort_schema_inference.reference index 1c60e40942c..8ad0a566c62 100644 --- a/tests/queries/0_stateless/02246_tsv_csv_best_effort_schema_inference.reference +++ b/tests/queries/0_stateless/02246_tsv_csv_best_effort_schema_inference.reference @@ -97,8 +97,8 @@ c1 Array(Nullable(Bool)) [] [NULL] [false] -c1 Nullable(String) -(1, 2, 3) +c1 Tuple(Nullable(Int64), Nullable(Int64), Nullable(Int64)) +(1,2,3) c1 Nullable(String) 123.123 c1 Array(Tuple(Nullable(Int64), Nullable(Int64), Nullable(Int64))) diff --git a/tests/queries/0_stateless/02287_type_object_convert.reference b/tests/queries/0_stateless/02287_type_object_convert.reference index 501536f1f3e..af4407ee052 100644 --- a/tests/queries/0_stateless/02287_type_object_convert.reference +++ b/tests/queries/0_stateless/02287_type_object_convert.reference @@ -16,3 +16,4 @@ {"x":[[],[1,2]]} {"x":[[],[[1],[2]]]} {"x":[[],[[],[2]]]} +{"a.a":[[1],[]]} diff --git a/tests/queries/0_stateless/02287_type_object_convert.sql b/tests/queries/0_stateless/02287_type_object_convert.sql index 2bc08cf7fe1..21c1aba63d3 100644 --- a/tests/queries/0_stateless/02287_type_object_convert.sql +++ b/tests/queries/0_stateless/02287_type_object_convert.sql @@ -32,3 +32,4 @@ SELECT CAST('{"x" : [ 1 , [ 1 , 2] ]}', 'Object(\'json\')'); SELECT CAST('{"x" : [ {} , [ 1 , 2] ]}', 'Object(\'json\')'); SELECT CAST('{"x" : [ {} , [ 1 , [2]] ]}', 'Object(\'json\')'); SELECT CAST('{"x" : [ {} , [ {} , [2]] ]}', 'Object(\'json\')'); +SELECT CAST(' {"a": { "a": [ [1], null ] } }', 'Object(Nullable(\'json\'))'); diff --git a/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.reference b/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.reference index 43282d09bab..5028adae32e 100644 --- a/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.reference +++ b/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.reference @@ -1,2 +1,3 @@ 4 2 100 99 +(3.22,1) diff --git a/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.sql b/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.sql index 5eba14ea528..8491018eb72 100644 --- a/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.sql +++ b/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.sql @@ -14,3 +14,5 @@ INSERT INTO test_table VALUES (1, 1); SELECT sum((2 * id) as func), func FROM test_table GROUP BY id; SELECT max(100-number), min(100-number) FROM numbers(2); + +select (sum(toDecimal64(2.11, 15) - number), 1) FROM numbers(2); diff --git a/tests/queries/0_stateless/02503_insert_storage_snapshot.sh b/tests/queries/0_stateless/02503_insert_storage_snapshot.sh index b494adeb785..13561947e08 100755 --- a/tests/queries/0_stateless/02503_insert_storage_snapshot.sh +++ b/tests/queries/0_stateless/02503_insert_storage_snapshot.sh @@ -20,6 +20,13 @@ counter=0 retries=60 # for a short period of time. To avoid flakyness we check that refcount became 1 at least once during long INSERT query. # It proves that the INSERT query doesn't hold redundant references to parts. while [[ $counter -lt $retries ]]; do + query_result=$($CLICKHOUSE_CLIENT -q "select count() from system.processes where query_id = '$query_id' FORMAT CSV") + if [ "$query_result" -lt 1 ]; then + sleep 0.1 + ((++counter)) + continue; + fi + query_result=$($CLICKHOUSE_CLIENT -q "SELECT name, active, refcount FROM system.parts WHERE database = '$CLICKHOUSE_DATABASE' AND table = 't_insert_storage_snapshot' FORMAT CSV") if [ "$query_result" == '"all_1_1_0",1,1' ]; then echo "$query_result" diff --git a/tests/queries/0_stateless/02516_projections_and_context.reference b/tests/queries/0_stateless/02516_projections_and_context.reference index 6ed281c757a..e69de29bb2d 100644 --- a/tests/queries/0_stateless/02516_projections_and_context.reference +++ b/tests/queries/0_stateless/02516_projections_and_context.reference @@ -1,2 +0,0 @@ -1 -1 diff --git a/tests/queries/0_stateless/02516_projections_and_context.sql b/tests/queries/0_stateless/02516_projections_and_context.sql index 2b659eafabc..334544eb4fa 100644 --- a/tests/queries/0_stateless/02516_projections_and_context.sql +++ b/tests/queries/0_stateless/02516_projections_and_context.sql @@ -5,6 +5,6 @@ set allow_experimental_analyzer = 0; SELECT count() FROM test1__fuzz_37 GROUP BY dictHas(NULL, (dictHas(NULL, (('', materialize(NULL)), materialize(NULL))), 'KeyKey')), dictHas('test_dictionary', tuple(materialize('Ke\0'))), tuple(dictHas(NULL, (tuple('Ke\0Ke\0Ke\0Ke\0Ke\0Ke\0\0\0\0Ke\0'), materialize(NULL)))), 'test_dicti\0nary', (('', materialize(NULL)), dictHas(NULL, (dictHas(NULL, tuple(materialize(NULL))), 'KeyKeyKeyKeyKeyKeyKeyKey')), materialize(NULL)); -- { serverError BAD_ARGUMENTS } SELECT count() FROM test1__fuzz_37 GROUP BY dictHas('non_existing_dictionary', materialize('a')); -- { serverError BAD_ARGUMENTS } set allow_experimental_analyzer = 1; -SELECT count() FROM test1__fuzz_37 GROUP BY dictHas(NULL, (dictHas(NULL, (('', materialize(NULL)), materialize(NULL))), 'KeyKey')), dictHas('test_dictionary', tuple(materialize('Ke\0'))), tuple(dictHas(NULL, (tuple('Ke\0Ke\0Ke\0Ke\0Ke\0Ke\0\0\0\0Ke\0'), materialize(NULL)))), 'test_dicti\0nary', (('', materialize(NULL)), dictHas(NULL, (dictHas(NULL, tuple(materialize(NULL))), 'KeyKeyKeyKeyKeyKeyKeyKey')), materialize(NULL)); -SELECT count() FROM test1__fuzz_37 GROUP BY dictHas('non_existing_dictionary', materialize('a')); +SELECT count() FROM test1__fuzz_37 GROUP BY dictHas(NULL, (dictHas(NULL, (('', materialize(NULL)), materialize(NULL))), 'KeyKey')), dictHas('test_dictionary', tuple(materialize('Ke\0'))), tuple(dictHas(NULL, (tuple('Ke\0Ke\0Ke\0Ke\0Ke\0Ke\0\0\0\0Ke\0'), materialize(NULL)))), 'test_dicti\0nary', (('', materialize(NULL)), dictHas(NULL, (dictHas(NULL, tuple(materialize(NULL))), 'KeyKeyKeyKeyKeyKeyKeyKey')), materialize(NULL)); -- { serverError BAD_ARGUMENTS } +SELECT count() FROM test1__fuzz_37 GROUP BY dictHas('non_existing_dictionary', materialize('a')); -- { serverError BAD_ARGUMENTS } DROP TABLE test1__fuzz_37; diff --git a/tests/queries/0_stateless/02532_send_logs_level_test.reference b/tests/queries/0_stateless/02532_send_logs_level_test.reference index dbd49cfc0a4..7e51b888d9c 100644 --- a/tests/queries/0_stateless/02532_send_logs_level_test.reference +++ b/tests/queries/0_stateless/02532_send_logs_level_test.reference @@ -1,2 +1,3 @@ + MergeTreeMarksLoader: Loading marks from path data.cmrk3 MergeTreeRangeReader: First reader returned: num_rows: 1, columns: 1, total_rows_per_granule: 1, no filter, column[0]: Int32(size = 1), requested columns: key MergeTreeRangeReader: read() returned num_rows: 1, columns: 1, total_rows_per_granule: 1, no filter, column[0]: Int32(size = 1), sample block key diff --git a/tests/queries/0_stateless/02532_send_logs_level_test.sh b/tests/queries/0_stateless/02532_send_logs_level_test.sh index f65d8705569..4afc6d4496b 100755 --- a/tests/queries/0_stateless/02532_send_logs_level_test.sh +++ b/tests/queries/0_stateless/02532_send_logs_level_test.sh @@ -9,7 +9,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT -nm -q " drop table if exists data; - create table data (key Int) engine=MergeTree order by tuple(); + create table data (key Int) engine=MergeTree order by tuple() settings min_bytes_for_wide_part = '1G', compress_marks = 1; insert into data values (1); " diff --git a/tests/queries/0_stateless/02537_distributed_loosing_files_after_exception.sql.j2 b/tests/queries/0_stateless/02537_distributed_loosing_files_after_exception.sql.j2 index e7f7126d6bf..3eabbfb82e7 100644 --- a/tests/queries/0_stateless/02537_distributed_loosing_files_after_exception.sql.j2 +++ b/tests/queries/0_stateless/02537_distributed_loosing_files_after_exception.sql.j2 @@ -12,9 +12,9 @@ SYSTEM STOP DISTRIBUTED SENDS dist; INSERT INTO dist SETTINGS prefer_localhost_replica=0, max_threads=1 VALUES (1); INSERT INTO dist SETTINGS prefer_localhost_replica=0, max_threads=2 VALUES (1); -SYSTEM FLUSH DISTRIBUTED dist; -- { serverError UNKNOWN_TABLE } +SYSTEM FLUSH DISTRIBUTED dist; -- { serverError ALL_CONNECTION_TRIES_FAILED } -- check the second since after using queue it may got lost from it -SYSTEM FLUSH DISTRIBUTED dist; -- { serverError UNKNOWN_TABLE } +SYSTEM FLUSH DISTRIBUTED dist; -- { serverError ALL_CONNECTION_TRIES_FAILED } SELECT is_blocked, data_files FROM system.distribution_queue WHERE database = currentDatabase() AND table = 'dist'; diff --git a/tests/queries/0_stateless/02681_undrop_query.sql b/tests/queries/0_stateless/02681_undrop_query.sql index 66447fc6c44..c33b74d49e5 100644 --- a/tests/queries/0_stateless/02681_undrop_query.sql +++ b/tests/queries/0_stateless/02681_undrop_query.sql @@ -17,7 +17,7 @@ drop table if exists 02681_undrop_detach sync; create table 02681_undrop_detach (id Int32, num Int32) Engine=MergeTree() order by id; insert into 02681_undrop_detach values (1, 1); detach table 02681_undrop_detach sync; -undrop table 02681_undrop_detach; -- { serverError 57 } +undrop table 02681_undrop_detach; -- { serverError TABLE_ALREADY_EXISTS } attach table 02681_undrop_detach; alter table 02681_undrop_detach update num = 2 where id = 1; select command from system.mutations where table='02681_undrop_detach' and database=currentDatabase() limit 1; @@ -85,5 +85,5 @@ drop table 02681_undrop_multiple; select table from system.dropped_tables where table = '02681_undrop_multiple' limit 1; undrop table 02681_undrop_multiple; select * from 02681_undrop_multiple order by id; -undrop table 02681_undrop_multiple; -- { serverError 57 } +undrop table 02681_undrop_multiple; -- { serverError TABLE_ALREADY_EXISTS } drop table 02681_undrop_multiple sync; diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge.reference b/tests/queries/0_stateless/02763_row_policy_storage_merge.reference index 9fa5612e7cd..6510a3c933e 100644 --- a/tests/queries/0_stateless/02763_row_policy_storage_merge.reference +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge.reference @@ -166,6 +166,10 @@ SELECT x, y from merge(currentDatabase(), 02763_merge 4 14 4 14 4 14 +USING 0 +USING 1 +zzz +==== SETTINGS optimize_move_to_prewhere= 1 SELECT * FROM 02763_merge_log_1 3 13 @@ -312,3 +316,7 @@ SELECT x, y from merge(currentDatabase(), 02763_merge 4 14 4 14 4 14 +USING 0 +USING 1 +zzz +==== diff --git a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 index 0263e1a974f..eabbde9e9dd 100644 --- a/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 +++ b/tests/queries/0_stateless/02763_row_policy_storage_merge.sql.j2 @@ -129,6 +129,27 @@ SELECT x, lc, cnst from merge(currentDatabase(), '02763_merge_fancycols') ORDER SELECT 'SELECT x, y from merge(currentDatabase(), 02763_merge'; SELECT x, y from merge(currentDatabase(), '02763_merge') ORDER BY x SETTINGS optimize_move_to_prewhere= {{prew}}; + + +CREATE TABLE 02763_t1 ( b String ) ENGINE = MergeTree() ORDER BY tuple(); +INSERT INTO 02763_t1 VALUES('zzz'); +CREATE TABLE 02763_t2 AS 02763_t1 ENGINE = Merge(currentDatabase(), '02763_t1'); + +SELECT 'USING 0'; +CREATE ROW POLICY OR REPLACE 02763_filter_t1 ON 02763_t1 USING 0 TO ALL; +SELECT * FROM 02763_t2 SETTINGS optimize_move_to_prewhere= {{prew}}; + +SELECT 'USING 1'; +CREATE ROW POLICY OR REPLACE 02763_filter_t1 ON 02763_t1 USING 1 TO ALL; +SELECT * FROM 02763_t2 SETTINGS optimize_move_to_prewhere= {{prew}}; + + +DROP TABLE 02763_t1; +DROP TABLE 02763_t2; +SELECT '===='; + + + DROP TABLE 02763_merge_fancycols; DROP ROW POLICY 02763_filter_1 ON 02763_merge_log_1; @@ -140,4 +161,6 @@ DROP ROW POLICY 02763_filter_4 ON 02763_merge_merge_1; DROP ROW POLICY 02763_filter_5 ON 02763_merge_fancycols; DROP ROW POLICY 02763_filter_6 ON 02763_merge_fancycols; +DROP ROW POLICY 02763_filter_t1 ON 02763_t1; + {% endfor %} diff --git a/tests/queries/0_stateless/02783_parallel_replicas_trivial_count_optimization.sh b/tests/queries/0_stateless/02783_parallel_replicas_trivial_count_optimization.sh index bafab249b47..20b3efedd49 100755 --- a/tests/queries/0_stateless/02783_parallel_replicas_trivial_count_optimization.sh +++ b/tests/queries/0_stateless/02783_parallel_replicas_trivial_count_optimization.sh @@ -25,7 +25,7 @@ function run_query_with_pure_parallel_replicas () { $CLICKHOUSE_CLIENT \ --query "$2" \ --query_id "${1}_disabled" \ - --max_parallel_replicas 0 + --max_parallel_replicas 1 $CLICKHOUSE_CLIENT \ --query "$2" \ @@ -50,7 +50,7 @@ function run_query_with_custom_key_parallel_replicas () { $CLICKHOUSE_CLIENT \ --query "$2" \ --query_id "${1}_disabled" \ - --max_parallel_replicas 0 + --max_parallel_replicas 1 $CLICKHOUSE_CLIENT \ --query "$2" \ diff --git a/tests/queries/0_stateless/02802_with_cube_with_totals.reference b/tests/queries/0_stateless/02802_with_cube_with_totals.reference index c7b7b570456..206c32e562b 100644 --- a/tests/queries/0_stateless/02802_with_cube_with_totals.reference +++ b/tests/queries/0_stateless/02802_with_cube_with_totals.reference @@ -1,5 +1,35 @@ ((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 ((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 +((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 ((2147483648,(-0,1.1754943508222875e-38,2147483646,'-9223372036854775808',NULL))) 0 \N diff --git a/tests/queries/0_stateless/02802_with_cube_with_totals.sql b/tests/queries/0_stateless/02802_with_cube_with_totals.sql index 77adb68eb4b..168e4d61b68 100644 --- a/tests/queries/0_stateless/02802_with_cube_with_totals.sql +++ b/tests/queries/0_stateless/02802_with_cube_with_totals.sql @@ -1,2 +1,3 @@ +set allow_experimental_analyzer=1; SELECT tuple((2147483648, (-0., 1.1754943508222875e-38, 2147483646, '-9223372036854775808', NULL))), toInt128(0.0001) GROUP BY ((256, toInt64(1.1754943508222875e-38), NULL), NULL, -0., ((65535, '-92233720368547758.07'), 0.9999), tuple(((1., 3.4028234663852886e38, '1', 0.5), NULL, tuple('0.1')))) WITH CUBE WITH TOTALS; SELECT NULL GROUP BY toUUID(NULL, '0', NULL, '0.0000065535'), 1 WITH CUBE WITH TOTALS; diff --git a/tests/queries/0_stateless/02814_age_datediff.reference b/tests/queries/0_stateless/02814_age_datediff.reference index cbcb8c8a7b6..85dc205a499 100644 --- a/tests/queries/0_stateless/02814_age_datediff.reference +++ b/tests/queries/0_stateless/02814_age_datediff.reference @@ -1,10 +1,14 @@ -- { echo } -- DateTime64 vs DateTime64 with fractional part +SELECT age('nanosecond', toDateTime64('2015-08-18 20:30:36.100200005', 9, 'UTC'), toDateTime64('2015-08-18 20:30:41.200400005', 9, 'UTC')); +5100200000 +SELECT age('nanosecond', toDateTime64('2015-08-18 20:30:36.100200005', 9, 'UTC'), toDateTime64('2015-08-18 20:30:41.200400004', 9, 'UTC')); +5100199999 SELECT age('microsecond', toDateTime64('2015-08-18 20:30:36.100200005', 9, 'UTC'), toDateTime64('2015-08-18 20:30:41.200400005', 9, 'UTC')); 5100200 SELECT age('microsecond', toDateTime64('2015-08-18 20:30:36.100200005', 9, 'UTC'), toDateTime64('2015-08-18 20:30:41.200400004', 9, 'UTC')); -5100200 +5100199 SELECT age('millisecond', toDateTime64('2015-08-18 20:30:36.450299', 6, 'UTC'), toDateTime64('2015-08-18 20:30:41.550299', 6, 'UTC')); 5100 SELECT age('millisecond', toDateTime64('2015-08-18 20:30:36.450299', 6, 'UTC'), toDateTime64('2015-08-18 20:30:41.550298', 6, 'UTC')); diff --git a/tests/queries/0_stateless/02814_age_datediff.sql b/tests/queries/0_stateless/02814_age_datediff.sql index 934a95c035f..64e329b2fc5 100644 --- a/tests/queries/0_stateless/02814_age_datediff.sql +++ b/tests/queries/0_stateless/02814_age_datediff.sql @@ -1,6 +1,9 @@ -- { echo } -- DateTime64 vs DateTime64 with fractional part +SELECT age('nanosecond', toDateTime64('2015-08-18 20:30:36.100200005', 9, 'UTC'), toDateTime64('2015-08-18 20:30:41.200400005', 9, 'UTC')); +SELECT age('nanosecond', toDateTime64('2015-08-18 20:30:36.100200005', 9, 'UTC'), toDateTime64('2015-08-18 20:30:41.200400004', 9, 'UTC')); + SELECT age('microsecond', toDateTime64('2015-08-18 20:30:36.100200005', 9, 'UTC'), toDateTime64('2015-08-18 20:30:41.200400005', 9, 'UTC')); SELECT age('microsecond', toDateTime64('2015-08-18 20:30:36.100200005', 9, 'UTC'), toDateTime64('2015-08-18 20:30:41.200400004', 9, 'UTC')); diff --git a/tests/queries/0_stateless/02841_valid_json_and_xml_on_http_exception.reference b/tests/queries/0_stateless/02841_valid_json_and_xml_on_http_exception.reference index 1818ca7b5f2..2cfba492361 100644 --- a/tests/queries/0_stateless/02841_valid_json_and_xml_on_http_exception.reference +++ b/tests/queries/0_stateless/02841_valid_json_and_xml_on_http_exception.reference @@ -1,3 +1,4 @@ +wait_end_of_query=0 One block Parallel formatting: 0 JSON @@ -430,3 +431,350 @@ Test 2 Test 3 1 1 +wait_end_of_query=1 +One block +Parallel formatting: 0 +JSON +{ + "meta": + [ + { + "name": "number", + "type": "UInt64" + }, + { + "name": "res", + "type": "UInt8" + } + ], + + "data": + [ + + ], + + "rows": 0, + + "exception": "Code: 395. : Value passed to 'throwIf' function is non-zero: while executing 'FUNCTION throwIf(greater(number, 3) :: 2) -> throwIf(greater(number, 3)) UInt8 : 1'. (FUNCTION_THROW_IF_VALUE_IS_NON_ZERO) " +} +JSONEachRow +{"exception": "Code: 395. : Value passed to 'throwIf' function is non-zero: while executing 'FUNCTION throwIf(greater(number, 3) :: 2) -> throwIf(greater(number, 3)) UInt8 : 1'. (FUNCTION_THROW_IF_VALUE_IS_NON_ZERO) "} +JSONCompact +{ + "meta": + [ + { + "name": "number", + "type": "UInt64" + }, + { + "name": "res", + "type": "UInt8" + } + ], + + "data": + [ + + ], + + "rows": 0, + + "exception": "Code: 395. : Value passed to 'throwIf' function is non-zero: while executing 'FUNCTION throwIf(greater(number, 3) :: 2) -> throwIf(greater(number, 3)) UInt8 : 1'. (FUNCTION_THROW_IF_VALUE_IS_NON_ZERO) " +} +JSONCompactEachRow +["Code: 395. : Value passed to 'throwIf' function is non-zero: while executing 'FUNCTION throwIf(greater(number, 3) :: 2) -> throwIf(greater(number, 3)) UInt8 : 1'. (FUNCTION_THROW_IF_VALUE_IS_NON_ZERO) "] +JSONObjectEachRow +{ + "exception": "Code: 395. : Value passed to 'throwIf' function is non-zero: while executing 'FUNCTION throwIf(greater(number, 3) :: 2) -> throwIf(greater(number, 3)) UInt8 : 1'. (FUNCTION_THROW_IF_VALUE_IS_NON_ZERO) " +} +XML + + + + + + number + UInt64 + + + res + UInt8 + + + + + + 0 + Code: 395. : Value passed to 'throwIf' function is non-zero: while executing 'FUNCTION throwIf(greater(number, 3) :: 2) -> throwIf(greater(number, 3)) UInt8 : 1'. (FUNCTION_THROW_IF_VALUE_IS_NON_ZERO) + +Parallel formatting: 1 +JSON +{ + "meta": + [ + { + "name": "number", + "type": "UInt64" + }, + { + "name": "res", + "type": "UInt8" + } + ], + + "data": + [ + + ], + + "rows": 0, + + "exception": "Code: 395. : Value passed to 'throwIf' function is non-zero: while executing 'FUNCTION throwIf(greater(number, 3) :: 2) -> throwIf(greater(number, 3)) UInt8 : 1'. (FUNCTION_THROW_IF_VALUE_IS_NON_ZERO) " +} +JSONEachRow +{"exception": "Code: 395. : Value passed to 'throwIf' function is non-zero: while executing 'FUNCTION throwIf(greater(number, 3) :: 2) -> throwIf(greater(number, 3)) UInt8 : 1'. (FUNCTION_THROW_IF_VALUE_IS_NON_ZERO) "} +JSONCompact +{ + "meta": + [ + { + "name": "number", + "type": "UInt64" + }, + { + "name": "res", + "type": "UInt8" + } + ], + + "data": + [ + + ], + + "rows": 0, + + "exception": "Code: 395. : Value passed to 'throwIf' function is non-zero: while executing 'FUNCTION throwIf(greater(number, 3) :: 2) -> throwIf(greater(number, 3)) UInt8 : 1'. (FUNCTION_THROW_IF_VALUE_IS_NON_ZERO) " +} +JSONCompactEachRow +["Code: 395. : Value passed to 'throwIf' function is non-zero: while executing 'FUNCTION throwIf(greater(number, 3) :: 2) -> throwIf(greater(number, 3)) UInt8 : 1'. (FUNCTION_THROW_IF_VALUE_IS_NON_ZERO) "] +JSONObjectEachRow +{ + "exception": "Code: 395. : Value passed to 'throwIf' function is non-zero: while executing 'FUNCTION throwIf(greater(number, 3) :: 2) -> throwIf(greater(number, 3)) UInt8 : 1'. (FUNCTION_THROW_IF_VALUE_IS_NON_ZERO) " +} +XML + + + + + + number + UInt64 + + + res + UInt8 + + + + + + 0 + Code: 395. : Value passed to 'throwIf' function is non-zero: while executing 'FUNCTION throwIf(greater(number, 3) :: 2) -> throwIf(greater(number, 3)) UInt8 : 1'. (FUNCTION_THROW_IF_VALUE_IS_NON_ZERO) + +Several blocks +Without parallel formatting +JSON +{ + "meta": + [ + { + "name": "number", + "type": "UInt64" + }, + { + "name": "res", + "type": "UInt8" + } + ], + + "data": + [ + + ], + + "rows": 0, + + "exception": "Code: 395. : Value passed to 'throwIf' function is non-zero: while executing 'FUNCTION throwIf(greater(number, 3) :: 2) -> throwIf(greater(number, 3)) UInt8 : 1'. (FUNCTION_THROW_IF_VALUE_IS_NON_ZERO) " +} +JSONEachRow +{"exception": "Code: 395. : Value passed to 'throwIf' function is non-zero: while executing 'FUNCTION throwIf(greater(number, 3) :: 2) -> throwIf(greater(number, 3)) UInt8 : 1'. (FUNCTION_THROW_IF_VALUE_IS_NON_ZERO) "} +JSONCompact +{ + "meta": + [ + { + "name": "number", + "type": "UInt64" + }, + { + "name": "res", + "type": "UInt8" + } + ], + + "data": + [ + + ], + + "rows": 0, + + "exception": "Code: 395. : Value passed to 'throwIf' function is non-zero: while executing 'FUNCTION throwIf(greater(number, 3) :: 2) -> throwIf(greater(number, 3)) UInt8 : 1'. (FUNCTION_THROW_IF_VALUE_IS_NON_ZERO) " +} +JSONCompactEachRow +["Code: 395. : Value passed to 'throwIf' function is non-zero: while executing 'FUNCTION throwIf(greater(number, 3) :: 2) -> throwIf(greater(number, 3)) UInt8 : 1'. (FUNCTION_THROW_IF_VALUE_IS_NON_ZERO) "] +JSONObjectEachRow +{ + "exception": "Code: 395. : Value passed to 'throwIf' function is non-zero: while executing 'FUNCTION throwIf(greater(number, 3) :: 2) -> throwIf(greater(number, 3)) UInt8 : 1'. (FUNCTION_THROW_IF_VALUE_IS_NON_ZERO) " +} +XML + + + + + + number + UInt64 + + + res + UInt8 + + + + + + 0 + Code: 395. : Value passed to 'throwIf' function is non-zero: while executing 'FUNCTION throwIf(greater(number, 3) :: 2) -> throwIf(greater(number, 3)) UInt8 : 1'. (FUNCTION_THROW_IF_VALUE_IS_NON_ZERO) + +With parallel formatting +JSON +1 +JSONCompact +1 +JSONObjectEachRow +1 +JSONEachRow +1 +JSONCompactEachRow +1 +Formatting error +Without parallel formatting +JSON +{ + "meta": + [ + { + "name": "x", + "type": "UInt32" + }, + { + "name": "s", + "type": "String" + }, + { + "name": "y", + "type": "Enum8('a' = 1)" + } + ], + + "data": + [ + + ], + + "rows": 0, + + "exception": "Code: 36. : Unexpected value 99 in enum: While executing JSONRowOutputFormat. (BAD_ARGUMENTS) " +} +JSONEachRow +{"exception": "Code: 36. : Unexpected value 99 in enum: While executing JSONEachRowRowOutputFormat. (BAD_ARGUMENTS) "} +JSONCompact +{ + "meta": + [ + { + "name": "x", + "type": "UInt32" + }, + { + "name": "s", + "type": "String" + }, + { + "name": "y", + "type": "Enum8('a' = 1)" + } + ], + + "data": + [ + + ], + + "rows": 0, + + "exception": "Code: 36. : Unexpected value 99 in enum: While executing JSONCompactRowOutputFormat. (BAD_ARGUMENTS) " +} +JSONCompactEachRow +["Code: 36. : Unexpected value 99 in enum: While executing JSONCompactEachRowRowOutputFormat. (BAD_ARGUMENTS) "] +JSONObjectEachRow +{ + "exception": "Code: 36. : Unexpected value 99 in enum: While executing JSONObjectEachRowRowOutputFormat. (BAD_ARGUMENTS) " +} +XML + + + + + + x + UInt32 + + + s + String + + + y + Enum8('a' = 1) + + + + + + 0 + Code: 36. : Unexpected value 99 in enum: While executing XMLRowOutputFormat. (BAD_ARGUMENTS) + +With parallel formatting +JSON +1 +JSONCompact +1 +JSONObjectEachRow +1 +JSONEachRow +1 +JSONCompactEachRow +1 +Test 1 +1 +1 +Test 2 +1 +1 +Test 3 +1 +1 diff --git a/tests/queries/0_stateless/02841_valid_json_and_xml_on_http_exception.sh b/tests/queries/0_stateless/02841_valid_json_and_xml_on_http_exception.sh index 26b3ef64d61..c47fe5c7e94 100755 --- a/tests/queries/0_stateless/02841_valid_json_and_xml_on_http_exception.sh +++ b/tests/queries/0_stateless/02841_valid_json_and_xml_on_http_exception.sh @@ -4,7 +4,12 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -CH_URL="$CLICKHOUSE_URL&http_write_exception_in_output_format=1&allow_experimental_analyzer=0" +CH_URL_BASE="$CLICKHOUSE_URL&http_write_exception_in_output_format=1&allow_experimental_analyzer=0" + +for wait_end_of_query in 0 1 +do +echo "wait_end_of_query=$wait_end_of_query" +CH_URL="$CH_URL_BASE&wait_end_of_query=$wait_end_of_query" echo "One block" for parallel in 0 1 @@ -106,3 +111,4 @@ ${CLICKHOUSE_CURL} -sS "$CH_URL" -d "select * from test_02841 format JSON settin $CLICKHOUSE_CLIENT -q "drop table test_02841" +done diff --git a/tests/queries/0_stateless/02890_describe_table_options.reference b/tests/queries/0_stateless/02890_describe_table_options.reference index 15ff1dc0f15..9181cb27cb0 100644 --- a/tests/queries/0_stateless/02890_describe_table_options.reference +++ b/tests/queries/0_stateless/02890_describe_table_options.reference @@ -1,198 +1,182 @@ -- { echoOn } SET describe_compact_output = 0, describe_include_virtual_columns = 0, describe_include_subcolumns = 0; -DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes; - ┌─name─┬─type──────────────────────┬─default_type─┬─default_expression─┬─comment──────┬─codec_expression─┬─ttl_expression─┐ -1. │ id │ UInt64 │ │ │ index column │ │ │ -2. │ arr │ Array(UInt64) │ DEFAULT │ [10, 20] │ │ ZSTD(1) │ │ -3. │ t │ Tuple(a String, b UInt64) │ DEFAULT │ ('foo', 0) │ │ ZSTD(1) │ │ - └──────┴───────────────────────────┴──────────────┴────────────────────┴──────────────┴──────────────────┴────────────────┘ -DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes; - ┌─name─┬─type──────────────────────┬─default_type─┬─default_expression─┬─comment──────┬─codec_expression─┬─ttl_expression─┐ -1. │ id │ UInt64 │ │ │ index column │ │ │ -2. │ arr │ Array(UInt64) │ DEFAULT │ [10, 20] │ │ ZSTD(1) │ │ -3. │ t │ Tuple(a String, b UInt64) │ DEFAULT │ ('foo', 0) │ │ ZSTD(1) │ │ - └──────┴───────────────────────────┴──────────────┴────────────────────┴──────────────┴──────────────────┴────────────────┘ +DESCRIBE TABLE t_describe_options; +id UInt64 index column +arr Array(UInt64) DEFAULT [10, 20] ZSTD(1) +t Tuple(a String, b UInt64) DEFAULT (\'foo\', 0) ZSTD(1) +DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options); +id UInt64 index column +arr Array(UInt64) DEFAULT [10, 20] ZSTD(1) +t Tuple(a String, b UInt64) DEFAULT (\'foo\', 0) ZSTD(1) SET describe_compact_output = 0, describe_include_virtual_columns = 0, describe_include_subcolumns = 1; -DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes; - ┌─name──────┬─type──────────────────────┬─default_type─┬─default_expression─┬─comment──────┬─codec_expression─┬─ttl_expression─┬─is_subcolumn─┐ -1. │ id │ UInt64 │ │ │ index column │ │ │ 0 │ -2. │ arr │ Array(UInt64) │ DEFAULT │ [10, 20] │ │ ZSTD(1) │ │ 0 │ -3. │ t │ Tuple(a String, b UInt64) │ DEFAULT │ ('foo', 0) │ │ ZSTD(1) │ │ 0 │ -4. │ arr.size0 │ UInt64 │ │ │ │ │ │ 1 │ -5. │ t.a │ String │ │ │ │ ZSTD(1) │ │ 1 │ -6. │ t.b │ UInt64 │ │ │ │ ZSTD(1) │ │ 1 │ - └───────────┴───────────────────────────┴──────────────┴────────────────────┴──────────────┴──────────────────┴────────────────┴──────────────┘ -DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes; - ┌─name──────┬─type──────────────────────┬─default_type─┬─default_expression─┬─comment──────┬─codec_expression─┬─ttl_expression─┬─is_subcolumn─┐ -1. │ id │ UInt64 │ │ │ index column │ │ │ 0 │ -2. │ arr │ Array(UInt64) │ DEFAULT │ [10, 20] │ │ ZSTD(1) │ │ 0 │ -3. │ t │ Tuple(a String, b UInt64) │ DEFAULT │ ('foo', 0) │ │ ZSTD(1) │ │ 0 │ -4. │ arr.size0 │ UInt64 │ │ │ │ │ │ 1 │ -5. │ t.a │ String │ │ │ │ ZSTD(1) │ │ 1 │ -6. │ t.b │ UInt64 │ │ │ │ ZSTD(1) │ │ 1 │ - └───────────┴───────────────────────────┴──────────────┴────────────────────┴──────────────┴──────────────────┴────────────────┴──────────────┘ +DESCRIBE TABLE t_describe_options; +id UInt64 index column 0 +arr Array(UInt64) DEFAULT [10, 20] ZSTD(1) 0 +t Tuple(a String, b UInt64) DEFAULT (\'foo\', 0) ZSTD(1) 0 +arr.size0 UInt64 1 +t.a String ZSTD(1) 1 +t.b UInt64 ZSTD(1) 1 +DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options); +id UInt64 index column 0 +arr Array(UInt64) DEFAULT [10, 20] ZSTD(1) 0 +t Tuple(a String, b UInt64) DEFAULT (\'foo\', 0) ZSTD(1) 0 +arr.size0 UInt64 1 +t.a String ZSTD(1) 1 +t.b UInt64 ZSTD(1) 1 SET describe_compact_output = 0, describe_include_virtual_columns = 1, describe_include_subcolumns = 0; -DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes; - ┌─name───────────┬─type──────────────────────┬─default_type─┬─default_expression─┬─comment─────────────────────────────────────────────────────────────────────────────────┬─codec_expression─┬─ttl_expression─┬─is_virtual─┐ - 1. │ id │ UInt64 │ │ │ index column │ │ │ 0 │ - 2. │ arr │ Array(UInt64) │ DEFAULT │ [10, 20] │ │ ZSTD(1) │ │ 0 │ - 3. │ t │ Tuple(a String, b UInt64) │ DEFAULT │ ('foo', 0) │ │ ZSTD(1) │ │ 0 │ - 4. │ _part │ LowCardinality(String) │ │ │ Name of part │ │ │ 1 │ - 5. │ _part_index │ UInt64 │ │ │ Sequential index of the part in the query result │ │ │ 1 │ - 6. │ _part_uuid │ UUID │ │ │ Unique part identifier (if enabled MergeTree setting assign_part_uuids) │ │ │ 1 │ - 7. │ _partition_id │ LowCardinality(String) │ │ │ Name of partition │ │ │ 1 │ - 8. │ _sample_factor │ Float64 │ │ │ Sample factor (from the query) │ │ │ 1 │ - 9. │ _part_offset │ UInt64 │ │ │ Number of row in the part │ │ │ 1 │ -10. │ _row_exists │ UInt8 │ │ │ Persisted mask created by lightweight delete that show whether row exists or is deleted │ │ │ 1 │ -11. │ _block_number │ UInt64 │ │ │ Persisted original number of block that was assigned at insert │ Delta, LZ4 │ │ 1 │ - └────────────────┴───────────────────────────┴──────────────┴────────────────────┴─────────────────────────────────────────────────────────────────────────────────────────┴──────────────────┴────────────────┴────────────┘ -DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes; - ┌─name───────────┬─type──────────────────────┬─default_type─┬─default_expression─┬─comment─────────────────────────────────────────────────────────────────────────────────┬─codec_expression─┬─ttl_expression─┬─is_virtual─┐ - 1. │ id │ UInt64 │ │ │ index column │ │ │ 0 │ - 2. │ arr │ Array(UInt64) │ DEFAULT │ [10, 20] │ │ ZSTD(1) │ │ 0 │ - 3. │ t │ Tuple(a String, b UInt64) │ DEFAULT │ ('foo', 0) │ │ ZSTD(1) │ │ 0 │ - 4. │ _part │ LowCardinality(String) │ │ │ Name of part │ │ │ 1 │ - 5. │ _part_index │ UInt64 │ │ │ Sequential index of the part in the query result │ │ │ 1 │ - 6. │ _part_uuid │ UUID │ │ │ Unique part identifier (if enabled MergeTree setting assign_part_uuids) │ │ │ 1 │ - 7. │ _partition_id │ LowCardinality(String) │ │ │ Name of partition │ │ │ 1 │ - 8. │ _sample_factor │ Float64 │ │ │ Sample factor (from the query) │ │ │ 1 │ - 9. │ _part_offset │ UInt64 │ │ │ Number of row in the part │ │ │ 1 │ -10. │ _row_exists │ UInt8 │ │ │ Persisted mask created by lightweight delete that show whether row exists or is deleted │ │ │ 1 │ -11. │ _block_number │ UInt64 │ │ │ Persisted original number of block that was assigned at insert │ Delta, LZ4 │ │ 1 │ -12. │ _shard_num │ UInt32 │ │ │ Deprecated. Use function shardNum instead │ │ │ 1 │ - └────────────────┴───────────────────────────┴──────────────┴────────────────────┴─────────────────────────────────────────────────────────────────────────────────────────┴──────────────────┴────────────────┴────────────┘ +DESCRIBE TABLE t_describe_options; +id UInt64 index column 0 +arr Array(UInt64) DEFAULT [10, 20] ZSTD(1) 0 +t Tuple(a String, b UInt64) DEFAULT (\'foo\', 0) ZSTD(1) 0 +_part LowCardinality(String) Name of part 1 +_part_index UInt64 Sequential index of the part in the query result 1 +_part_uuid UUID Unique part identifier (if enabled MergeTree setting assign_part_uuids) 1 +_partition_id LowCardinality(String) Name of partition 1 +_sample_factor Float64 Sample factor (from the query) 1 +_part_offset UInt64 Number of row in the part 1 +_part_data_version UInt64 Data version of part (either min block number or mutation version) 1 +_row_exists UInt8 Persisted mask created by lightweight delete that show whether row exists or is deleted 1 +_block_number UInt64 Persisted original number of block that was assigned at insert Delta, LZ4 1 +_block_offset UInt64 Persisted original number of row in block that was assigned at insert Delta, LZ4 1 +DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options); +id UInt64 index column 0 +arr Array(UInt64) DEFAULT [10, 20] ZSTD(1) 0 +t Tuple(a String, b UInt64) DEFAULT (\'foo\', 0) ZSTD(1) 0 +_part LowCardinality(String) Name of part 1 +_part_index UInt64 Sequential index of the part in the query result 1 +_part_uuid UUID Unique part identifier (if enabled MergeTree setting assign_part_uuids) 1 +_partition_id LowCardinality(String) Name of partition 1 +_sample_factor Float64 Sample factor (from the query) 1 +_part_offset UInt64 Number of row in the part 1 +_part_data_version UInt64 Data version of part (either min block number or mutation version) 1 +_row_exists UInt8 Persisted mask created by lightweight delete that show whether row exists or is deleted 1 +_block_number UInt64 Persisted original number of block that was assigned at insert Delta, LZ4 1 +_block_offset UInt64 Persisted original number of row in block that was assigned at insert Delta, LZ4 1 +_shard_num UInt32 Deprecated. Use function shardNum instead 1 SET describe_compact_output = 0, describe_include_virtual_columns = 1, describe_include_subcolumns = 1; -DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes; - ┌─name───────────┬─type──────────────────────┬─default_type─┬─default_expression─┬─comment─────────────────────────────────────────────────────────────────────────────────┬─codec_expression─┬─ttl_expression─┬─is_subcolumn─┬─is_virtual─┐ - 1. │ id │ UInt64 │ │ │ index column │ │ │ 0 │ 0 │ - 2. │ arr │ Array(UInt64) │ DEFAULT │ [10, 20] │ │ ZSTD(1) │ │ 0 │ 0 │ - 3. │ t │ Tuple(a String, b UInt64) │ DEFAULT │ ('foo', 0) │ │ ZSTD(1) │ │ 0 │ 0 │ - 4. │ _part │ LowCardinality(String) │ │ │ Name of part │ │ │ 0 │ 1 │ - 5. │ _part_index │ UInt64 │ │ │ Sequential index of the part in the query result │ │ │ 0 │ 1 │ - 6. │ _part_uuid │ UUID │ │ │ Unique part identifier (if enabled MergeTree setting assign_part_uuids) │ │ │ 0 │ 1 │ - 7. │ _partition_id │ LowCardinality(String) │ │ │ Name of partition │ │ │ 0 │ 1 │ - 8. │ _sample_factor │ Float64 │ │ │ Sample factor (from the query) │ │ │ 0 │ 1 │ - 9. │ _part_offset │ UInt64 │ │ │ Number of row in the part │ │ │ 0 │ 1 │ -10. │ _row_exists │ UInt8 │ │ │ Persisted mask created by lightweight delete that show whether row exists or is deleted │ │ │ 0 │ 1 │ -11. │ _block_number │ UInt64 │ │ │ Persisted original number of block that was assigned at insert │ Delta, LZ4 │ │ 0 │ 1 │ -12. │ arr.size0 │ UInt64 │ │ │ │ │ │ 1 │ 0 │ -13. │ t.a │ String │ │ │ │ ZSTD(1) │ │ 1 │ 0 │ -14. │ t.b │ UInt64 │ │ │ │ ZSTD(1) │ │ 1 │ 0 │ - └────────────────┴───────────────────────────┴──────────────┴────────────────────┴─────────────────────────────────────────────────────────────────────────────────────────┴──────────────────┴────────────────┴──────────────┴────────────┘ -DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes; - ┌─name───────────┬─type──────────────────────┬─default_type─┬─default_expression─┬─comment─────────────────────────────────────────────────────────────────────────────────┬─codec_expression─┬─ttl_expression─┬─is_subcolumn─┬─is_virtual─┐ - 1. │ id │ UInt64 │ │ │ index column │ │ │ 0 │ 0 │ - 2. │ arr │ Array(UInt64) │ DEFAULT │ [10, 20] │ │ ZSTD(1) │ │ 0 │ 0 │ - 3. │ t │ Tuple(a String, b UInt64) │ DEFAULT │ ('foo', 0) │ │ ZSTD(1) │ │ 0 │ 0 │ - 4. │ _part │ LowCardinality(String) │ │ │ Name of part │ │ │ 0 │ 1 │ - 5. │ _part_index │ UInt64 │ │ │ Sequential index of the part in the query result │ │ │ 0 │ 1 │ - 6. │ _part_uuid │ UUID │ │ │ Unique part identifier (if enabled MergeTree setting assign_part_uuids) │ │ │ 0 │ 1 │ - 7. │ _partition_id │ LowCardinality(String) │ │ │ Name of partition │ │ │ 0 │ 1 │ - 8. │ _sample_factor │ Float64 │ │ │ Sample factor (from the query) │ │ │ 0 │ 1 │ - 9. │ _part_offset │ UInt64 │ │ │ Number of row in the part │ │ │ 0 │ 1 │ -10. │ _row_exists │ UInt8 │ │ │ Persisted mask created by lightweight delete that show whether row exists or is deleted │ │ │ 0 │ 1 │ -11. │ _block_number │ UInt64 │ │ │ Persisted original number of block that was assigned at insert │ Delta, LZ4 │ │ 0 │ 1 │ -12. │ _shard_num │ UInt32 │ │ │ Deprecated. Use function shardNum instead │ │ │ 0 │ 1 │ -13. │ arr.size0 │ UInt64 │ │ │ │ │ │ 1 │ 0 │ -14. │ t.a │ String │ │ │ │ ZSTD(1) │ │ 1 │ 0 │ -15. │ t.b │ UInt64 │ │ │ │ ZSTD(1) │ │ 1 │ 0 │ - └────────────────┴───────────────────────────┴──────────────┴────────────────────┴─────────────────────────────────────────────────────────────────────────────────────────┴──────────────────┴────────────────┴──────────────┴────────────┘ +DESCRIBE TABLE t_describe_options; +id UInt64 index column 0 0 +arr Array(UInt64) DEFAULT [10, 20] ZSTD(1) 0 0 +t Tuple(a String, b UInt64) DEFAULT (\'foo\', 0) ZSTD(1) 0 0 +_part LowCardinality(String) Name of part 0 1 +_part_index UInt64 Sequential index of the part in the query result 0 1 +_part_uuid UUID Unique part identifier (if enabled MergeTree setting assign_part_uuids) 0 1 +_partition_id LowCardinality(String) Name of partition 0 1 +_sample_factor Float64 Sample factor (from the query) 0 1 +_part_offset UInt64 Number of row in the part 0 1 +_part_data_version UInt64 Data version of part (either min block number or mutation version) 0 1 +_row_exists UInt8 Persisted mask created by lightweight delete that show whether row exists or is deleted 0 1 +_block_number UInt64 Persisted original number of block that was assigned at insert Delta, LZ4 0 1 +_block_offset UInt64 Persisted original number of row in block that was assigned at insert Delta, LZ4 0 1 +arr.size0 UInt64 1 0 +t.a String ZSTD(1) 1 0 +t.b UInt64 ZSTD(1) 1 0 +DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options); +id UInt64 index column 0 0 +arr Array(UInt64) DEFAULT [10, 20] ZSTD(1) 0 0 +t Tuple(a String, b UInt64) DEFAULT (\'foo\', 0) ZSTD(1) 0 0 +_part LowCardinality(String) Name of part 0 1 +_part_index UInt64 Sequential index of the part in the query result 0 1 +_part_uuid UUID Unique part identifier (if enabled MergeTree setting assign_part_uuids) 0 1 +_partition_id LowCardinality(String) Name of partition 0 1 +_sample_factor Float64 Sample factor (from the query) 0 1 +_part_offset UInt64 Number of row in the part 0 1 +_part_data_version UInt64 Data version of part (either min block number or mutation version) 0 1 +_row_exists UInt8 Persisted mask created by lightweight delete that show whether row exists or is deleted 0 1 +_block_number UInt64 Persisted original number of block that was assigned at insert Delta, LZ4 0 1 +_block_offset UInt64 Persisted original number of row in block that was assigned at insert Delta, LZ4 0 1 +_shard_num UInt32 Deprecated. Use function shardNum instead 0 1 +arr.size0 UInt64 1 0 +t.a String ZSTD(1) 1 0 +t.b UInt64 ZSTD(1) 1 0 SET describe_compact_output = 1, describe_include_virtual_columns = 0, describe_include_subcolumns = 0; -DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes; - ┌─name─┬─type──────────────────────┐ -1. │ id │ UInt64 │ -2. │ arr │ Array(UInt64) │ -3. │ t │ Tuple(a String, b UInt64) │ - └──────┴───────────────────────────┘ -DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes; - ┌─name─┬─type──────────────────────┐ -1. │ id │ UInt64 │ -2. │ arr │ Array(UInt64) │ -3. │ t │ Tuple(a String, b UInt64) │ - └──────┴───────────────────────────┘ +DESCRIBE TABLE t_describe_options; +id UInt64 +arr Array(UInt64) +t Tuple(a String, b UInt64) +DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options); +id UInt64 +arr Array(UInt64) +t Tuple(a String, b UInt64) SET describe_compact_output = 1, describe_include_virtual_columns = 0, describe_include_subcolumns = 1; -DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes; - ┌─name──────┬─type──────────────────────┬─is_subcolumn─┐ -1. │ id │ UInt64 │ 0 │ -2. │ arr │ Array(UInt64) │ 0 │ -3. │ t │ Tuple(a String, b UInt64) │ 0 │ -4. │ arr.size0 │ UInt64 │ 1 │ -5. │ t.a │ String │ 1 │ -6. │ t.b │ UInt64 │ 1 │ - └───────────┴───────────────────────────┴──────────────┘ -DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes; - ┌─name──────┬─type──────────────────────┬─is_subcolumn─┐ -1. │ id │ UInt64 │ 0 │ -2. │ arr │ Array(UInt64) │ 0 │ -3. │ t │ Tuple(a String, b UInt64) │ 0 │ -4. │ arr.size0 │ UInt64 │ 1 │ -5. │ t.a │ String │ 1 │ -6. │ t.b │ UInt64 │ 1 │ - └───────────┴───────────────────────────┴──────────────┘ +DESCRIBE TABLE t_describe_options; +id UInt64 0 +arr Array(UInt64) 0 +t Tuple(a String, b UInt64) 0 +arr.size0 UInt64 1 +t.a String 1 +t.b UInt64 1 +DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options); +id UInt64 0 +arr Array(UInt64) 0 +t Tuple(a String, b UInt64) 0 +arr.size0 UInt64 1 +t.a String 1 +t.b UInt64 1 SET describe_compact_output = 1, describe_include_virtual_columns = 1, describe_include_subcolumns = 0; -DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes; - ┌─name───────────┬─type──────────────────────┬─is_virtual─┐ - 1. │ id │ UInt64 │ 0 │ - 2. │ arr │ Array(UInt64) │ 0 │ - 3. │ t │ Tuple(a String, b UInt64) │ 0 │ - 4. │ _part │ LowCardinality(String) │ 1 │ - 5. │ _part_index │ UInt64 │ 1 │ - 6. │ _part_uuid │ UUID │ 1 │ - 7. │ _partition_id │ LowCardinality(String) │ 1 │ - 8. │ _sample_factor │ Float64 │ 1 │ - 9. │ _part_offset │ UInt64 │ 1 │ -10. │ _row_exists │ UInt8 │ 1 │ -11. │ _block_number │ UInt64 │ 1 │ - └────────────────┴───────────────────────────┴────────────┘ -DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes; - ┌─name───────────┬─type──────────────────────┬─is_virtual─┐ - 1. │ id │ UInt64 │ 0 │ - 2. │ arr │ Array(UInt64) │ 0 │ - 3. │ t │ Tuple(a String, b UInt64) │ 0 │ - 4. │ _part │ LowCardinality(String) │ 1 │ - 5. │ _part_index │ UInt64 │ 1 │ - 6. │ _part_uuid │ UUID │ 1 │ - 7. │ _partition_id │ LowCardinality(String) │ 1 │ - 8. │ _sample_factor │ Float64 │ 1 │ - 9. │ _part_offset │ UInt64 │ 1 │ -10. │ _row_exists │ UInt8 │ 1 │ -11. │ _block_number │ UInt64 │ 1 │ -12. │ _shard_num │ UInt32 │ 1 │ - └────────────────┴───────────────────────────┴────────────┘ +DESCRIBE TABLE t_describe_options; +id UInt64 0 +arr Array(UInt64) 0 +t Tuple(a String, b UInt64) 0 +_part LowCardinality(String) 1 +_part_index UInt64 1 +_part_uuid UUID 1 +_partition_id LowCardinality(String) 1 +_sample_factor Float64 1 +_part_offset UInt64 1 +_part_data_version UInt64 1 +_row_exists UInt8 1 +_block_number UInt64 1 +_block_offset UInt64 1 +DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options); +id UInt64 0 +arr Array(UInt64) 0 +t Tuple(a String, b UInt64) 0 +_part LowCardinality(String) 1 +_part_index UInt64 1 +_part_uuid UUID 1 +_partition_id LowCardinality(String) 1 +_sample_factor Float64 1 +_part_offset UInt64 1 +_part_data_version UInt64 1 +_row_exists UInt8 1 +_block_number UInt64 1 +_block_offset UInt64 1 +_shard_num UInt32 1 SET describe_compact_output = 1, describe_include_virtual_columns = 1, describe_include_subcolumns = 1; -DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes; - ┌─name───────────┬─type──────────────────────┬─is_subcolumn─┬─is_virtual─┐ - 1. │ id │ UInt64 │ 0 │ 0 │ - 2. │ arr │ Array(UInt64) │ 0 │ 0 │ - 3. │ t │ Tuple(a String, b UInt64) │ 0 │ 0 │ - 4. │ _part │ LowCardinality(String) │ 0 │ 1 │ - 5. │ _part_index │ UInt64 │ 0 │ 1 │ - 6. │ _part_uuid │ UUID │ 0 │ 1 │ - 7. │ _partition_id │ LowCardinality(String) │ 0 │ 1 │ - 8. │ _sample_factor │ Float64 │ 0 │ 1 │ - 9. │ _part_offset │ UInt64 │ 0 │ 1 │ -10. │ _row_exists │ UInt8 │ 0 │ 1 │ -11. │ _block_number │ UInt64 │ 0 │ 1 │ -12. │ arr.size0 │ UInt64 │ 1 │ 0 │ -13. │ t.a │ String │ 1 │ 0 │ -14. │ t.b │ UInt64 │ 1 │ 0 │ - └────────────────┴───────────────────────────┴──────────────┴────────────┘ -DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes; - ┌─name───────────┬─type──────────────────────┬─is_subcolumn─┬─is_virtual─┐ - 1. │ id │ UInt64 │ 0 │ 0 │ - 2. │ arr │ Array(UInt64) │ 0 │ 0 │ - 3. │ t │ Tuple(a String, b UInt64) │ 0 │ 0 │ - 4. │ _part │ LowCardinality(String) │ 0 │ 1 │ - 5. │ _part_index │ UInt64 │ 0 │ 1 │ - 6. │ _part_uuid │ UUID │ 0 │ 1 │ - 7. │ _partition_id │ LowCardinality(String) │ 0 │ 1 │ - 8. │ _sample_factor │ Float64 │ 0 │ 1 │ - 9. │ _part_offset │ UInt64 │ 0 │ 1 │ -10. │ _row_exists │ UInt8 │ 0 │ 1 │ -11. │ _block_number │ UInt64 │ 0 │ 1 │ -12. │ _shard_num │ UInt32 │ 0 │ 1 │ -13. │ arr.size0 │ UInt64 │ 1 │ 0 │ -14. │ t.a │ String │ 1 │ 0 │ -15. │ t.b │ UInt64 │ 1 │ 0 │ - └────────────────┴───────────────────────────┴──────────────┴────────────┘ +DESCRIBE TABLE t_describe_options; +id UInt64 0 0 +arr Array(UInt64) 0 0 +t Tuple(a String, b UInt64) 0 0 +_part LowCardinality(String) 0 1 +_part_index UInt64 0 1 +_part_uuid UUID 0 1 +_partition_id LowCardinality(String) 0 1 +_sample_factor Float64 0 1 +_part_offset UInt64 0 1 +_part_data_version UInt64 0 1 +_row_exists UInt8 0 1 +_block_number UInt64 0 1 +_block_offset UInt64 0 1 +arr.size0 UInt64 1 0 +t.a String 1 0 +t.b UInt64 1 0 +DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options); +id UInt64 0 0 +arr Array(UInt64) 0 0 +t Tuple(a String, b UInt64) 0 0 +_part LowCardinality(String) 0 1 +_part_index UInt64 0 1 +_part_uuid UUID 0 1 +_partition_id LowCardinality(String) 0 1 +_sample_factor Float64 0 1 +_part_offset UInt64 0 1 +_part_data_version UInt64 0 1 +_row_exists UInt8 0 1 +_block_number UInt64 0 1 +_block_offset UInt64 0 1 +_shard_num UInt32 0 1 +arr.size0 UInt64 1 0 +t.a String 1 0 +t.b UInt64 1 0 diff --git a/tests/queries/0_stateless/02890_describe_table_options.sql b/tests/queries/0_stateless/02890_describe_table_options.sql index 419aec1a0f8..74ecfa9a33f 100644 --- a/tests/queries/0_stateless/02890_describe_table_options.sql +++ b/tests/queries/0_stateless/02890_describe_table_options.sql @@ -13,43 +13,43 @@ ORDER BY id; SET describe_compact_output = 0, describe_include_virtual_columns = 0, describe_include_subcolumns = 0; -DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes; -DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes; +DESCRIBE TABLE t_describe_options; +DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options); SET describe_compact_output = 0, describe_include_virtual_columns = 0, describe_include_subcolumns = 1; -DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes; -DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes; +DESCRIBE TABLE t_describe_options; +DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options); SET describe_compact_output = 0, describe_include_virtual_columns = 1, describe_include_subcolumns = 0; -DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes; -DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes; +DESCRIBE TABLE t_describe_options; +DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options); SET describe_compact_output = 0, describe_include_virtual_columns = 1, describe_include_subcolumns = 1; -DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes; -DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes; +DESCRIBE TABLE t_describe_options; +DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options); SET describe_compact_output = 1, describe_include_virtual_columns = 0, describe_include_subcolumns = 0; -DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes; -DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes; +DESCRIBE TABLE t_describe_options; +DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options); SET describe_compact_output = 1, describe_include_virtual_columns = 0, describe_include_subcolumns = 1; -DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes; -DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes; +DESCRIBE TABLE t_describe_options; +DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options); SET describe_compact_output = 1, describe_include_virtual_columns = 1, describe_include_subcolumns = 0; -DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes; -DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes; +DESCRIBE TABLE t_describe_options; +DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options); SET describe_compact_output = 1, describe_include_virtual_columns = 1, describe_include_subcolumns = 1; -DESCRIBE TABLE t_describe_options FORMAT PrettyCompactNoEscapes; -DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options) FORMAT PrettyCompactNoEscapes; +DESCRIBE TABLE t_describe_options; +DESCRIBE remote(test_shard_localhost, currentDatabase(), t_describe_options); -- { echoOff } diff --git a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference index 17c94686470..f376bb87044 100644 --- a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference +++ b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference @@ -1,4 +1,8 @@ Creating 300 tables +900 1 1 +900 1 1 +900 1 1 +900 1 1 Making 200 requests to system.replicas Query system.replicas while waiting for other concurrent requests to finish 0 diff --git a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh index 17e1d87963a..144831a2cdc 100755 --- a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh +++ b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh @@ -45,6 +45,13 @@ done wait; +# Check results with different max_block_size +$CLICKHOUSE_CLIENT -q 'SELECT count(), sum(total_replicas) >= 2700, sum(active_replicas) >= 2700 FROM system.replicas WHERE database=currentDatabase()' +$CLICKHOUSE_CLIENT -q 'SELECT count(), sum(total_replicas) >= 2700, sum(active_replicas) >= 2700 FROM system.replicas WHERE database=currentDatabase() SETTINGS max_block_size=1' +$CLICKHOUSE_CLIENT -q 'SELECT count(), sum(total_replicas) >= 2700, sum(active_replicas) >= 2700 FROM system.replicas WHERE database=currentDatabase() SETTINGS max_block_size=77' +$CLICKHOUSE_CLIENT -q 'SELECT count(), sum(total_replicas) >= 2700, sum(active_replicas) >= 2700 FROM system.replicas WHERE database=currentDatabase() SETTINGS max_block_size=11111' + + echo "Making $CONCURRENCY requests to system.replicas" for i in $(seq 1 $CONCURRENCY) diff --git a/tests/queries/0_stateless/02916_move_partition_inactive_replica.sql b/tests/queries/0_stateless/02916_move_partition_inactive_replica.sql index ca153eea221..b0699539ac1 100644 --- a/tests/queries/0_stateless/02916_move_partition_inactive_replica.sql +++ b/tests/queries/0_stateless/02916_move_partition_inactive_replica.sql @@ -8,8 +8,8 @@ drop table if exists shard_1.from_1; drop table if exists shard_0.to; drop table if exists shard_1.to; -create table shard_0.from_1 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_1_' || currentDatabase(), '0') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1; -create table shard_1.from_1 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_1_' || currentDatabase(), '1') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1; +create table shard_0.from_1 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_1_' || currentDatabase(), '0') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1, shared_merge_tree_disable_merges_and_mutations_assignment=1; +create table shard_1.from_1 (x UInt32) engine = ReplicatedMergeTree('/clickhouse/tables/from_1_' || currentDatabase(), '1') order by x settings old_parts_lifetime=1, max_cleanup_delay_period=1, cleanup_delay_period=1, shared_merge_tree_disable_merges_and_mutations_assignment=1; system stop merges shard_0.from_1; system stop merges shard_1.from_1; diff --git a/tests/queries/0_stateless/02935_date_trunc_case_unsensitiveness.sql b/tests/queries/0_stateless/02935_date_trunc_case_unsensitiveness.sql index ecf6877d477..06cda16fb01 100644 --- a/tests/queries/0_stateless/02935_date_trunc_case_unsensitiveness.sql +++ b/tests/queries/0_stateless/02935_date_trunc_case_unsensitiveness.sql @@ -7,3 +7,6 @@ SELECT dateTrunc('Week', toDate('2022-03-01')); SELECT dateTrunc('day', toDateTime('2022-03-01 12:55:55')); SELECT dateTrunc('month', toDateTime64('2022-03-01 12:55:55', 2)); SELECT dateTrunc('week', toDate('2022-03-01')); +SELECT dateTrunc('Nanosecond', toDate('2022-03-01')); -- { serverError 36 } +SELECT dateTrunc('MicroSecond', toDate('2022-03-01')); -- { serverError 36 } +SELECT dateTrunc('MILLISECOND', toDate('2022-03-01')); -- { serverError 36 } diff --git a/tests/queries/0_stateless/02935_external_table_enum_type.reference b/tests/queries/0_stateless/02935_external_table_enum_type.reference index ed152e608b9..d13bb1e990c 100644 --- a/tests/queries/0_stateless/02935_external_table_enum_type.reference +++ b/tests/queries/0_stateless/02935_external_table_enum_type.reference @@ -2,3 +2,9 @@ foo 1 bar 2 foo 1 bar 2 +true +true +true +[1] +('foo',1) +(1,1) diff --git a/tests/queries/0_stateless/02935_external_table_enum_type.sh b/tests/queries/0_stateless/02935_external_table_enum_type.sh index 61d5508e9f9..bbbf79fce4a 100755 --- a/tests/queries/0_stateless/02935_external_table_enum_type.sh +++ b/tests/queries/0_stateless/02935_external_table_enum_type.sh @@ -10,3 +10,13 @@ curl -s "${http_url}temp_structure=x+Enum8('foo'%3D1,'bar'%3D2),y+Int" -F "$(pri curl -s "${http_url}temp_types=Enum8('foo'%3D1,'bar'%3D2),Int" -F "$(printf 'temp='"bar"'\t2');filename=data1" -F "query=SELECT * FROM temp" echo -ne 'foo\t1' | ${CLICKHOUSE_CLIENT} --query="select * from tmp" --external --file=- --name=tmp --structure="x Enum8('foo'=1,'bar'=2),y Int" echo -ne 'bar\t2' | ${CLICKHOUSE_CLIENT} --query="select * from tmp" --external --file=- --name=tmp --types="Enum8('foo'=1,'bar'=2),Int" + +# https://github.com/ClickHouse/ClickHouse/issues/62108 +echo -ne 'true' | ${CLICKHOUSE_CLIENT} --query="select * from tmp" --external --file=- --name=tmp --structure="x Bool" +echo -ne 'true' | ${CLICKHOUSE_CLIENT} --query="select * from tmp" --external --file=- --name=tmp --types="Bool" + +# Test for some complex and custome types +echo -ne 'true' | ${CLICKHOUSE_CLIENT} --query="select * from tmp" --external --file=- --name=tmp --structure="x Nullable(FixedString(4))" +echo -ne '[1]' | ${CLICKHOUSE_CLIENT} --query="select * from tmp" --external --file=- --name=tmp --structure="x Array(UInt8)" +echo -ne '('"'"'foo'"'"',1)' | ${CLICKHOUSE_CLIENT} --query="select * from tmp" --external --file=- --name=tmp --structure="x Tuple(String, UInt8)" +echo -ne '(1,1)' | ${CLICKHOUSE_CLIENT} --query="select * from tmp" --external --file=- --name=tmp --structure="x Point" diff --git a/tests/queries/0_stateless/02961_drop_tables.reference b/tests/queries/0_stateless/02961_drop_tables.reference new file mode 100644 index 00000000000..c0465dc592a --- /dev/null +++ b/tests/queries/0_stateless/02961_drop_tables.reference @@ -0,0 +1,8 @@ +-- check which tables exist in 02961_db1 +-- check which tables exist in 02961_db2 +02961_tb4 +02961_tb5 +Test when deletion of existing table fails +-- check which tables exist in 02961_db1 +-- check which tables exist in 02961_db2 +02961_tb5 diff --git a/tests/queries/0_stateless/02961_drop_tables.sql b/tests/queries/0_stateless/02961_drop_tables.sql new file mode 100644 index 00000000000..e91ac4bfe19 --- /dev/null +++ b/tests/queries/0_stateless/02961_drop_tables.sql @@ -0,0 +1,32 @@ +-- Tags: no-parallel +DROP DATABASE IF EXISTS 02961_db1; +CREATE DATABASE IF NOT EXISTS 02961_db1; +DROP DATABASE IF EXISTS 02961_db2; +CREATE DATABASE IF NOT EXISTS 02961_db2; + + +CREATE TABLE IF NOT EXISTS 02961_db1.02961_tb1 (id UInt32) Engine=Memory(); +CREATE TABLE IF NOT EXISTS 02961_db1.02961_tb2 (id UInt32) Engine=Memory(); + +CREATE TABLE IF NOT EXISTS 02961_db2.02961_tb3 (id UInt32) Engine=Memory(); +CREATE TABLE IF NOT EXISTS 02961_db2.02961_tb4 (id UInt32) Engine=Memory(); +CREATE TABLE IF NOT EXISTS 02961_db2.02961_tb5 (id UInt32) Engine=Memory(); + +DROP TABLE 02961_db1.02961_tb1, 02961_db1.02961_tb2, 02961_db2.02961_tb3; + +SELECT '-- check which tables exist in 02961_db1'; +SHOW TABLES FROM 02961_db1; +SELECT '-- check which tables exist in 02961_db2'; +SHOW TABLES FROM 02961_db2; + +SELECT 'Test when deletion of existing table fails'; +DROP TABLE 02961_db2.02961_tb4, 02961_db1.02961_tb1, 02961_db2.02961_tb5; -- { serverError UNKNOWN_TABLE } + +SELECT '-- check which tables exist in 02961_db1'; +SHOW TABLES FROM 02961_db1; +SELECT '-- check which tables exist in 02961_db2'; +SHOW TABLES FROM 02961_db2; + + +DROP DATABASE IF EXISTS 02961_db1; +DROP DATABASE IF EXISTS 02961_db2; diff --git a/tests/queries/0_stateless/02969_auto_format_detection.reference b/tests/queries/0_stateless/02969_auto_format_detection.reference index 4b86be04996..865db11defc 100644 --- a/tests/queries/0_stateless/02969_auto_format_detection.reference +++ b/tests/queries/0_stateless/02969_auto_format_detection.reference @@ -82,8 +82,7 @@ CSV c1 Nullable(UInt64) c2 Nullable(String) c3 Array(Nullable(UInt64)) -c4 Nullable(UInt64) -c5 Nullable(String) +c4 Tuple(Nullable(UInt64), Nullable(String)) a Nullable(String) b Nullable(String) c Array(Nullable(String)) diff --git a/tests/queries/0_stateless/02977_csv_format_support_tuple.reference b/tests/queries/0_stateless/02977_csv_format_support_tuple.reference new file mode 100644 index 00000000000..be68d164cff --- /dev/null +++ b/tests/queries/0_stateless/02977_csv_format_support_tuple.reference @@ -0,0 +1,6 @@ +c1 Nullable(String) +c2 Nullable(Int64) +c3 Array(Nullable(String)) +c4 Map(String, Nullable(Int64)) +c5 Tuple(Nullable(String), Nullable(Int64), Map(String, Nullable(Int64))) +20240305 1 ['s','d'] {'a':2} ('222',33,{'abc':5}) diff --git a/tests/queries/0_stateless/02977_csv_format_support_tuple.sql b/tests/queries/0_stateless/02977_csv_format_support_tuple.sql new file mode 100644 index 00000000000..d00cc00e097 --- /dev/null +++ b/tests/queries/0_stateless/02977_csv_format_support_tuple.sql @@ -0,0 +1,5 @@ +-- Tags: no-parallel + +insert into function file('02977_1.csv') select '20240305', 1, ['s', 'd'], map('a', 2), tuple('222', 33, map('abc', 5)) SETTINGS engine_file_truncate_on_insert=1; +desc file('02977_1.csv'); +select * from file('02977_1.csv') settings max_threads=1; diff --git a/tests/queries/0_stateless/02980_dist_insert_readonly_replica.reference b/tests/queries/0_stateless/02980_dist_insert_readonly_replica.reference new file mode 100644 index 00000000000..2ee76705e90 --- /dev/null +++ b/tests/queries/0_stateless/02980_dist_insert_readonly_replica.reference @@ -0,0 +1,3 @@ +shard_0.data 0 +shard_1.data 1 +DistributedConnectionSkipReadOnlyReplica 1 1 diff --git a/tests/queries/0_stateless/02980_dist_insert_readonly_replica.sql.j2 b/tests/queries/0_stateless/02980_dist_insert_readonly_replica.sql.j2 new file mode 100644 index 00000000000..5bf40f34f5c --- /dev/null +++ b/tests/queries/0_stateless/02980_dist_insert_readonly_replica.sql.j2 @@ -0,0 +1,83 @@ +-- Tags: no-parallel, no-fasttest +-- Tag no-parallel - due to static databases +-- Tag no-fasttest - S3 is required + +drop database if exists shard_0; +drop database if exists shard_1; +create database shard_0; +create database shard_1; + +drop table if exists dist; +drop table if exists dist_batch; + +-- normal replica +create table shard_0.data (key Int) engine=ReplicatedMergeTree('/tables/{database}/data', 'write') order by key; +-- create read-only replica +create table shard_1.data (key Int) engine=ReplicatedMergeTree('/tables/{database}/data', 'read') order by key; + +-- broke replica and leave it read-only +detach table shard_1.data; +insert into system.zookeeper (path, name, value) values ('/tables/shard_1/data/replicas/read', 'columns', ''); +attach table shard_1.data; +select database||'.'||table, is_readonly from system.replicas where database in ('shard_0', 'shard_1') and table = 'data'; + +create table dist as shard_0.data engine=Distributed('test_cluster_two_replicas_different_databases_internal_replication', '', 'data'); +system stop distributed sends dist; + +create table dist_batch as shard_0.data engine=Distributed('test_cluster_two_replicas_different_databases_internal_replication', '', 'data') settings background_insert_batch=1; +system stop distributed sends dist_batch; + +create table dist_single_no_internal_replication_read_only as shard_0.data engine=Distributed('test_shard_localhost', 'shard_1', 'data'); +system stop distributed sends dist_single_no_internal_replication_read_only; + +create table dist_single as shard_0.data engine=Distributed('test_cluster_two_replicas_different_databases_internal_replication', 'shard_1', 'data'); +system stop distributed sends dist_single; + +set prefer_localhost_replica=0; +set insert_deduplicate=0; +-- replica is readonly, avoid too many retries +set insert_keeper_max_retries=3; +-- and disable the fault injection to avoid failures +set insert_keeper_fault_injection_probability=0; + +-- for internal_replication==false, distributed_insert_skip_read_only_replicas does not changes anything +-- because in case of internal_replication=false it cannot skip any replicas, since it need to duplicate the data to all replicas on the initiator +insert into dist_single_no_internal_replication_read_only settings distributed_foreground_insert=1, distributed_insert_skip_read_only_replicas=0 values (0); -- { serverError TABLE_IS_READ_ONLY } +insert into dist_single_no_internal_replication_read_only settings distributed_foreground_insert=1, distributed_insert_skip_read_only_replicas=1 values (0); -- { serverError TABLE_IS_READ_ONLY } + +-- for internal_replication==true, it does +insert into dist_single settings distributed_foreground_insert=1, distributed_insert_skip_read_only_replicas=0 values (0); -- { serverError TABLE_IS_READ_ONLY } +insert into dist_single settings distributed_foreground_insert=1, distributed_insert_skip_read_only_replicas=1 values (0); -- { serverError ALL_CONNECTION_TRIES_FAILED } + +-- min_insert_block_size_rows is not enough, since replica will be selected +-- before, so we need to perform INSERT multiple times to ensure that at least +-- once read-only replica should be selected (if it wasn't not filtered out) +{% for i in range(1, 30) %} +insert into dist settings distributed_foreground_insert=1, distributed_insert_skip_read_only_replicas=1, load_balancing='round_robin' values ({{ i }}); +{% endfor %} +-- cannot check for background inserts, so only here +system flush logs; +select + 'DistributedConnectionSkipReadOnlyReplica', count() > 0, sum(ProfileEvents['DistributedConnectionSkipReadOnlyReplica']) > 0 +from system.query_log +where + event_date >= yesterday() + and current_database = currentDatabase() + and query_kind = 'Insert' + and type != 'QueryStart' + and Settings['distributed_foreground_insert'] = '1'; + +{% for i in range(1, 30) %} +insert into dist settings distributed_foreground_insert=0, distributed_insert_skip_read_only_replicas=1 values ({{ i }}); +system flush distributed dist; +{% endfor %} + +{% for i in range(1, 30) %} +insert into dist_batch settings distributed_foreground_insert=0, distributed_insert_skip_read_only_replicas=1 values ({{ i }}); +system flush distributed dist_batch; +{% endfor %} + +drop database shard_0; +drop database shard_1; + +-- vim: ft=sql diff --git a/tests/queries/0_stateless/03001_block_offset_column.reference.j2 b/tests/queries/0_stateless/03001_block_offset_column.reference.j2 new file mode 100644 index 00000000000..641409167bf --- /dev/null +++ b/tests/queries/0_stateless/03001_block_offset_column.reference.j2 @@ -0,0 +1,43 @@ +{% for enable_vertical_merge_algorithm in [0, 1] -%} +*** BEFORE MUTATION BEFORE MERGE *** +1 1 1 0 all_1_1_0 +2 2 2 0 all_2_2_0 +3 3 1 1 all_1_1_0 +4 4 1 2 all_1_1_0 +5 5 2 1 all_2_2_0 +6 6 2 2 all_2_2_0 +*** AFTER MUTATION BEFORE MERGE *** +1 0 1 0 all_1_1_0_3 +2 0 2 0 all_2_2_0_3 +3 0 1 1 all_1_1_0_3 +4 4 1 2 all_1_1_0_3 +5 5 2 1 all_2_2_0_3 +6 6 2 2 all_2_2_0_3 +*** AFTER MUTATION AFTER MERGE *** +1 0 1 0 all_1_2_1_3 +2 0 2 0 all_1_2_1_3 +3 0 1 1 all_1_2_1_3 +4 4 1 2 all_1_2_1_3 +5 5 2 1 all_1_2_1_3 +6 6 2 2 all_1_2_1_3 +*** AFTER MUTATION AFTER MERGE , NEW BLOCK *** +1 0 1 0 all_1_2_1_3 +2 0 2 0 all_1_2_1_3 +3 0 1 1 all_1_2_1_3 +4 4 1 2 all_1_2_1_3 +5 5 2 1 all_1_2_1_3 +6 6 2 2 all_1_2_1_3 +7 7 4 0 all_4_4_0 +8 8 4 1 all_4_4_0 +9 9 4 2 all_4_4_0 +*** AFTER MUTATION AFTER MERGE , NEW BLOCK MERGED *** +1 0 1 0 all_1_4_2_3 +2 0 2 0 all_1_4_2_3 +3 0 1 1 all_1_4_2_3 +4 4 1 2 all_1_4_2_3 +5 5 2 1 all_1_4_2_3 +6 6 2 2 all_1_4_2_3 +7 7 4 0 all_1_4_2_3 +8 8 4 1 all_1_4_2_3 +9 9 4 2 all_1_4_2_3 +{% endfor -%} diff --git a/tests/queries/0_stateless/03001_block_offset_column.sql.j2 b/tests/queries/0_stateless/03001_block_offset_column.sql.j2 new file mode 100644 index 00000000000..b161183f884 --- /dev/null +++ b/tests/queries/0_stateless/03001_block_offset_column.sql.j2 @@ -0,0 +1,44 @@ +{% for enable_vertical_merge_algorithm in [0, 1] -%} + +DROP TABLE IF EXISTS t_block_offset; + +CREATE TABLE t_block_offset (id UInt32, a UInt32) ENGINE = MergeTree +ORDER BY id +SETTINGS + enable_block_number_column = 1, + enable_block_offset_column = 1, + index_granularity = 2, + vertical_merge_algorithm_min_bytes_to_activate = 1, + vertical_merge_algorithm_min_columns_to_activate = 1, + enable_vertical_merge_algorithm = {{ enable_vertical_merge_algorithm }}; + +INSERT INTO t_block_offset(id,a) VALUES (1,1),(3,3),(4,4); +INSERT INTO t_block_offset(id,a) VALUES (2,2),(5,5),(6,6); + +SELECT '*** BEFORE MUTATION BEFORE MERGE ***'; +SELECT id,a,_block_number,_block_offset,_part from t_block_offset ORDER BY id; + +set mutations_sync=1; +ALTER TABLE t_block_offset UPDATE a=0 WHERE id<4; + +SELECT '*** AFTER MUTATION BEFORE MERGE ***'; +SELECT id,a,_block_number,_block_offset,_part from t_block_offset ORDER BY id; + +OPTIMIZE TABLE t_block_offset FINAL; + +SELECT '*** AFTER MUTATION AFTER MERGE ***'; +SELECT *,_block_number,_block_offset,_part from t_block_offset ORDER BY id; + +INSERT INTO t_block_offset(id,a) VALUES (7,7),(8,8),(9,9); + +SELECT '*** AFTER MUTATION AFTER MERGE , NEW BLOCK ***'; +SELECT *,_block_number,_block_offset,_part from t_block_offset ORDER BY id; + +OPTIMIZE TABLE t_block_offset FINAL; + +SELECT '*** AFTER MUTATION AFTER MERGE , NEW BLOCK MERGED ***'; +SELECT *,_block_number,_block_offset,_part from t_block_offset ORDER BY id; + +DROP TABLE t_block_offset; + +{% endfor %} diff --git a/tests/queries/0_stateless/03001_block_offset_column_2.reference b/tests/queries/0_stateless/03001_block_offset_column_2.reference new file mode 100644 index 00000000000..a7ad8232678 --- /dev/null +++ b/tests/queries/0_stateless/03001_block_offset_column_2.reference @@ -0,0 +1,49 @@ +all_1_2_1 1 0 0 0 +all_1_2_1 1 1 1 2 +all_1_2_1 1 2 2 4 +all_1_2_1 1 3 3 6 +all_1_2_1 1 4 4 8 +all_1_2_1 1 5 5 10 +all_1_2_1 1 6 6 12 +all_1_2_1 1 7 7 14 +all_1_2_1 1 8 8 16 +all_1_2_1 1 9 9 18 +all_1_2_1 1 10 10 20 +all_1_2_1 1 11 11 22 +all_1_2_1 1 12 12 24 +all_1_2_1 1 13 13 26 +all_1_2_1 1 14 14 28 +all_1_2_1 1 15 15 30 +=========== +all_1_3_2 1 0 0 0 +all_1_3_2 1 1 2 2 +all_1_3_2 1 2 4 4 +all_1_3_2 1 3 6 6 +all_1_3_2 1 4 8 8 +all_1_3_2 1 5 10 10 +all_1_3_2 1 6 12 12 +all_1_3_2 1 7 14 14 +all_1_3_2 1 8 16 16 +all_1_3_2 1 9 18 18 +all_1_3_2 1 10 20 20 +all_1_3_2 1 11 22 22 +all_1_3_2 1 12 24 24 +all_1_3_2 1 13 26 26 +all_1_3_2 1 14 28 28 +all_1_3_2 1 15 30 30 +all_1_3_2 3 0 1 1 +all_1_3_2 3 1 3 3 +all_1_3_2 3 2 5 5 +all_1_3_2 3 3 7 7 +all_1_3_2 3 4 9 9 +all_1_3_2 3 5 11 11 +all_1_3_2 3 6 13 13 +all_1_3_2 3 7 15 15 +all_1_3_2 3 8 17 17 +all_1_3_2 3 9 19 19 +all_1_3_2 3 10 21 21 +all_1_3_2 3 11 23 23 +all_1_3_2 3 12 25 25 +all_1_3_2 3 13 27 27 +all_1_3_2 3 14 29 29 +all_1_3_2 3 15 31 31 diff --git a/tests/queries/0_stateless/03001_block_offset_column_2.sql b/tests/queries/0_stateless/03001_block_offset_column_2.sql new file mode 100644 index 00000000000..b994e37b952 --- /dev/null +++ b/tests/queries/0_stateless/03001_block_offset_column_2.sql @@ -0,0 +1,25 @@ + +DROP TABLE IF EXISTS t_block_offset; + +CREATE TABLE t_block_offset (id UInt32) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 3; + +INSERT INTO t_block_offset SELECT number * 2 FROM numbers(8); + +INSERT INTO t_block_offset SELECT number * 2 FROM numbers(8, 8); + +OPTIMIZE TABLE t_block_offset FINAL; + +SELECT _part, _block_number, _block_offset, _part_offset, id FROM t_block_offset ORDER BY _block_number, _block_offset; + +ALTER TABLE t_block_offset MODIFY SETTING enable_block_number_column = 1; +ALTER TABLE t_block_offset MODIFY SETTING enable_block_offset_column = 1; + +INSERT INTO t_block_offset SELECT number * 2 + 1 FROM numbers(16); + +OPTIMIZE TABLE t_block_offset FINAL; + +SELECT '==========='; +SELECT _part, _block_number, _block_offset, _part_offset, id FROM t_block_offset ORDER BY _block_number, _block_offset; + + +DROP TABLE t_block_offset; diff --git a/tests/queries/0_stateless/03001_data_version_column.reference b/tests/queries/0_stateless/03001_data_version_column.reference new file mode 100644 index 00000000000..cd85ce9cc63 --- /dev/null +++ b/tests/queries/0_stateless/03001_data_version_column.reference @@ -0,0 +1,5 @@ +all_1_1_0 1 1 1 +all_2_2_0 2 2 2 +all_1_1_0_3 3 1 100 +all_2_2_0_3 3 2 200 +all_4_4_0 4 3 3 diff --git a/tests/queries/0_stateless/03001_data_version_column.sql b/tests/queries/0_stateless/03001_data_version_column.sql new file mode 100644 index 00000000000..4e3377ebf47 --- /dev/null +++ b/tests/queries/0_stateless/03001_data_version_column.sql @@ -0,0 +1,20 @@ +DROP TABLE IF EXISTS t_data_version; + +CREATE TABLE t_data_version (a UInt64, b UInt64) ENGINE = MergeTree ORDER BY a; + +INSERT INTO t_data_version VALUES (1, 1); +INSERT INTO t_data_version VALUES (2, 2); + +SELECT _part, _part_data_version, * FROM t_data_version ORDER BY a; + +ALTER TABLE t_data_version UPDATE b = a * 100 WHERE 1 SETTINGS mutations_sync = 2; + +SELECT _part, _part_data_version, * FROM t_data_version ORDER BY a; + +INSERT INTO t_data_version VALUES (3, 3); + +-- Check parts pruning. +SELECT _part, _part_data_version, * FROM t_data_version WHERE _part_data_version = 4 ORDER BY a SETTINGS max_rows_to_read = 1; + +DROP TABLE t_data_version; + diff --git a/tests/queries/0_stateless/03001_max_parallel_replicas_zero_value.reference b/tests/queries/0_stateless/03001_max_parallel_replicas_zero_value.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/03001_max_parallel_replicas_zero_value.sql b/tests/queries/0_stateless/03001_max_parallel_replicas_zero_value.sql new file mode 100644 index 00000000000..499486713a6 --- /dev/null +++ b/tests/queries/0_stateless/03001_max_parallel_replicas_zero_value.sql @@ -0,0 +1,5 @@ +drop table if exists test_d; +create table test_d engine=Distributed(test_cluster_two_shard_three_replicas_localhost, system, numbers); +select * from test_d limit 10 settings max_parallel_replicas = 0, prefer_localhost_replica = 0; --{clientError BAD_ARGUMENTS} +drop table test_d; + diff --git a/tests/queries/0_stateless/03002_filter_skip_virtual_columns_with_non_deterministic_functions.sql b/tests/queries/0_stateless/03002_filter_skip_virtual_columns_with_non_deterministic_functions.sql index 9f8bc6bd3d7..8ccc3cf61da 100644 --- a/tests/queries/0_stateless/03002_filter_skip_virtual_columns_with_non_deterministic_functions.sql +++ b/tests/queries/0_stateless/03002_filter_skip_virtual_columns_with_non_deterministic_functions.sql @@ -1,6 +1,5 @@ create table test (number UInt64) engine=MergeTree order by number; -insert into test select * from numbers(100000000); +insert into test select * from numbers(50000000); select ignore(number) from test where RAND() > 4292390314 limit 10; select count() > 0 from test where RAND() > 4292390314; drop table test; - diff --git a/tests/queries/0_stateless/03002_sample_factor_where.reference b/tests/queries/0_stateless/03002_sample_factor_where.reference new file mode 100644 index 00000000000..2f0d8589603 --- /dev/null +++ b/tests/queries/0_stateless/03002_sample_factor_where.reference @@ -0,0 +1,3 @@ +2 +0 +0 diff --git a/tests/queries/0_stateless/03002_sample_factor_where.sql b/tests/queries/0_stateless/03002_sample_factor_where.sql new file mode 100644 index 00000000000..6430034349a --- /dev/null +++ b/tests/queries/0_stateless/03002_sample_factor_where.sql @@ -0,0 +1,11 @@ +DROP TABLE IF EXISTS t_sample_factor; + +CREATE TABLE t_sample_factor(a UInt64, b UInt64) ENGINE = MergeTree ORDER BY (a, b) SAMPLE BY b; +INSERT INTO t_sample_factor(a, b) VALUES (1, 2), (3, 4); + +SELECT uniq(b) * any(_sample_factor) FROM t_sample_factor SAMPLE 200000; + +SELECT uniq(b) * any(_sample_factor) FROM t_sample_factor SAMPLE 200000 WHERE a < -1; +SELECT uniq(b) * any(_sample_factor) FROM t_sample_factor SAMPLE 200000 PREWHERE a < -1; + +DROP TABLE t_sample_factor; diff --git a/tests/queries/0_stateless/03006_correct_revoke_for_partial_rights.sh b/tests/queries/0_stateless/03006_correct_revoke_for_partial_rights.sh index b58dda8648a..8c79dfdbafc 100755 --- a/tests/queries/0_stateless/03006_correct_revoke_for_partial_rights.sh +++ b/tests/queries/0_stateless/03006_correct_revoke_for_partial_rights.sh @@ -5,8 +5,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh db=${CLICKHOUSE_DATABASE} -user1="user1_03006_$db_$RANDOM" -user2="user2_03006_$db_$RANDOM" +user1="user1_03006_${db}_$RANDOM" +user2="user2_03006_${db}_$RANDOM" ${CLICKHOUSE_CLIENT} --multiquery <