diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index c6555fd9f91..2ef05fe989b 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -15,7 +15,8 @@ jobs:
- name: Deploy packages and assets
run: |
GITHUB_TAG="${GITHUB_REF#refs/tags/}"
- curl '${{ secrets.PACKAGES_RELEASE_URL }}/release/'"${GITHUB_TAG}"'?binary=binary_darwin&binary=binary_darwin_aarch64&sync=true' -d ''
+ curl --silent --data '' \
+ '${{ secrets.PACKAGES_RELEASE_URL }}/release/'"${GITHUB_TAG}"'?binary=binary_darwin&binary=binary_darwin_aarch64&sync=true'
############################################################################################
##################################### Docker images #######################################
############################################################################################
diff --git a/.gitmodules b/.gitmodules
index b4673f113b7..e395860d957 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -330,3 +330,6 @@
[submodule "contrib/crc32-vpmsum"]
path = contrib/crc32-vpmsum
url = https://github.com/antonblanchard/crc32-vpmsum.git
+[submodule "contrib/liburing"]
+ path = contrib/liburing
+ url = https://github.com/axboe/liburing
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4200d583a3f..a89619aa7ca 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,43 +13,45 @@
* The `PREALLOCATE` option for `HASHED`/`SPARSE_HASHED` dictionaries becomes a no-op. [#45388](https://github.com/ClickHouse/ClickHouse/pull/45388) ([Azat Khuzhin](https://github.com/azat)). It does not give significant advantages anymore.
* Disallow `Gorilla` codec on columns of non-Float32 or non-Float64 type. [#45252](https://github.com/ClickHouse/ClickHouse/pull/45252) ([Robert Schulze](https://github.com/rschu1ze)). It was pointless and led to inconsistencies.
* Parallel quorum inserts might work incorrectly with `*MergeTree` tables created with the deprecated syntax. Therefore, parallel quorum inserts support is completely disabled for such tables. It does not affect tables created with a new syntax. [#45430](https://github.com/ClickHouse/ClickHouse/pull/45430) ([Alexander Tokmakov](https://github.com/tavplubix)).
-* Use `GetObjectAttributes` request instead of `HeadObject` request to get the size of an object in AWS S3. This change fixes handling endpoints without explicit region after updating the AWS SDK, for example. [#45288](https://github.com/ClickHouse/ClickHouse/pull/45288) ([Vitaly Baranov](https://github.com/vitlibar)). AWS S3 and Minio are tested, but keep in mind that various S3-compatible services (GCS, R2, B2) may have subtle incompatibilities. This change also may require you to adjust the ACL to allow the `GetObjectAttributes` request.
+* Use the `GetObjectAttributes` request instead of the `HeadObject` request to get the size of an object in AWS S3. This change fixes handling endpoints without explicit regions after updating the AWS SDK, for example. [#45288](https://github.com/ClickHouse/ClickHouse/pull/45288) ([Vitaly Baranov](https://github.com/vitlibar)). AWS S3 and Minio are tested, but keep in mind that various S3-compatible services (GCS, R2, B2) may have subtle incompatibilities. This change also may require you to adjust the ACL to allow the `GetObjectAttributes` request.
* Forbid paths in timezone names. For example, a timezone name like `/usr/share/zoneinfo/Asia/Aden` is not allowed; the IANA timezone database name like `Asia/Aden` should be used. [#44225](https://github.com/ClickHouse/ClickHouse/pull/44225) ([Kruglov Pavel](https://github.com/Avogar)).
+* Queries combining equijoin and constant expressions (e.g., `JOIN ON t1.x = t2.x AND 1 = 1`) are forbidden due to incorrect results. [#44016](https://github.com/ClickHouse/ClickHouse/pull/44016) ([Vladimir C](https://github.com/vdimir)).
+
#### New Feature
* Dictionary source for extracting keys by traversing regular expressions tree. It can be used for User-Agent parsing. [#40878](https://github.com/ClickHouse/ClickHouse/pull/40878) ([Vage Ogannisian](https://github.com/nooblose)). [#43858](https://github.com/ClickHouse/ClickHouse/pull/43858) ([Han Fei](https://github.com/hanfei1991)).
-* Added parametrized view functionality, now it's possible to specify query parameters for View table engine. resolves [#40907](https://github.com/ClickHouse/ClickHouse/issues/40907). [#41687](https://github.com/ClickHouse/ClickHouse/pull/41687) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Added parametrized view functionality, now it's possible to specify query parameters for the View table engine. resolves [#40907](https://github.com/ClickHouse/ClickHouse/issues/40907). [#41687](https://github.com/ClickHouse/ClickHouse/pull/41687) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
* Add `quantileInterpolatedWeighted`/`quantilesInterpolatedWeighted` functions. [#38252](https://github.com/ClickHouse/ClickHouse/pull/38252) ([Bharat Nallan](https://github.com/bharatnc)).
* Array join support for the `Map` type, like the function "explode" in Spark. [#43239](https://github.com/ClickHouse/ClickHouse/pull/43239) ([李扬](https://github.com/taiyang-li)).
* Support SQL standard binary and hex string literals. [#43785](https://github.com/ClickHouse/ClickHouse/pull/43785) ([Mo Xuan](https://github.com/mo-avatar)).
-* Allow to format `DateTime` in Joda-Time style. Refer to [the Joda-Time docs](https://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html). [#43818](https://github.com/ClickHouse/ClickHouse/pull/43818) ([李扬](https://github.com/taiyang-li)).
+* Allow formatting `DateTime` in Joda-Time style. Refer to [the Joda-Time docs](https://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html). [#43818](https://github.com/ClickHouse/ClickHouse/pull/43818) ([李扬](https://github.com/taiyang-li)).
* Implemented a fractional second formatter (`%f`) for `formatDateTime`. [#44060](https://github.com/ClickHouse/ClickHouse/pull/44060) ([ltrk2](https://github.com/ltrk2)). [#44497](https://github.com/ClickHouse/ClickHouse/pull/44497) ([Alexander Gololobov](https://github.com/davenger)).
-* Added `age` function to calculate difference between two dates or dates with time values expressed as number of full units. Closes [#41115](https://github.com/ClickHouse/ClickHouse/issues/41115). [#44421](https://github.com/ClickHouse/ClickHouse/pull/44421) ([Robert Schulze](https://github.com/rschu1ze)).
+* Added `age` function to calculate the difference between two dates or dates with time values expressed as the number of full units. Closes [#41115](https://github.com/ClickHouse/ClickHouse/issues/41115). [#44421](https://github.com/ClickHouse/ClickHouse/pull/44421) ([Robert Schulze](https://github.com/rschu1ze)).
* Add `Null` source for dictionaries. Closes [#44240](https://github.com/ClickHouse/ClickHouse/issues/44240). [#44502](https://github.com/ClickHouse/ClickHouse/pull/44502) ([mayamika](https://github.com/mayamika)).
* Allow configuring the S3 storage class with the `s3_storage_class` configuration option. Such as `STANDARD/INTELLIGENT_TIERING` Closes [#44443](https://github.com/ClickHouse/ClickHouse/issues/44443). [#44707](https://github.com/ClickHouse/ClickHouse/pull/44707) ([chen](https://github.com/xiedeyantu)).
* Insert default values in case of missing elements in JSON object while parsing named tuple. Add setting `input_format_json_defaults_for_missing_elements_in_named_tuple` that controls this behaviour. Closes [#45142](https://github.com/ClickHouse/ClickHouse/issues/45142)#issuecomment-1380153217. [#45231](https://github.com/ClickHouse/ClickHouse/pull/45231) ([Kruglov Pavel](https://github.com/Avogar)).
* Record server startup time in ProfileEvents (`ServerStartupMilliseconds`). Resolves [#43188](https://github.com/ClickHouse/ClickHouse/issues/43188). [#45250](https://github.com/ClickHouse/ClickHouse/pull/45250) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
-* Refactor and Improve streaming engines Kafka/RabbitMQ/NATS and add support for all formats, also refactor formats a bit: - Fix producing messages in row-based formats with suffixes/prefixes. Now every message is formatted completely with all delimiters and can be parsed back using input format. - Support block-based formats like Native, Parquet, ORC, etc. Every block is formatted as a separated message. The number of rows in one message depends on the block size, so you can control it via setting `max_block_size`. - Add new engine settings `kafka_max_rows_per_message/rabbitmq_max_rows_per_message/nats_max_rows_per_message`. They control the number of rows formatted in one message in row-based formats. Default value: 1. - Fix high memory consumption in NATS table engine. - Support arbitrary binary data in NATS producer (previously it worked only with strings contained \0 at the end) - Add missing Kafka/RabbitMQ/NATS engine settings in documentation. - Refactor producing and consuming in Kafka/RabbitMQ/NATS, separate it from WriteBuffers/ReadBuffers semantic. - Refactor output formats: remove callbacks on each row used in Kafka/RabbitMQ/NATS (now we don't use callbacks there), allow to use IRowOutputFormat directly, clarify row end and row between delimiters, make it possible to reset output format to start formatting again - Add proper implementation in formatRow function (bonus after formats refactoring). [#42777](https://github.com/ClickHouse/ClickHouse/pull/42777) ([Kruglov Pavel](https://github.com/Avogar)).
+* Refactor and Improve streaming engines Kafka/RabbitMQ/NATS and add support for all formats, also refactor formats a bit: - Fix producing messages in row-based formats with suffixes/prefixes. Now every message is formatted completely with all delimiters and can be parsed back using input format. - Support block-based formats like Native, Parquet, ORC, etc. Every block is formatted as a separate message. The number of rows in one message depends on the block size, so you can control it via the setting `max_block_size`. - Add new engine settings `kafka_max_rows_per_message/rabbitmq_max_rows_per_message/nats_max_rows_per_message`. They control the number of rows formatted in one message in row-based formats. Default value: 1. - Fix high memory consumption in the NATS table engine. - Support arbitrary binary data in NATS producer (previously it worked only with strings contained \0 at the end) - Add missing Kafka/RabbitMQ/NATS engine settings in the documentation. - Refactor producing and consuming in Kafka/RabbitMQ/NATS, separate it from WriteBuffers/ReadBuffers semantic. - Refactor output formats: remove callbacks on each row used in Kafka/RabbitMQ/NATS (now we don't use callbacks there), allow to use IRowOutputFormat directly, clarify row end and row between delimiters, make it possible to reset output format to start formatting again - Add proper implementation in formatRow function (bonus after formats refactoring). [#42777](https://github.com/ClickHouse/ClickHouse/pull/42777) ([Kruglov Pavel](https://github.com/Avogar)).
* Support reading/writing `Nested` tables as `List` of `Struct` in `CapnProto` format. Read/write `Decimal32/64` as `Int32/64`. Closes [#43319](https://github.com/ClickHouse/ClickHouse/issues/43319). [#43379](https://github.com/ClickHouse/ClickHouse/pull/43379) ([Kruglov Pavel](https://github.com/Avogar)).
-* Added a `message_format_string` column to `system.text_log`. The column contains a pattern that was used to format the message. [#44543](https://github.com/ClickHouse/ClickHouse/pull/44543) ([Alexander Tokmakov](https://github.com/tavplubix)). This allows various analytics over ClickHouse own logs.
-* Try to autodetect header with column names (and maybe types) for CSV/TSV/CustomSeparated input formats.
-Add settings input_format_tsv/csv/custom_detect_header that enables this behaviour (enabled by default). Closes [#44640](https://github.com/ClickHouse/ClickHouse/issues/44640). [#44953](https://github.com/ClickHouse/ClickHouse/pull/44953) ([Kruglov Pavel](https://github.com/Avogar)).
+* Added a `message_format_string` column to `system.text_log`. The column contains a pattern that was used to format the message. [#44543](https://github.com/ClickHouse/ClickHouse/pull/44543) ([Alexander Tokmakov](https://github.com/tavplubix)). This allows various analytics over the ClickHouse logs.
+* Try to autodetect headers with column names (and maybe types) for CSV/TSV/CustomSeparated input formats.
+Add settings input_format_tsv/csv/custom_detect_header that enable this behaviour (enabled by default). Closes [#44640](https://github.com/ClickHouse/ClickHouse/issues/44640). [#44953](https://github.com/ClickHouse/ClickHouse/pull/44953) ([Kruglov Pavel](https://github.com/Avogar)).
#### Experimental Feature
* Add an experimental inverted index as a new secondary index type for efficient text search. [#38667](https://github.com/ClickHouse/ClickHouse/pull/38667) ([larryluogit](https://github.com/larryluogit)).
* Add experimental query result cache. [#43797](https://github.com/ClickHouse/ClickHouse/pull/43797) ([Robert Schulze](https://github.com/rschu1ze)).
* Added extendable and configurable scheduling subsystem for IO requests (not yet integrated with IO code itself). [#41840](https://github.com/ClickHouse/ClickHouse/pull/41840) ([Sergei Trifonov](https://github.com/serxa)). This feature does nothing at all, enjoy.
-* Added `SYSTEM DROP DATABASE REPLICA` that removes metadata of dead replica of `Replicated` database. Resolves [#41794](https://github.com/ClickHouse/ClickHouse/issues/41794). [#42807](https://github.com/ClickHouse/ClickHouse/pull/42807) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Added `SYSTEM DROP DATABASE REPLICA` that removes metadata of a dead replica of a `Replicated` database. Resolves [#41794](https://github.com/ClickHouse/ClickHouse/issues/41794). [#42807](https://github.com/ClickHouse/ClickHouse/pull/42807) ([Alexander Tokmakov](https://github.com/tavplubix)).
#### Performance Improvement
* Do not load inactive parts at startup of `MergeTree` tables. [#42181](https://github.com/ClickHouse/ClickHouse/pull/42181) ([Anton Popov](https://github.com/CurtizJ)).
-* Improved latency of reading from storage `S3` and table function `s3` with large number of small files. Now settings `remote_filesystem_read_method` and `remote_filesystem_read_prefetch` take effect while reading from storage `S3`. [#43726](https://github.com/ClickHouse/ClickHouse/pull/43726) ([Anton Popov](https://github.com/CurtizJ)).
+* Improved latency of reading from storage `S3` and table function `s3` with large numbers of small files. Now settings `remote_filesystem_read_method` and `remote_filesystem_read_prefetch` take effect while reading from storage `S3`. [#43726](https://github.com/ClickHouse/ClickHouse/pull/43726) ([Anton Popov](https://github.com/CurtizJ)).
* Optimization for reading struct fields in Parquet/ORC files. Only the required fields are loaded. [#44484](https://github.com/ClickHouse/ClickHouse/pull/44484) ([lgbo](https://github.com/lgbo-ustc)).
-* Two-level aggregation algorithm was mistakenly disabled for queries over HTTP interface. It was enabled back, and it leads to a major performance improvement. [#45450](https://github.com/ClickHouse/ClickHouse/pull/45450) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Two-level aggregation algorithm was mistakenly disabled for queries over the HTTP interface. It was enabled back, and it leads to a major performance improvement. [#45450](https://github.com/ClickHouse/ClickHouse/pull/45450) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Added mmap support for StorageFile, which should improve the performance of clickhouse-local. [#43927](https://github.com/ClickHouse/ClickHouse/pull/43927) ([pufit](https://github.com/pufit)).
* Added sharding support in HashedDictionary to allow parallel load (almost linear scaling based on number of shards). [#40003](https://github.com/ClickHouse/ClickHouse/pull/40003) ([Azat Khuzhin](https://github.com/azat)).
* Speed up query parsing. [#42284](https://github.com/ClickHouse/ClickHouse/pull/42284) ([Raúl Marín](https://github.com/Algunenano)).
-* Always replace OR chain `expr = x1 OR ... OR expr = xN` to `expr IN (x1, ..., xN)` in case if `expr` is a `LowCardinality` column. Setting `optimize_min_equality_disjunction_chain_length` is ignored in this case. [#42889](https://github.com/ClickHouse/ClickHouse/pull/42889) ([Guo Wangyang](https://github.com/guowangy)).
+* Always replace OR chain `expr = x1 OR ... OR expr = xN` to `expr IN (x1, ..., xN)` in the case where `expr` is a `LowCardinality` column. Setting `optimize_min_equality_disjunction_chain_length` is ignored in this case. [#42889](https://github.com/ClickHouse/ClickHouse/pull/42889) ([Guo Wangyang](https://github.com/guowangy)).
* Slightly improve performance by optimizing the code around ThreadStatus. [#43586](https://github.com/ClickHouse/ClickHouse/pull/43586) ([Zhiguo Zhou](https://github.com/ZhiguoZh)).
* Optimize the column-wise ternary logic evaluation by achieving auto-vectorization. In the performance test of this [microbenchmark](https://github.com/ZhiguoZh/ClickHouse/blob/20221123-ternary-logic-opt-example/src/Functions/examples/associative_applier_perf.cpp), we've observed a peak **performance gain** of **21x** on the ICX device (Intel Xeon Platinum 8380 CPU). [#43669](https://github.com/ClickHouse/ClickHouse/pull/43669) ([Zhiguo Zhou](https://github.com/ZhiguoZh)).
* Avoid acquiring read locks in the `system.tables` table if possible. [#43840](https://github.com/ClickHouse/ClickHouse/pull/43840) ([Raúl Marín](https://github.com/Algunenano)).
@@ -59,10 +61,10 @@ Add settings input_format_tsv/csv/custom_detect_header that enables this behavio
* Add fast path for: - `col like '%%'`; - `col like '%'`; - `col not like '%'`; - `col not like '%'`; - `match(col, '.*')`. [#45244](https://github.com/ClickHouse/ClickHouse/pull/45244) ([李扬](https://github.com/taiyang-li)).
* Slightly improve happy path optimisation in filtering (WHERE clause). [#45289](https://github.com/ClickHouse/ClickHouse/pull/45289) ([Nikita Taranov](https://github.com/nickitat)).
* Provide monotonicity info for `toUnixTimestamp64*` to enable more algebraic optimizations for index analysis. [#44116](https://github.com/ClickHouse/ClickHouse/pull/44116) ([Nikita Taranov](https://github.com/nickitat)).
-* Allow to configure temporary data for query processing (spilling to disk) to cooperate with filesystem cache (taking up the space from the cache disk) [#43972](https://github.com/ClickHouse/ClickHouse/pull/43972) ([Vladimir C](https://github.com/vdimir)). This mainly improves [ClickHouse Cloud](https://clickhouse.cloud/), but can be used for self-managed setups as well, if you know what to do.
+* Allow the configuration of temporary data for query processing (spilling to disk) to cooperate with the filesystem cache (taking up the space from the cache disk) [#43972](https://github.com/ClickHouse/ClickHouse/pull/43972) ([Vladimir C](https://github.com/vdimir)). This mainly improves [ClickHouse Cloud](https://clickhouse.cloud/), but can be used for self-managed setups as well, if you know what to do.
* Make `system.replicas` table do parallel fetches of replicas statuses. Closes [#43918](https://github.com/ClickHouse/ClickHouse/issues/43918). [#43998](https://github.com/ClickHouse/ClickHouse/pull/43998) ([Nikolay Degterinsky](https://github.com/evillique)).
* Optimize memory consumption during backup to S3: files to S3 now will be copied directly without using `WriteBufferFromS3` (which could use a lot of memory). [#45188](https://github.com/ClickHouse/ClickHouse/pull/45188) ([Vitaly Baranov](https://github.com/vitlibar)).
-* Add a cache for async block ids. This will reduce the requests of zookeeper when we enable async inserts deduplication. [#45106](https://github.com/ClickHouse/ClickHouse/pull/45106) ([Han Fei](https://github.com/hanfei1991)).
+* Add a cache for async block ids. This will reduce the number of requests of ZooKeeper when we enable async inserts deduplication. [#45106](https://github.com/ClickHouse/ClickHouse/pull/45106) ([Han Fei](https://github.com/hanfei1991)).
#### Improvement
@@ -71,25 +73,25 @@ Add settings input_format_tsv/csv/custom_detect_header that enables this behavio
* Added fields `supports_parallel_parsing` and `supports_parallel_formatting` to table `system.formats` for better introspection. [#45499](https://github.com/ClickHouse/ClickHouse/pull/45499) ([Anton Popov](https://github.com/CurtizJ)).
* Improve reading CSV field in CustomSeparated/Template format. Closes [#42352](https://github.com/ClickHouse/ClickHouse/issues/42352) Closes [#39620](https://github.com/ClickHouse/ClickHouse/issues/39620). [#43332](https://github.com/ClickHouse/ClickHouse/pull/43332) ([Kruglov Pavel](https://github.com/Avogar)).
* Unify query elapsed time measurements. [#43455](https://github.com/ClickHouse/ClickHouse/pull/43455) ([Raúl Marín](https://github.com/Algunenano)).
-* Improve automatic usage of structure from insertion table in table functions file/hdfs/s3 when virtual columns present in select query, it fixes possible error `Block structure mismatch` or `number of columns mismatch`. [#43695](https://github.com/ClickHouse/ClickHouse/pull/43695) ([Kruglov Pavel](https://github.com/Avogar)).
-* Add support for signed arguments in function `range`. Fixes [#43333](https://github.com/ClickHouse/ClickHouse/issues/43333). [#43733](https://github.com/ClickHouse/ClickHouse/pull/43733) ([sanyu](https://github.com/wineternity)).
+* Improve automatic usage of structure from insertion table in table functions file/hdfs/s3 when virtual columns are present in a select query, it fixes the possible error `Block structure mismatch` or `number of columns mismatch`. [#43695](https://github.com/ClickHouse/ClickHouse/pull/43695) ([Kruglov Pavel](https://github.com/Avogar)).
+* Add support for signed arguments in the function `range`. Fixes [#43333](https://github.com/ClickHouse/ClickHouse/issues/43333). [#43733](https://github.com/ClickHouse/ClickHouse/pull/43733) ([sanyu](https://github.com/wineternity)).
* Remove redundant sorting, for example, sorting related ORDER BY clauses in subqueries. Implemented on top of query plan. It does similar optimization as `optimize_duplicate_order_by_and_distinct` regarding `ORDER BY` clauses, but more generic, since it's applied to any redundant sorting steps (not only caused by ORDER BY clause) and applied to subqueries of any depth. Related to [#42648](https://github.com/ClickHouse/ClickHouse/issues/42648). [#43905](https://github.com/ClickHouse/ClickHouse/pull/43905) ([Igor Nikonov](https://github.com/devcrafter)).
-* Add ability to disable deduplication of files for BACKUP (for backups wiithout deduplication ATTACH can be used instead of full RESTORE), example `BACKUP foo TO S3(...) SETTINGS deduplicate_files=0` (default `deduplicate_files=1`). [#43947](https://github.com/ClickHouse/ClickHouse/pull/43947) ([Azat Khuzhin](https://github.com/azat)).
+* Add the ability to disable deduplication of files for BACKUP (for backups without deduplication ATTACH can be used instead of full RESTORE). For example `BACKUP foo TO S3(...) SETTINGS deduplicate_files=0` (default `deduplicate_files=1`). [#43947](https://github.com/ClickHouse/ClickHouse/pull/43947) ([Azat Khuzhin](https://github.com/azat)).
* Refactor and improve schema inference for text formats. Add new setting `schema_inference_make_columns_nullable` that controls making result types `Nullable` (enabled by default);. [#44019](https://github.com/ClickHouse/ClickHouse/pull/44019) ([Kruglov Pavel](https://github.com/Avogar)).
* Better support for `PROXYv1` protocol. [#44135](https://github.com/ClickHouse/ClickHouse/pull/44135) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
* Add information about the latest part check by cleanup threads into `system.parts` table. [#44244](https://github.com/ClickHouse/ClickHouse/pull/44244) ([Dmitry Novik](https://github.com/novikd)).
* Disable table functions in readonly mode for inserts. [#44290](https://github.com/ClickHouse/ClickHouse/pull/44290) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
-* Add a setting `simultaneous_parts_removal_limit` to allow to limit the number of parts being processed by one iteration of CleanupThread. [#44461](https://github.com/ClickHouse/ClickHouse/pull/44461) ([Dmitry Novik](https://github.com/novikd)).
-* If user only need virtual columns, we don't need to initialize ReadBufferFromS3. May be helpful to [#44246](https://github.com/ClickHouse/ClickHouse/issues/44246). [#44493](https://github.com/ClickHouse/ClickHouse/pull/44493) ([chen](https://github.com/xiedeyantu)).
+* Add a setting `simultaneous_parts_removal_limit` to allow limiting the number of parts being processed by one iteration of CleanupThread. [#44461](https://github.com/ClickHouse/ClickHouse/pull/44461) ([Dmitry Novik](https://github.com/novikd)).
+* Do not initialize ReadBufferFromS3 when only virtual columns are needed in a query. This may be helpful to [#44246](https://github.com/ClickHouse/ClickHouse/issues/44246). [#44493](https://github.com/ClickHouse/ClickHouse/pull/44493) ([chen](https://github.com/xiedeyantu)).
* Prevent duplicate column names hints. Closes [#44130](https://github.com/ClickHouse/ClickHouse/issues/44130). [#44519](https://github.com/ClickHouse/ClickHouse/pull/44519) ([Joanna Hulboj](https://github.com/jh0x)).
* Allow macro substitution in endpoint of disks. Resolve [#40951](https://github.com/ClickHouse/ClickHouse/issues/40951). [#44533](https://github.com/ClickHouse/ClickHouse/pull/44533) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
* Improve schema inference when `input_format_json_read_object_as_string` is enabled. [#44546](https://github.com/ClickHouse/ClickHouse/pull/44546) ([Kruglov Pavel](https://github.com/Avogar)).
-* Add a user-level setting `database_replicated_allow_replicated_engine_arguments` which allow to ban creation of `ReplicatedMergeTree` tables with arguments in `DatabaseReplicated`. [#44566](https://github.com/ClickHouse/ClickHouse/pull/44566) ([alesapin](https://github.com/alesapin)).
+* Add a user-level setting `database_replicated_allow_replicated_engine_arguments` which allows banning the creation of `ReplicatedMergeTree` tables with arguments in `DatabaseReplicated`. [#44566](https://github.com/ClickHouse/ClickHouse/pull/44566) ([alesapin](https://github.com/alesapin)).
* Prevent users from mistakenly specifying zero (invalid) value for `index_granularity`. This closes [#44536](https://github.com/ClickHouse/ClickHouse/issues/44536). [#44578](https://github.com/ClickHouse/ClickHouse/pull/44578) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Added possibility to set path to service keytab file in `keytab` parameter in `kerberos` section of config.xml. [#44594](https://github.com/ClickHouse/ClickHouse/pull/44594) ([Roman Vasin](https://github.com/rvasin)).
* Use already written part of the query for fuzzy search (pass to the `skim` library, which is written in Rust and linked statically to ClickHouse). [#44600](https://github.com/ClickHouse/ClickHouse/pull/44600) ([Azat Khuzhin](https://github.com/azat)).
* Enable `input_format_json_read_objects_as_strings` by default to be able to read nested JSON objects while JSON Object type is experimental. [#44657](https://github.com/ClickHouse/ClickHouse/pull/44657) ([Kruglov Pavel](https://github.com/Avogar)).
-* Improvement for deduplication of async inserts: when users do duplicate async inserts, we should dedup inside the memory before we query keeper. [#44682](https://github.com/ClickHouse/ClickHouse/pull/44682) ([Han Fei](https://github.com/hanfei1991)).
+* Improvement for deduplication of async inserts: when users do duplicate async inserts, we should deduplicate inside the memory before we query Keeper. [#44682](https://github.com/ClickHouse/ClickHouse/pull/44682) ([Han Fei](https://github.com/hanfei1991)).
* Input/ouptut `Avro` format will parse bool type as ClickHouse bool type. [#44684](https://github.com/ClickHouse/ClickHouse/pull/44684) ([Kruglov Pavel](https://github.com/Avogar)).
* Support Bool type in Arrow/Parquet/ORC. Closes [#43970](https://github.com/ClickHouse/ClickHouse/issues/43970). [#44698](https://github.com/ClickHouse/ClickHouse/pull/44698) ([Kruglov Pavel](https://github.com/Avogar)).
* Don't greedily parse beyond the quotes when reading UUIDs - it may lead to mistakenly successful parsing of incorrect data. [#44686](https://github.com/ClickHouse/ClickHouse/pull/44686) ([Raúl Marín](https://github.com/Algunenano)).
@@ -98,7 +100,7 @@ Add settings input_format_tsv/csv/custom_detect_header that enables this behavio
* Fix `output_format_pretty_row_numbers` does not preserve the counter across the blocks. Closes [#44815](https://github.com/ClickHouse/ClickHouse/issues/44815). [#44832](https://github.com/ClickHouse/ClickHouse/pull/44832) ([flynn](https://github.com/ucasfl)).
* Don't report errors in `system.errors` due to parts being merged concurrently with the background cleanup process. [#44874](https://github.com/ClickHouse/ClickHouse/pull/44874) ([Raúl Marín](https://github.com/Algunenano)).
* Optimize and fix metrics for Distributed async INSERT. [#44922](https://github.com/ClickHouse/ClickHouse/pull/44922) ([Azat Khuzhin](https://github.com/azat)).
-* Added settings to disallow concurrent backups and restores resolves [#43891](https://github.com/ClickHouse/ClickHouse/issues/43891) Implementation: * Added server level settings to disallow concurrent backups and restores, which are read and set when BackupWorker is created in Context. * Settings are set to true by default. * Before starting backup or restores, added a check to see if any other backups/restores are running. For internal request it checks if its from the self node using backup_uuid. [#45072](https://github.com/ClickHouse/ClickHouse/pull/45072) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Added settings to disallow concurrent backups and restores resolves [#43891](https://github.com/ClickHouse/ClickHouse/issues/43891) Implementation: * Added server-level settings to disallow concurrent backups and restores, which are read and set when BackupWorker is created in Context. * Settings are set to true by default. * Before starting backup or restores, added a check to see if any other backups/restores are running. For internal requests, it checks if it is from the self node using backup_uuid. [#45072](https://github.com/ClickHouse/ClickHouse/pull/45072) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
* Add `` config parameter for system logs. [#45320](https://github.com/ClickHouse/ClickHouse/pull/45320) ([Stig Bakken](https://github.com/stigsb)).
#### Build/Testing/Packaging Improvement
@@ -114,22 +116,21 @@ Add settings input_format_tsv/csv/custom_detect_header that enables this behavio
#### Bug Fix
* Replace domain IP types (IPv4, IPv6) with native. [#43221](https://github.com/ClickHouse/ClickHouse/pull/43221) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). It automatically fixes some missing implementations in the code.
-* Fix backup process if mutations get killed during the backup process. [#45351](https://github.com/ClickHouse/ClickHouse/pull/45351) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix the backup process if mutations get killed during the backup process. [#45351](https://github.com/ClickHouse/ClickHouse/pull/45351) ([Vitaly Baranov](https://github.com/vitlibar)).
* Fix the `Invalid number of rows in Chunk` exception message. [#41404](https://github.com/ClickHouse/ClickHouse/issues/41404). [#42126](https://github.com/ClickHouse/ClickHouse/pull/42126) ([Alexander Gololobov](https://github.com/davenger)).
-* Fix possible use of uninitialized value after executing expressions after sorting. Closes [#43386](https://github.com/ClickHouse/ClickHouse/issues/43386) [#43635](https://github.com/ClickHouse/ClickHouse/pull/43635) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix possible use of an uninitialized value after executing expressions after sorting. Closes [#43386](https://github.com/ClickHouse/ClickHouse/issues/43386) [#43635](https://github.com/ClickHouse/ClickHouse/pull/43635) ([Kruglov Pavel](https://github.com/Avogar)).
* Better handling of NULL in aggregate combinators, fix possible segfault/logical error while using an obscure optimization `optimize_rewrite_sum_if_to_count_if`. Closes [#43758](https://github.com/ClickHouse/ClickHouse/issues/43758). [#43813](https://github.com/ClickHouse/ClickHouse/pull/43813) ([Kruglov Pavel](https://github.com/Avogar)).
* Fix CREATE USER/ROLE query settings constraints. [#43993](https://github.com/ClickHouse/ClickHouse/pull/43993) ([Nikolay Degterinsky](https://github.com/evillique)).
-* Fix wrong behavior of `JOIN ON t1.x = t2.x AND 1 = 1`, forbid such queries. [#44016](https://github.com/ClickHouse/ClickHouse/pull/44016) ([Vladimir C](https://github.com/vdimir)).
* Fixed bug with non-parsable default value for `EPHEMERAL` column in table metadata. [#44026](https://github.com/ClickHouse/ClickHouse/pull/44026) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
* Fix parsing of bad version from compatibility setting. [#44224](https://github.com/ClickHouse/ClickHouse/pull/44224) ([Kruglov Pavel](https://github.com/Avogar)).
* Bring interval subtraction from datetime in line with addition. [#44241](https://github.com/ClickHouse/ClickHouse/pull/44241) ([ltrk2](https://github.com/ltrk2)).
-* Remove limits on maximum size of the result for view. [#44261](https://github.com/ClickHouse/ClickHouse/pull/44261) ([lizhuoyu5](https://github.com/lzydmxy)).
+* Remove limits on the maximum size of the result for view. [#44261](https://github.com/ClickHouse/ClickHouse/pull/44261) ([lizhuoyu5](https://github.com/lzydmxy)).
* Fix possible logical error in cache if `do_not_evict_index_and_mrk_files=1`. Closes [#42142](https://github.com/ClickHouse/ClickHouse/issues/42142). [#44268](https://github.com/ClickHouse/ClickHouse/pull/44268) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Fix possible too early cache write interruption in write-through cache (caching could be stopped due to false assumption when it shouldn't have). [#44289](https://github.com/ClickHouse/ClickHouse/pull/44289) ([Kseniia Sumarokova](https://github.com/kssenii)).
-* Fix possible crash in case function `IN` with constant arguments was used as a constant argument together with `LowCardinality`. Fixes [#44221](https://github.com/ClickHouse/ClickHouse/issues/44221). [#44346](https://github.com/ClickHouse/ClickHouse/pull/44346) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix possible crash in the case function `IN` with constant arguments was used as a constant argument together with `LowCardinality`. Fixes [#44221](https://github.com/ClickHouse/ClickHouse/issues/44221). [#44346](https://github.com/ClickHouse/ClickHouse/pull/44346) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Fix support for complex parameters (like arrays) of parametric aggregate functions. This closes [#30975](https://github.com/ClickHouse/ClickHouse/issues/30975). The aggregate function `sumMapFiltered` was unusable in distributed queries before this change. [#44358](https://github.com/ClickHouse/ClickHouse/pull/44358) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Fix reading ObjectId in BSON schema inference. [#44382](https://github.com/ClickHouse/ClickHouse/pull/44382) ([Kruglov Pavel](https://github.com/Avogar)).
-* Fix race which can lead to premature temp parts removal before merge finished in ReplicatedMergeTree. This issue could lead to errors like `No such file or directory: xxx`. Fixes [#43983](https://github.com/ClickHouse/ClickHouse/issues/43983). [#44383](https://github.com/ClickHouse/ClickHouse/pull/44383) ([alesapin](https://github.com/alesapin)).
+* Fix race which can lead to premature temp parts removal before merge finishes in ReplicatedMergeTree. This issue could lead to errors like `No such file or directory: xxx`. Fixes [#43983](https://github.com/ClickHouse/ClickHouse/issues/43983). [#44383](https://github.com/ClickHouse/ClickHouse/pull/44383) ([alesapin](https://github.com/alesapin)).
* Some invalid `SYSTEM ... ON CLUSTER` queries worked in an unexpected way if a cluster name was not specified. It's fixed, now invalid queries throw `SYNTAX_ERROR` as they should. Fixes [#44264](https://github.com/ClickHouse/ClickHouse/issues/44264). [#44387](https://github.com/ClickHouse/ClickHouse/pull/44387) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Fix reading Map type in ORC format. [#44400](https://github.com/ClickHouse/ClickHouse/pull/44400) ([Kruglov Pavel](https://github.com/Avogar)).
* Fix reading columns that are not presented in input data in Parquet/ORC formats. Previously it could lead to error `INCORRECT_NUMBER_OF_COLUMNS`. Closes [#44333](https://github.com/ClickHouse/ClickHouse/issues/44333). [#44405](https://github.com/ClickHouse/ClickHouse/pull/44405) ([Kruglov Pavel](https://github.com/Avogar)).
@@ -137,49 +138,49 @@ Add settings input_format_tsv/csv/custom_detect_header that enables this behavio
* Placing profile settings after profile settings constraints in the configuration file made constraints ineffective. [#44411](https://github.com/ClickHouse/ClickHouse/pull/44411) ([Konstantin Bogdanov](https://github.com/thevar1able)).
* Fix `SYNTAX_ERROR` while running `EXPLAIN AST INSERT` queries with data. Closes [#44207](https://github.com/ClickHouse/ClickHouse/issues/44207). [#44413](https://github.com/ClickHouse/ClickHouse/pull/44413) ([save-my-heart](https://github.com/save-my-heart)).
* Fix reading bool value with CRLF in CSV format. Closes [#44401](https://github.com/ClickHouse/ClickHouse/issues/44401). [#44442](https://github.com/ClickHouse/ClickHouse/pull/44442) ([Kruglov Pavel](https://github.com/Avogar)).
-* Don't execute and/or/if/multiIf on LowCardinality dictionary, so the result type cannot be LowCardinality. It could lead to error `Illegal column ColumnLowCardinality` in some cases. Fixes [#43603](https://github.com/ClickHouse/ClickHouse/issues/43603). [#44469](https://github.com/ClickHouse/ClickHouse/pull/44469) ([Kruglov Pavel](https://github.com/Avogar)).
-* Fix mutations with setting `max_streams_for_merge_tree_reading`. [#44472](https://github.com/ClickHouse/ClickHouse/pull/44472) ([Anton Popov](https://github.com/CurtizJ)).
+* Don't execute and/or/if/multiIf on a LowCardinality dictionary, so the result type cannot be LowCardinality. It could lead to the error `Illegal column ColumnLowCardinality` in some cases. Fixes [#43603](https://github.com/ClickHouse/ClickHouse/issues/43603). [#44469](https://github.com/ClickHouse/ClickHouse/pull/44469) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix mutations with the setting `max_streams_for_merge_tree_reading`. [#44472](https://github.com/ClickHouse/ClickHouse/pull/44472) ([Anton Popov](https://github.com/CurtizJ)).
* Fix potential null pointer dereference with GROUPING SETS in ASTSelectQuery::formatImpl ([#43049](https://github.com/ClickHouse/ClickHouse/issues/43049)). [#44479](https://github.com/ClickHouse/ClickHouse/pull/44479) ([Robert Schulze](https://github.com/rschu1ze)).
* Validate types in table function arguments, CAST function arguments, JSONAsObject schema inference according to settings. [#44501](https://github.com/ClickHouse/ClickHouse/pull/44501) ([Kruglov Pavel](https://github.com/Avogar)).
* Fix IN function with LowCardinality and const column, close [#44503](https://github.com/ClickHouse/ClickHouse/issues/44503). [#44506](https://github.com/ClickHouse/ClickHouse/pull/44506) ([Duc Canh Le](https://github.com/canhld94)).
-* Fixed a bug in normalization of a `DEFAULT` expression in `CREATE TABLE` statement. The second argument of function `in` (or the right argument of operator `IN`) might be replaced with the result of its evaluation during CREATE query execution. Fixes [#44496](https://github.com/ClickHouse/ClickHouse/issues/44496). [#44547](https://github.com/ClickHouse/ClickHouse/pull/44547) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fixed a bug in the normalization of a `DEFAULT` expression in `CREATE TABLE` statement. The second argument of the function `in` (or the right argument of operator `IN`) might be replaced with the result of its evaluation during CREATE query execution. Fixes [#44496](https://github.com/ClickHouse/ClickHouse/issues/44496). [#44547](https://github.com/ClickHouse/ClickHouse/pull/44547) ([Alexander Tokmakov](https://github.com/tavplubix)).
* Projections do not work in presence of WITH ROLLUP, WITH CUBE and WITH TOTALS. In previous versions, a query produced an exception instead of skipping the usage of projections. This closes [#44614](https://github.com/ClickHouse/ClickHouse/issues/44614). This closes [#42772](https://github.com/ClickHouse/ClickHouse/issues/42772). [#44615](https://github.com/ClickHouse/ClickHouse/pull/44615) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Async blocks were not cleaned because the function `get all blocks sorted by time` didn't get async blocks. [#44651](https://github.com/ClickHouse/ClickHouse/pull/44651) ([Han Fei](https://github.com/hanfei1991)).
* Fix `LOGICAL_ERROR` `The top step of the right pipeline should be ExpressionStep` for JOIN with subquery, UNION, and TOTALS. Fixes [#43687](https://github.com/ClickHouse/ClickHouse/issues/43687). [#44673](https://github.com/ClickHouse/ClickHouse/pull/44673) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Avoid `std::out_of_range` exception in the Executable table engine. [#44681](https://github.com/ClickHouse/ClickHouse/pull/44681) ([Kruglov Pavel](https://github.com/Avogar)).
* Do not apply `optimize_syntax_fuse_functions` to quantiles on AST, close [#44712](https://github.com/ClickHouse/ClickHouse/issues/44712). [#44713](https://github.com/ClickHouse/ClickHouse/pull/44713) ([Vladimir C](https://github.com/vdimir)).
* Fix bug with wrong type in Merge table and PREWHERE, close [#43324](https://github.com/ClickHouse/ClickHouse/issues/43324). [#44716](https://github.com/ClickHouse/ClickHouse/pull/44716) ([Vladimir C](https://github.com/vdimir)).
-* Fix possible crash during shutdown (while destroying TraceCollector). Fixes [#44757](https://github.com/ClickHouse/ClickHouse/issues/44757). [#44758](https://github.com/ClickHouse/ClickHouse/pull/44758) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
-* Fix a possible crash in distributed query processing. The crash could happen if a query with totals or extremes returned an empty result and there are mismatched types in the Distrubuted and the local tables. Fixes [#44738](https://github.com/ClickHouse/ClickHouse/issues/44738). [#44760](https://github.com/ClickHouse/ClickHouse/pull/44760) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix a possible crash during shutdown (while destroying TraceCollector). Fixes [#44757](https://github.com/ClickHouse/ClickHouse/issues/44757). [#44758](https://github.com/ClickHouse/ClickHouse/pull/44758) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix a possible crash in distributed query processing. The crash could happen if a query with totals or extremes returned an empty result and there are mismatched types in the Distributed and the local tables. Fixes [#44738](https://github.com/ClickHouse/ClickHouse/issues/44738). [#44760](https://github.com/ClickHouse/ClickHouse/pull/44760) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Fix fsync for fetches (`min_compressed_bytes_to_fsync_after_fetch`)/small files (ttl.txt, columns.txt) in mutations (`min_rows_to_fsync_after_merge`/`min_compressed_bytes_to_fsync_after_merge`). [#44781](https://github.com/ClickHouse/ClickHouse/pull/44781) ([Azat Khuzhin](https://github.com/azat)).
* A rare race condition was possible when querying the `system.parts` or `system.parts_columns` tables in the presence of parts being moved between disks. Introduced in [#41145](https://github.com/ClickHouse/ClickHouse/issues/41145). [#44809](https://github.com/ClickHouse/ClickHouse/pull/44809) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Fix the error `Context has expired` which could appear with enabled projections optimization. Can be reproduced for queries with specific functions, like `dictHas/dictGet` which use context in runtime. Fixes [#44844](https://github.com/ClickHouse/ClickHouse/issues/44844). [#44850](https://github.com/ClickHouse/ClickHouse/pull/44850) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* A fix for `Cannot read all data` error which could happen while reading `LowCardinality` dictionary from remote fs. Fixes [#44709](https://github.com/ClickHouse/ClickHouse/issues/44709). [#44875](https://github.com/ClickHouse/ClickHouse/pull/44875) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Ignore cases when hardware monitor sensors cannot be read instead of showing a full exception message in logs. [#44895](https://github.com/ClickHouse/ClickHouse/pull/44895) ([Raúl Marín](https://github.com/Algunenano)).
-* Use `max_delay_to_insert` value in case calculated time to delay INSERT exceeds the setting value. Related to [#44902](https://github.com/ClickHouse/ClickHouse/issues/44902). [#44916](https://github.com/ClickHouse/ClickHouse/pull/44916) ([Igor Nikonov](https://github.com/devcrafter)).
+* Use `max_delay_to_insert` value in case the calculated time to delay INSERT exceeds the setting value. Related to [#44902](https://github.com/ClickHouse/ClickHouse/issues/44902). [#44916](https://github.com/ClickHouse/ClickHouse/pull/44916) ([Igor Nikonov](https://github.com/devcrafter)).
* Fix error `Different order of columns in UNION subquery` for queries with `UNION`. Fixes [#44866](https://github.com/ClickHouse/ClickHouse/issues/44866). [#44920](https://github.com/ClickHouse/ClickHouse/pull/44920) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Delay for INSERT can be calculated incorrectly, which can lead to always using `max_delay_to_insert` setting as delay instead of a correct value. Using simple formula `max_delay_to_insert * (parts_over_threshold/max_allowed_parts_over_threshold)` i.e. delay grows proportionally to parts over threshold. Closes [#44902](https://github.com/ClickHouse/ClickHouse/issues/44902). [#44954](https://github.com/ClickHouse/ClickHouse/pull/44954) ([Igor Nikonov](https://github.com/devcrafter)).
-* fix alter table ttl error when wide part has light weight delete mask. [#44959](https://github.com/ClickHouse/ClickHouse/pull/44959) ([Mingliang Pan](https://github.com/liangliangpan)).
+* Fix alter table TTL error when a wide part has the lightweight delete mask. [#44959](https://github.com/ClickHouse/ClickHouse/pull/44959) ([Mingliang Pan](https://github.com/liangliangpan)).
* Follow-up fix for Replace domain IP types (IPv4, IPv6) with native [#43221](https://github.com/ClickHouse/ClickHouse/issues/43221). [#45024](https://github.com/ClickHouse/ClickHouse/pull/45024) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
* Follow-up fix for Replace domain IP types (IPv4, IPv6) with native https://github.com/ClickHouse/ClickHouse/pull/43221. [#45043](https://github.com/ClickHouse/ClickHouse/pull/45043) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
* A buffer overflow was possible in the parser. Found by fuzzer. [#45047](https://github.com/ClickHouse/ClickHouse/pull/45047) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Fix possible cannot-read-all-data error in storage FileLog. Closes [#45051](https://github.com/ClickHouse/ClickHouse/issues/45051), [#38257](https://github.com/ClickHouse/ClickHouse/issues/38257). [#45057](https://github.com/ClickHouse/ClickHouse/pull/45057) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Memory efficient aggregation (setting `distributed_aggregation_memory_efficient`) is disabled when grouping sets are present in the query. [#45058](https://github.com/ClickHouse/ClickHouse/pull/45058) ([Nikita Taranov](https://github.com/nickitat)).
-* Fix `RANGE_HASHED` dictionary to count range columns as part of primary key during updates when `update_field` is specified. Closes [#44588](https://github.com/ClickHouse/ClickHouse/issues/44588). [#45061](https://github.com/ClickHouse/ClickHouse/pull/45061) ([Maksim Kita](https://github.com/kitaisreal)).
-* Fix error `Cannot capture column` for `LowCardinality` captured argument of nested labmda. Fixes [#45028](https://github.com/ClickHouse/ClickHouse/issues/45028). [#45065](https://github.com/ClickHouse/ClickHouse/pull/45065) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
-* Fix the wrong query result of `additional_table_filters` (additional filter was not applied) in case if minmax/count projection is used. [#45133](https://github.com/ClickHouse/ClickHouse/pull/45133) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix `RANGE_HASHED` dictionary to count range columns as part of the primary key during updates when `update_field` is specified. Closes [#44588](https://github.com/ClickHouse/ClickHouse/issues/44588). [#45061](https://github.com/ClickHouse/ClickHouse/pull/45061) ([Maksim Kita](https://github.com/kitaisreal)).
+* Fix error `Cannot capture column` for `LowCardinality` captured argument of nested lambda. Fixes [#45028](https://github.com/ClickHouse/ClickHouse/issues/45028). [#45065](https://github.com/ClickHouse/ClickHouse/pull/45065) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix the wrong query result of `additional_table_filters` (additional filter was not applied) in case the minmax/count projection is used. [#45133](https://github.com/ClickHouse/ClickHouse/pull/45133) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Fixed bug in `histogram` function accepting negative values. [#45147](https://github.com/ClickHouse/ClickHouse/pull/45147) ([simpleton](https://github.com/rgzntrade)).
* Fix wrong column nullability in StoreageJoin, close [#44940](https://github.com/ClickHouse/ClickHouse/issues/44940). [#45184](https://github.com/ClickHouse/ClickHouse/pull/45184) ([Vladimir C](https://github.com/vdimir)).
* Fix `background_fetches_pool_size` settings reload (increase at runtime). [#45189](https://github.com/ClickHouse/ClickHouse/pull/45189) ([Raúl Marín](https://github.com/Algunenano)).
* Correctly process `SELECT` queries on KV engines (e.g. KeeperMap, EmbeddedRocksDB) using `IN` on the key with subquery producing different type. [#45215](https://github.com/ClickHouse/ClickHouse/pull/45215) ([Antonio Andelic](https://github.com/antonio2368)).
* Fix logical error in SEMI JOIN & join_use_nulls in some cases, close [#45163](https://github.com/ClickHouse/ClickHouse/issues/45163), close [#45209](https://github.com/ClickHouse/ClickHouse/issues/45209). [#45230](https://github.com/ClickHouse/ClickHouse/pull/45230) ([Vladimir C](https://github.com/vdimir)).
* Fix heap-use-after-free in reading from s3. [#45253](https://github.com/ClickHouse/ClickHouse/pull/45253) ([Kruglov Pavel](https://github.com/Avogar)).
-* Fix bug when the Avro Union type is ['null', Nested type], closes [#45275](https://github.com/ClickHouse/ClickHouse/issues/45275). Fix bug that incorrectly infer `bytes` type to `Float`. [#45276](https://github.com/ClickHouse/ClickHouse/pull/45276) ([flynn](https://github.com/ucasfl)).
-* Throw a correct exception when explicit PREWHERE cannot be used with table using storage engine `Merge`. [#45319](https://github.com/ClickHouse/ClickHouse/pull/45319) ([Antonio Andelic](https://github.com/antonio2368)).
-* Under WSL1 Ubuntu self-extracting clickhouse fails to decompress due to inconsistency - /proc/self/maps reporting 32bit file's inode, while stat reporting 64bit inode. [#45339](https://github.com/ClickHouse/ClickHouse/pull/45339) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Fix bug when the Avro Union type is ['null', Nested type], closes [#45275](https://github.com/ClickHouse/ClickHouse/issues/45275). Fix bug that incorrectly infers `bytes` type to `Float`. [#45276](https://github.com/ClickHouse/ClickHouse/pull/45276) ([flynn](https://github.com/ucasfl)).
+* Throw a correct exception when explicit PREWHERE cannot be used with a table using the storage engine `Merge`. [#45319](https://github.com/ClickHouse/ClickHouse/pull/45319) ([Antonio Andelic](https://github.com/antonio2368)).
+* Under WSL1 Ubuntu self-extracting ClickHouse fails to decompress due to inconsistency - /proc/self/maps reporting 32bit file's inode, while stat reporting 64bit inode. [#45339](https://github.com/ClickHouse/ClickHouse/pull/45339) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
* Fix race in Distributed table startup (that could lead to processing file of async INSERT multiple times). [#45360](https://github.com/ClickHouse/ClickHouse/pull/45360) ([Azat Khuzhin](https://github.com/azat)).
-* Fix possible crash while reading from storage `S3` and table function `s3` in case when `ListObject` request has failed. [#45371](https://github.com/ClickHouse/ClickHouse/pull/45371) ([Anton Popov](https://github.com/CurtizJ)).
-* Fix `SELECT ... FROM system.dictionaries` exception when there is a dictionary with a bad structure (e.g. incorrect type in xml config). [#45399](https://github.com/ClickHouse/ClickHouse/pull/45399) ([Aleksei Filatov](https://github.com/aalexfvk)).
+* Fix a possible crash while reading from storage `S3` and table function `s3` in the case when `ListObject` request has failed. [#45371](https://github.com/ClickHouse/ClickHouse/pull/45371) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix `SELECT ... FROM system.dictionaries` exception when there is a dictionary with a bad structure (e.g. incorrect type in XML config). [#45399](https://github.com/ClickHouse/ClickHouse/pull/45399) ([Aleksei Filatov](https://github.com/aalexfvk)).
* Fix s3Cluster schema inference when structure from insertion table is used in `INSERT INTO ... SELECT * FROM s3Cluster` queries. [#45422](https://github.com/ClickHouse/ClickHouse/pull/45422) ([Kruglov Pavel](https://github.com/Avogar)).
* Fix bug in JSON/BSONEachRow parsing with HTTP that could lead to using default values for some columns instead of values from data. [#45424](https://github.com/ClickHouse/ClickHouse/pull/45424) ([Kruglov Pavel](https://github.com/Avogar)).
* Fixed bug (Code: 632. DB::Exception: Unexpected data ... after parsed IPv6 value ...) with typed parsing of IP types from text source. [#45425](https://github.com/ClickHouse/ClickHouse/pull/45425) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
@@ -187,7 +188,7 @@ Add settings input_format_tsv/csv/custom_detect_header that enables this behavio
* Fix possible (likely distributed) query hung. [#45448](https://github.com/ClickHouse/ClickHouse/pull/45448) ([Azat Khuzhin](https://github.com/azat)).
* Fix possible deadlock with `allow_asynchronous_read_from_io_pool_for_merge_tree` enabled in case of exception from `ThreadPool::schedule`. [#45481](https://github.com/ClickHouse/ClickHouse/pull/45481) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Fix possible in-use table after DETACH. [#45493](https://github.com/ClickHouse/ClickHouse/pull/45493) ([Azat Khuzhin](https://github.com/azat)).
-* Fix rare abort in case when query is canceled and parallel parsing was used during its execution. [#45498](https://github.com/ClickHouse/ClickHouse/pull/45498) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix rare abort in the case when a query is canceled and parallel parsing was used during its execution. [#45498](https://github.com/ClickHouse/ClickHouse/pull/45498) ([Anton Popov](https://github.com/CurtizJ)).
* Fix a race between Distributed table creation and INSERT into it (could lead to CANNOT_LINK during INSERT into the table). [#45502](https://github.com/ClickHouse/ClickHouse/pull/45502) ([Azat Khuzhin](https://github.com/azat)).
* Add proper default (SLRU) to cache policy getter. Closes [#45514](https://github.com/ClickHouse/ClickHouse/issues/45514). [#45524](https://github.com/ClickHouse/ClickHouse/pull/45524) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Disallow array join in mutations closes [#42637](https://github.com/ClickHouse/ClickHouse/issues/42637) [#44447](https://github.com/ClickHouse/ClickHouse/pull/44447) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
diff --git a/README.md b/README.md
index db1aca87ee8..bcf2643c33d 100644
--- a/README.md
+++ b/README.md
@@ -9,7 +9,7 @@ ClickHouse® is an open-source column-oriented database management system that a
* [Tutorial](https://clickhouse.com/docs/en/getting_started/tutorial/) shows how to set up and query a small ClickHouse cluster.
* [Documentation](https://clickhouse.com/docs/en/) provides more in-depth information.
* [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format.
-* [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-rxm3rdrk-lIUmhLC3V8WTaL0TGxsOmg) and [Telegram](https://telegram.me/clickhouse_en) allow chatting with ClickHouse users in real-time.
+* [Slack](https://clickhousedb.slack.com/) and [Telegram](https://telegram.me/clickhouse_en) allow chatting with ClickHouse users in real-time.
* [Blog](https://clickhouse.com/blog/) contains various ClickHouse-related articles, as well as announcements and reports about events.
* [Code Browser (Woboq)](https://clickhouse.com/codebrowser/ClickHouse/index.html) with syntax highlight and navigation.
* [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlight, powered by github.dev.
diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index f5d1315cc02..5fc8d960f56 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -140,6 +140,7 @@ add_contrib (simdjson-cmake simdjson)
add_contrib (rapidjson-cmake rapidjson)
add_contrib (fastops-cmake fastops)
add_contrib (libuv-cmake libuv)
+add_contrib (liburing-cmake liburing)
add_contrib (amqpcpp-cmake AMQP-CPP) # requires: libuv
add_contrib (cassandra-cmake cassandra) # requires: libuv
diff --git a/contrib/NuRaft b/contrib/NuRaft
index 545b8c810a9..b56784be1ae 160000
--- a/contrib/NuRaft
+++ b/contrib/NuRaft
@@ -1 +1 @@
-Subproject commit 545b8c810a956b2efdc116e86be219af7e83d68a
+Subproject commit b56784be1aec568fb72aff47f281097c017623cb
diff --git a/contrib/icu-cmake/CMakeLists.txt b/contrib/icu-cmake/CMakeLists.txt
index ce82155218c..a54bd8c1de2 100644
--- a/contrib/icu-cmake/CMakeLists.txt
+++ b/contrib/icu-cmake/CMakeLists.txt
@@ -460,8 +460,15 @@ set(ICUI18N_SOURCES
file(GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/empty.cpp" CONTENT " ")
enable_language(ASM)
+
+if (ARCH_S390X)
+ set(ICUDATA_SOURCE_FILE "${ICUDATA_SOURCE_DIR}/icudt70b_dat.S" )
+else()
+ set(ICUDATA_SOURCE_FILE "${ICUDATA_SOURCE_DIR}/icudt70l_dat.S" )
+endif()
+
set(ICUDATA_SOURCES
- "${ICUDATA_SOURCE_DIR}/icudt70l_dat.S"
+ "${ICUDATA_SOURCE_FILE}"
"${CMAKE_CURRENT_BINARY_DIR}/empty.cpp" # Without this cmake can incorrectly detects library type (OBJECT) instead of SHARED/STATIC
)
diff --git a/contrib/icudata b/contrib/icudata
index 72d9a4a7feb..c8e717892a5 160000
--- a/contrib/icudata
+++ b/contrib/icudata
@@ -1 +1 @@
-Subproject commit 72d9a4a7febc904e2b0a534ccb25ae40fac5f1e5
+Subproject commit c8e717892a557b4d2852317c7d628aacc0a0e5ab
diff --git a/contrib/krb5 b/contrib/krb5
index b89e20367b0..f8262a1b548 160000
--- a/contrib/krb5
+++ b/contrib/krb5
@@ -1 +1 @@
-Subproject commit b89e20367b074bd02dd118a6534099b21e88b3c3
+Subproject commit f8262a1b548eb29d97e059260042036255d07f8d
diff --git a/contrib/krb5-cmake/CMakeLists.txt b/contrib/krb5-cmake/CMakeLists.txt
index 7e184d424aa..ceaa270ad85 100644
--- a/contrib/krb5-cmake/CMakeLists.txt
+++ b/contrib/krb5-cmake/CMakeLists.txt
@@ -15,6 +15,10 @@ if(NOT AWK_PROGRAM)
message(FATAL_ERROR "You need the awk program to build ClickHouse with krb5 enabled.")
endif()
+if (NOT (ENABLE_OPENSSL OR ENABLE_OPENSSL_DYNAMIC))
+ add_compile_definitions(USE_BORINGSSL=1)
+endif ()
+
set(KRB5_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/krb5/src")
set(KRB5_ET_BIN_DIR "${CMAKE_CURRENT_BINARY_DIR}/include_private")
@@ -578,12 +582,6 @@ if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
list(APPEND ALL_SRCS "${CMAKE_CURRENT_BINARY_DIR}/include_private/kcmrpc.c")
endif()
-if (ENABLE_OPENSSL OR ENABLE_OPENSSL_DYNAMIC)
- list(REMOVE_ITEM ALL_SRCS "${KRB5_SOURCE_DIR}/lib/crypto/openssl/enc_provider/aes.c")
- list(APPEND ALL_SRCS "${CMAKE_CURRENT_SOURCE_DIR}/aes.c")
-endif ()
-
-
target_sources(_krb5 PRIVATE
${ALL_SRCS}
)
diff --git a/contrib/krb5-cmake/aes.c b/contrib/krb5-cmake/aes.c
deleted file mode 100644
index c0c8c728bff..00000000000
--- a/contrib/krb5-cmake/aes.c
+++ /dev/null
@@ -1,302 +0,0 @@
-/* -*- mode: c; c-basic-offset: 4; indent-tabs-mode: nil -*- */
-/* lib/crypto/openssl/enc_provider/aes.c */
-/*
- * Copyright (C) 2003, 2007, 2008, 2009 by the Massachusetts Institute of Technology.
- * All rights reserved.
- *
- * Export of this software from the United States of America may
- * require a specific license from the United States Government.
- * It is the responsibility of any person or organization contemplating
- * export to obtain such a license before exporting.
- *
- * WITHIN THAT CONSTRAINT, permission to use, copy, modify, and
- * distribute this software and its documentation for any purpose and
- * without fee is hereby granted, provided that the above copyright
- * notice appear in all copies and that both that copyright notice and
- * this permission notice appear in supporting documentation, and that
- * the name of M.I.T. not be used in advertising or publicity pertaining
- * to distribution of the software without specific, written prior
- * permission. Furthermore if you modify this software you must label
- * your software as modified software and not distribute it in such a
- * fashion that it might be confused with the original M.I.T. software.
- * M.I.T. makes no representations about the suitability of
- * this software for any purpose. It is provided "as is" without express
- * or implied warranty.
- */
-
-#include "crypto_int.h"
-#include
-#include
-
-/* proto's */
-static krb5_error_code
-cbc_enc(krb5_key key, const krb5_data *ivec, krb5_crypto_iov *data,
- size_t num_data);
-static krb5_error_code
-cbc_decr(krb5_key key, const krb5_data *ivec, krb5_crypto_iov *data,
- size_t num_data);
-static krb5_error_code
-cts_encr(krb5_key key, const krb5_data *ivec, krb5_crypto_iov *data,
- size_t num_data, size_t dlen);
-static krb5_error_code
-cts_decr(krb5_key key, const krb5_data *ivec, krb5_crypto_iov *data,
- size_t num_data, size_t dlen);
-
-#define BLOCK_SIZE 16
-#define NUM_BITS 8
-#define IV_CTS_BUF_SIZE 16 /* 16 - hardcoded in CRYPTO_cts128_en/decrypt */
-
-static const EVP_CIPHER *
-map_mode(unsigned int len)
-{
- if (len==16)
- return EVP_aes_128_cbc();
- if (len==32)
- return EVP_aes_256_cbc();
- else
- return NULL;
-}
-
-/* Encrypt one block using CBC. */
-static krb5_error_code
-cbc_enc(krb5_key key, const krb5_data *ivec, krb5_crypto_iov *data,
- size_t num_data)
-{
- int ret, olen = BLOCK_SIZE;
- unsigned char iblock[BLOCK_SIZE], oblock[BLOCK_SIZE];
- EVP_CIPHER_CTX *ctx;
- struct iov_cursor cursor;
-
- ctx = EVP_CIPHER_CTX_new();
- if (ctx == NULL)
- return ENOMEM;
-
- ret = EVP_EncryptInit_ex(ctx, map_mode(key->keyblock.length),
- NULL, key->keyblock.contents, (ivec) ? (unsigned char*)ivec->data : NULL);
- if (ret == 0) {
- EVP_CIPHER_CTX_free(ctx);
- return KRB5_CRYPTO_INTERNAL;
- }
-
- k5_iov_cursor_init(&cursor, data, num_data, BLOCK_SIZE, FALSE);
- k5_iov_cursor_get(&cursor, iblock);
- EVP_CIPHER_CTX_set_padding(ctx,0);
- ret = EVP_EncryptUpdate(ctx, oblock, &olen, iblock, BLOCK_SIZE);
- if (ret == 1)
- k5_iov_cursor_put(&cursor, oblock);
- EVP_CIPHER_CTX_free(ctx);
-
- zap(iblock, BLOCK_SIZE);
- zap(oblock, BLOCK_SIZE);
- return (ret == 1) ? 0 : KRB5_CRYPTO_INTERNAL;
-}
-
-/* Decrypt one block using CBC. */
-static krb5_error_code
-cbc_decr(krb5_key key, const krb5_data *ivec, krb5_crypto_iov *data,
- size_t num_data)
-{
- int ret = 0, olen = BLOCK_SIZE;
- unsigned char iblock[BLOCK_SIZE], oblock[BLOCK_SIZE];
- EVP_CIPHER_CTX *ctx;
- struct iov_cursor cursor;
-
- ctx = EVP_CIPHER_CTX_new();
- if (ctx == NULL)
- return ENOMEM;
-
- ret = EVP_DecryptInit_ex(ctx, map_mode(key->keyblock.length),
- NULL, key->keyblock.contents, (ivec) ? (unsigned char*)ivec->data : NULL);
- if (ret == 0) {
- EVP_CIPHER_CTX_free(ctx);
- return KRB5_CRYPTO_INTERNAL;
- }
-
- k5_iov_cursor_init(&cursor, data, num_data, BLOCK_SIZE, FALSE);
- k5_iov_cursor_get(&cursor, iblock);
- EVP_CIPHER_CTX_set_padding(ctx,0);
- ret = EVP_DecryptUpdate(ctx, oblock, &olen, iblock, BLOCK_SIZE);
- if (ret == 1)
- k5_iov_cursor_put(&cursor, oblock);
- EVP_CIPHER_CTX_free(ctx);
-
- zap(iblock, BLOCK_SIZE);
- zap(oblock, BLOCK_SIZE);
- return (ret == 1) ? 0 : KRB5_CRYPTO_INTERNAL;
-}
-
-static krb5_error_code
-cts_encr(krb5_key key, const krb5_data *ivec, krb5_crypto_iov *data,
- size_t num_data, size_t dlen)
-{
- int ret = 0;
- size_t size = 0;
- unsigned char *oblock = NULL, *dbuf = NULL;
- unsigned char iv_cts[IV_CTS_BUF_SIZE];
- struct iov_cursor cursor;
- AES_KEY enck;
-
- memset(iv_cts,0,sizeof(iv_cts));
- if (ivec && ivec->data){
- if (ivec->length != sizeof(iv_cts))
- return KRB5_CRYPTO_INTERNAL;
- memcpy(iv_cts, ivec->data,ivec->length);
- }
-
- oblock = OPENSSL_malloc(dlen);
- if (!oblock){
- return ENOMEM;
- }
- dbuf = OPENSSL_malloc(dlen);
- if (!dbuf){
- OPENSSL_free(oblock);
- return ENOMEM;
- }
-
- k5_iov_cursor_init(&cursor, data, num_data, dlen, FALSE);
- k5_iov_cursor_get(&cursor, dbuf);
-
- AES_set_encrypt_key(key->keyblock.contents,
- NUM_BITS * key->keyblock.length, &enck);
-
- size = CRYPTO_cts128_encrypt((unsigned char *)dbuf, oblock, dlen, &enck,
- iv_cts, AES_cbc_encrypt);
- if (size <= 0)
- ret = KRB5_CRYPTO_INTERNAL;
- else
- k5_iov_cursor_put(&cursor, oblock);
-
- if (!ret && ivec && ivec->data)
- memcpy(ivec->data, iv_cts, sizeof(iv_cts));
-
- zap(oblock, dlen);
- zap(dbuf, dlen);
- OPENSSL_free(oblock);
- OPENSSL_free(dbuf);
-
- return ret;
-}
-
-static krb5_error_code
-cts_decr(krb5_key key, const krb5_data *ivec, krb5_crypto_iov *data,
- size_t num_data, size_t dlen)
-{
- int ret = 0;
- size_t size = 0;
- unsigned char *oblock = NULL;
- unsigned char *dbuf = NULL;
- unsigned char iv_cts[IV_CTS_BUF_SIZE];
- struct iov_cursor cursor;
- AES_KEY deck;
-
- memset(iv_cts,0,sizeof(iv_cts));
- if (ivec && ivec->data){
- if (ivec->length != sizeof(iv_cts))
- return KRB5_CRYPTO_INTERNAL;
- memcpy(iv_cts, ivec->data,ivec->length);
- }
-
- oblock = OPENSSL_malloc(dlen);
- if (!oblock)
- return ENOMEM;
- dbuf = OPENSSL_malloc(dlen);
- if (!dbuf){
- OPENSSL_free(oblock);
- return ENOMEM;
- }
-
- AES_set_decrypt_key(key->keyblock.contents,
- NUM_BITS * key->keyblock.length, &deck);
-
- k5_iov_cursor_init(&cursor, data, num_data, dlen, FALSE);
- k5_iov_cursor_get(&cursor, dbuf);
-
- size = CRYPTO_cts128_decrypt((unsigned char *)dbuf, oblock,
- dlen, &deck,
- iv_cts, AES_cbc_encrypt);
- if (size <= 0)
- ret = KRB5_CRYPTO_INTERNAL;
- else
- k5_iov_cursor_put(&cursor, oblock);
-
- if (!ret && ivec && ivec->data)
- memcpy(ivec->data, iv_cts, sizeof(iv_cts));
-
- zap(oblock, dlen);
- zap(dbuf, dlen);
- OPENSSL_free(oblock);
- OPENSSL_free(dbuf);
-
- return ret;
-}
-
-krb5_error_code
-krb5int_aes_encrypt(krb5_key key, const krb5_data *ivec,
- krb5_crypto_iov *data, size_t num_data)
-{
- int ret = 0;
- size_t input_length, nblocks;
-
- input_length = iov_total_length(data, num_data, FALSE);
- nblocks = (input_length + BLOCK_SIZE - 1) / BLOCK_SIZE;
- if (nblocks == 1) {
- if (input_length != BLOCK_SIZE)
- return KRB5_BAD_MSIZE;
- ret = cbc_enc(key, ivec, data, num_data);
- } else if (nblocks > 1) {
- ret = cts_encr(key, ivec, data, num_data, input_length);
- }
-
- return ret;
-}
-
-krb5_error_code
-krb5int_aes_decrypt(krb5_key key, const krb5_data *ivec,
- krb5_crypto_iov *data, size_t num_data)
-{
- int ret = 0;
- size_t input_length, nblocks;
-
- input_length = iov_total_length(data, num_data, FALSE);
- nblocks = (input_length + BLOCK_SIZE - 1) / BLOCK_SIZE;
- if (nblocks == 1) {
- if (input_length != BLOCK_SIZE)
- return KRB5_BAD_MSIZE;
- ret = cbc_decr(key, ivec, data, num_data);
- } else if (nblocks > 1) {
- ret = cts_decr(key, ivec, data, num_data, input_length);
- }
-
- return ret;
-}
-
-static krb5_error_code
-krb5int_aes_init_state (const krb5_keyblock *key, krb5_keyusage usage,
- krb5_data *state)
-{
- state->length = 16;
- state->data = (void *) malloc(16);
- if (state->data == NULL)
- return ENOMEM;
- memset(state->data, 0, state->length);
- return 0;
-}
-const struct krb5_enc_provider krb5int_enc_aes128 = {
- 16,
- 16, 16,
- krb5int_aes_encrypt,
- krb5int_aes_decrypt,
- NULL,
- krb5int_aes_init_state,
- krb5int_default_free_state
-};
-
-const struct krb5_enc_provider krb5int_enc_aes256 = {
- 16,
- 32, 32,
- krb5int_aes_encrypt,
- krb5int_aes_decrypt,
- NULL,
- krb5int_aes_init_state,
- krb5int_default_free_state
-};
diff --git a/contrib/liburing b/contrib/liburing
new file mode 160000
index 00000000000..f5a48392c4e
--- /dev/null
+++ b/contrib/liburing
@@ -0,0 +1 @@
+Subproject commit f5a48392c4ea33f222cbebeb2e2fc31620162949
diff --git a/contrib/liburing-cmake/CMakeLists.txt b/contrib/liburing-cmake/CMakeLists.txt
new file mode 100644
index 00000000000..02bc116c660
--- /dev/null
+++ b/contrib/liburing-cmake/CMakeLists.txt
@@ -0,0 +1,53 @@
+set (ENABLE_LIBURING_DEFAULT ${ENABLE_LIBRARIES})
+
+if (NOT OS_LINUX)
+ set (ENABLE_LIBURING_DEFAULT OFF)
+endif ()
+
+option (ENABLE_LIBURING "Enable liburing" ${ENABLE_LIBURING_DEFAULT})
+
+if (NOT ENABLE_LIBURING)
+ message (STATUS "Not using liburing")
+ return ()
+endif ()
+
+set (LIBURING_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/liburing/src/include")
+set (LIBURING_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/liburing/src")
+
+set (SRCS
+ "${LIBURING_SOURCE_DIR}/queue.c"
+ "${LIBURING_SOURCE_DIR}/register.c"
+ "${LIBURING_SOURCE_DIR}/setup.c"
+ "${LIBURING_SOURCE_DIR}/syscall.c"
+ "${LIBURING_SOURCE_DIR}/version.c"
+)
+
+add_compile_definitions (_GNU_SOURCE)
+add_compile_definitions (LIBURING_INTERNAL)
+
+set (LIBURING_COMPAT_INCLUDE_DIR "${ClickHouse_BINARY_DIR}/contrib/liburing/src/include-compat")
+set (LIBURING_COMPAT_HEADER "${LIBURING_COMPAT_INCLUDE_DIR}/liburing/compat.h")
+
+set (LIBURING_CONFIG_HAS_KERNEL_RWF_T FALSE)
+set (LIBURING_CONFIG_HAS_KERNEL_TIMESPEC FALSE)
+set (LIBURING_CONFIG_HAS_OPEN_HOW FALSE)
+set (LIBURING_CONFIG_HAS_STATX FALSE)
+set (LIBURING_CONFIG_HAS_GLIBC_STATX FALSE)
+
+configure_file (compat.h.in ${LIBURING_COMPAT_HEADER})
+
+set (LIBURING_GENERATED_INCLUDE_DIR "${ClickHouse_BINARY_DIR}/contrib/liburing/src/include")
+set (LIBURING_VERSION_HEADER "${LIBURING_GENERATED_INCLUDE_DIR}/liburing/io_uring_version.h")
+
+file (READ "${LIBURING_SOURCE_DIR}/../liburing.spec" LIBURING_SPEC)
+
+string (REGEX MATCH "Version: ([0-9]+)\.([0-9]+)" _ ${LIBURING_SPEC})
+set (LIBURING_VERSION_MAJOR ${CMAKE_MATCH_1})
+set (LIBURING_VERSION_MINOR ${CMAKE_MATCH_2})
+
+configure_file (io_uring_version.h.in ${LIBURING_VERSION_HEADER})
+
+add_library (_liburing ${SRCS})
+add_library (ch_contrib::liburing ALIAS _liburing)
+
+target_include_directories (_liburing SYSTEM PUBLIC ${LIBURING_COMPAT_INCLUDE_DIR} ${LIBURING_GENERATED_INCLUDE_DIR} "${LIBURING_SOURCE_DIR}/include")
diff --git a/contrib/liburing-cmake/compat.h.in b/contrib/liburing-cmake/compat.h.in
new file mode 100644
index 00000000000..468e529cd33
--- /dev/null
+++ b/contrib/liburing-cmake/compat.h.in
@@ -0,0 +1,50 @@
+/* SPDX-License-Identifier: MIT */
+#ifndef LIBURING_COMPAT_H
+#define LIBURING_COMPAT_H
+
+# cmakedefine LIBURING_CONFIG_HAS_KERNEL_RWF_T
+# cmakedefine LIBURING_CONFIG_HAS_KERNEL_TIMESPEC
+# cmakedefine LIBURING_CONFIG_HAS_OPEN_HOW
+# cmakedefine LIBURING_CONFIG_HAS_GLIBC_STATX
+# cmakedefine LIBURING_CONFIG_HAS_STATX
+
+#if !defined(LIBURING_CONFIG_HAS_KERNEL_RWF_T)
+typedef int __kernel_rwf_t;
+#endif
+
+#if !defined(LIBURING_CONFIG_HAS_KERNEL_TIMESPEC)
+#include
+
+struct __kernel_timespec {
+ int64_t tv_sec;
+ long long tv_nsec;
+};
+
+/* is not available, so it can't be included */
+#define UAPI_LINUX_IO_URING_H_SKIP_LINUX_TIME_TYPES_H 1
+
+#else
+#include
+
+/* is included above and not needed again */
+#define UAPI_LINUX_IO_URING_H_SKIP_LINUX_TIME_TYPES_H 1
+
+#endif
+
+#if !defined(LIBURING_CONFIG_HAS_OPEN_HOW)
+#include
+
+struct open_how {
+ uint64_t flags;
+ uint64_t mode;
+ uint64_t resolve;
+};
+#else
+#include
+#endif
+
+#if !defined(LIBURING_CONFIG_HAS_GLIBC_STATX) && defined(LIBURING_CONFIG_HAS_STATX)
+#include
+#endif
+
+#endif
diff --git a/contrib/liburing-cmake/io_uring_version.h.in b/contrib/liburing-cmake/io_uring_version.h.in
new file mode 100644
index 00000000000..3fc6132b224
--- /dev/null
+++ b/contrib/liburing-cmake/io_uring_version.h.in
@@ -0,0 +1,8 @@
+/* SPDX-License-Identifier: MIT */
+#ifndef LIBURING_VERSION_H
+#define LIBURING_VERSION_H
+
+#define IO_URING_VERSION_MAJOR ${LIBURING_VERSION_MAJOR}
+#define IO_URING_VERSION_MINOR ${LIBURING_VERSION_MINOR}
+
+#endif
diff --git a/contrib/openssl-cmake/CMakeLists.txt b/contrib/openssl-cmake/CMakeLists.txt
index dff5dff0936..92739ff3608 100644
--- a/contrib/openssl-cmake/CMakeLists.txt
+++ b/contrib/openssl-cmake/CMakeLists.txt
@@ -1,3 +1,9 @@
+# Note: ClickHouse uses BoringSSL. The presence of OpenSSL is only due to IBM's port of ClickHouse to s390x. BoringSSL does not support
+# s390x, also FIPS validation provided by the OS vendor (Red Hat, Ubuntu) requires (preferrably dynamic) linking with OS packages which
+# ClickHouse generally avoids.
+#
+# Furthermore, the in-source OpenSSL dump in this directory is due to development purposes and non FIPS-compliant.
+
if(ENABLE_OPENSSL_DYNAMIC OR ENABLE_OPENSSL)
set(ENABLE_SSL 1 CACHE INTERNAL "")
set(OPENSSL_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/openssl)
diff --git a/contrib/snappy-cmake/CMakeLists.txt b/contrib/snappy-cmake/CMakeLists.txt
index 0997ea207e0..50cdc8732a1 100644
--- a/contrib/snappy-cmake/CMakeLists.txt
+++ b/contrib/snappy-cmake/CMakeLists.txt
@@ -1,6 +1,10 @@
set (SOURCE_DIR "${CMAKE_SOURCE_DIR}/contrib/snappy")
-set (SNAPPY_IS_BIG_ENDIAN 0)
+if (ARCH_S390X)
+ set (SNAPPY_IS_BIG_ENDIAN 1)
+else ()
+ set (SNAPPY_IS_BIG_ENDIAN 0)
+endif()
set (HAVE_BYTESWAP_H 1)
set (HAVE_SYS_MMAN_H 1)
diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine
index dcbfca68839..522fd354393 100644
--- a/docker/server/Dockerfile.alpine
+++ b/docker/server/Dockerfile.alpine
@@ -33,7 +33,7 @@ RUN arch=${TARGETARCH:-amd64} \
# lts / testing / prestable / etc
ARG REPO_CHANNEL="stable"
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="23.1.1.3077"
+ARG VERSION="23.1.2.9"
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
# user/group precreated explicitly with fixed uid/gid on purpose.
diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu
index f755a44ff32..434b4c3bff0 100644
--- a/docker/server/Dockerfile.ubuntu
+++ b/docker/server/Dockerfile.ubuntu
@@ -21,7 +21,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
ARG REPO_CHANNEL="stable"
ARG REPOSITORY="deb https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
-ARG VERSION="23.1.1.3077"
+ARG VERSION="23.1.2.9"
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
# set non-empty deb_location_url url to create a docker image
diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh
index 3cd50b06d5a..cf33dffa646 100755
--- a/docker/test/fasttest/run.sh
+++ b/docker/test/fasttest/run.sh
@@ -139,6 +139,7 @@ function clone_submodules
contrib/morton-nd
contrib/xxHash
contrib/simdjson
+ contrib/liburing
)
git submodule sync
@@ -161,6 +162,7 @@ function run_cmake
"-DENABLE_NURAFT=1"
"-DENABLE_SIMDJSON=1"
"-DENABLE_JEMALLOC=1"
+ "-DENABLE_LIBURING=1"
)
export CCACHE_DIR="$FASTTEST_WORKSPACE/ccache"
diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh
index d0d76fb0525..aa242bfa98d 100644
--- a/docker/test/stress/run.sh
+++ b/docker/test/stress/run.sh
@@ -11,6 +11,18 @@ set -x
# core.COMM.PID-TID
sysctl kernel.core_pattern='core.%e.%p-%P'
+OK="\tOK\t\\N\t"
+FAIL="\tFAIL\t\\N\t"
+function escaped()
+{
+ # That's the simplest way I found to escape a string in bash. Yep, bash is the most convenient programming language.
+ clickhouse local -S 's String' --input-format=LineAsString -q "select * from table format CustomSeparated settings format_custom_row_after_delimiter='\\\\\\\\n'"
+}
+
+function head_escaped()
+{
+ head -50 $1 | escaped
+}
function install_packages()
{
@@ -33,7 +45,9 @@ function configure()
ln -s /usr/share/clickhouse-test/ci/get_previous_release_tag.py /usr/bin/get_previous_release_tag
# avoid too slow startup
- sudo cat /etc/clickhouse-server/config.d/keeper_port.xml | sed "s|100000|10000|" > /etc/clickhouse-server/config.d/keeper_port.xml.tmp
+ sudo cat /etc/clickhouse-server/config.d/keeper_port.xml \
+ | sed "s|100000|10000|" \
+ > /etc/clickhouse-server/config.d/keeper_port.xml.tmp
sudo mv /etc/clickhouse-server/config.d/keeper_port.xml.tmp /etc/clickhouse-server/config.d/keeper_port.xml
sudo chown clickhouse /etc/clickhouse-server/config.d/keeper_port.xml
sudo chgrp clickhouse /etc/clickhouse-server/config.d/keeper_port.xml
@@ -136,6 +150,7 @@ function stop()
clickhouse stop --max-tries "$max_tries" --do-not-kill && return
# We failed to stop the server with SIGTERM. Maybe it hang, let's collect stacktraces.
+ echo -e "Possible deadlock on shutdown (see gdb.log)$FAIL" >> /test_output/test_results.tsv
kill -TERM "$(pidof gdb)" ||:
sleep 5
echo "thread apply all backtrace (on stop)" >> /test_output/gdb.log
@@ -151,10 +166,11 @@ function start()
if [ "$counter" -gt ${1:-120} ]
then
echo "Cannot start clickhouse-server"
- echo -e "Cannot start clickhouse-server\tFAIL" >> /test_output/test_results.tsv
+ rg --text ".*Application" /var/log/clickhouse-server/clickhouse-server.log > /test_output/application_errors.txt ||:
+ echo -e "Cannot start clickhouse-server$FAIL$(head_escaped /test_output/application_errors.txt)" >> /test_output/test_results.tsv
cat /var/log/clickhouse-server/stdout.log
- tail -n1000 /var/log/clickhouse-server/stderr.log
- tail -n100000 /var/log/clickhouse-server/clickhouse-server.log | rg -F -v -e ' RaftInstance:' -e ' RaftInstance' | tail -n1000
+ tail -n100 /var/log/clickhouse-server/stderr.log
+ tail -n100000 /var/log/clickhouse-server/clickhouse-server.log | rg -F -v -e ' RaftInstance:' -e ' RaftInstance' | tail -n100
break
fi
# use root to match with current uid
@@ -252,9 +268,92 @@ start
clickhouse-client --query "SHOW TABLES FROM datasets"
clickhouse-client --query "SHOW TABLES FROM test"
-clickhouse-client --query "CREATE TABLE test.hits_s3 (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
-clickhouse-client --query "CREATE TABLE test.hits (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
-clickhouse-client --query "CREATE TABLE test.visits (CounterID UInt32, StartDate Date, Sign Int8, IsNew UInt8, VisitID UInt64, UserID UInt64, StartTime DateTime, Duration UInt32, UTCStartTime DateTime, PageViews Int32, Hits Int32, IsBounce UInt8, Referer String, StartURL String, RefererDomain String, StartURLDomain String, EndURL String, LinkURL String, IsDownload UInt8, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, PlaceID Int32, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), IsYandex UInt8, GoalReachesDepth Int32, GoalReachesURL Int32, GoalReachesAny Int32, SocialSourceNetworkID UInt8, SocialSourcePage String, MobilePhoneModel String, ClientEventTime DateTime, RegionID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RemoteIP UInt32, RemoteIP6 FixedString(16), IPNetworkID UInt32, SilverlightVersion3 UInt32, CodeVersion UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, UserAgentMajor UInt16, UserAgentMinor UInt16, WindowClientWidth UInt16, WindowClientHeight UInt16, SilverlightVersion2 UInt8, SilverlightVersion4 UInt16, FlashVersion3 UInt16, FlashVersion4 UInt16, ClientTimeZone Int16, OS UInt8, UserAgent UInt8, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, NetMajor UInt8, NetMinor UInt8, MobilePhone UInt8, SilverlightVersion1 UInt8, Age UInt8, Sex UInt8, Income UInt8, JavaEnable UInt8, CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, BrowserLanguage UInt16, BrowserCountry UInt16, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), Params Array(String), Goals Nested(ID UInt32, Serial UInt32, EventTime DateTime, Price Int64, OrderID String, CurrencyID UInt32), WatchIDs Array(UInt64), ParamSumPrice Int64, ParamCurrency FixedString(3), ParamCurrencyID UInt16, ClickLogID UInt64, ClickEventID Int32, ClickGoodEvent Int32, ClickEventTime DateTime, ClickPriorityID Int32, ClickPhraseID Int32, ClickPageID Int32, ClickPlaceID Int32, ClickTypeID Int32, ClickResourceID Int32, ClickCost UInt32, ClickClientIP UInt32, ClickDomainID UInt32, ClickURL String, ClickAttempt UInt8, ClickOrderID UInt32, ClickBannerID UInt32, ClickMarketCategoryID UInt32, ClickMarketPP UInt32, ClickMarketCategoryName String, ClickMarketPPName String, ClickAWAPSCampaignName String, ClickPageName String, ClickTargetType UInt16, ClickTargetPhraseID UInt64, ClickContextType UInt8, ClickSelectType Int8, ClickOptions String, ClickGroupBannerID Int32, OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, FirstVisit DateTime, PredLastVisit Date, LastVisit Date, TotalVisits UInt32, TraficSource Nested(ID Int8, SearchEngineID UInt16, AdvEngineID UInt8, PlaceID UInt16, SocialSourceNetworkID UInt8, Domain String, SearchPhrase String, SocialSourcePage String), Attendance FixedString(16), CLID UInt32, YCLID UInt64, NormalizedRefererHash UInt64, SearchPhraseHash UInt64, RefererDomainHash UInt64, NormalizedStartURLHash UInt64, StartURLDomainHash UInt64, NormalizedEndURLHash UInt64, TopLevelDomain UInt64, URLScheme UInt64, OpenstatServiceNameHash UInt64, OpenstatCampaignIDHash UInt64, OpenstatAdIDHash UInt64, OpenstatSourceIDHash UInt64, UTMSourceHash UInt64, UTMMediumHash UInt64, UTMCampaignHash UInt64, UTMContentHash UInt64, UTMTermHash UInt64, FromHash UInt64, WebVisorEnabled UInt8, WebVisorActivity UInt32, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), Market Nested(Type UInt8, GoalID UInt32, OrderID String, OrderPrice Int64, PP UInt32, DirectPlaceID UInt32, DirectOrderID UInt32, DirectBannerID UInt32, GoodID String, GoodName String, GoodQuantity Int32, GoodPrice Int64), IslandID FixedString(16)) ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
+clickhouse-client --query "CREATE TABLE test.hits_s3 (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16,
+ EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32,
+ UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String,
+ Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32),
+ RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8,
+ FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2),
+ CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String,
+ IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8,
+ WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8,
+ SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32,
+ IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8,
+ IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8,
+ Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32,
+ RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2),
+ BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32,
+ DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32,
+ RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32,
+ LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32,
+ RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String,
+ ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String,
+ OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String,
+ UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64,
+ URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String,
+ ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64),
+ IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate)
+ ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
+clickhouse-client --query "CREATE TABLE test.hits (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16,
+ EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32,
+ UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String,
+ RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16),
+ URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8,
+ FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16,
+ UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8,
+ MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16,
+ SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16,
+ ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32,
+ SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8,
+ FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8,
+ HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8,
+ GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32,
+ HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String,
+ HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32,
+ FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32,
+ LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32,
+ RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String,
+ ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String,
+ OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String,
+ UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64,
+ URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String,
+ ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64),
+ IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate)
+ ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
+clickhouse-client --query "CREATE TABLE test.visits (CounterID UInt32, StartDate Date, Sign Int8, IsNew UInt8,
+ VisitID UInt64, UserID UInt64, StartTime DateTime, Duration UInt32, UTCStartTime DateTime, PageViews Int32,
+ Hits Int32, IsBounce UInt8, Referer String, StartURL String, RefererDomain String, StartURLDomain String,
+ EndURL String, LinkURL String, IsDownload UInt8, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String,
+ AdvEngineID UInt8, PlaceID Int32, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32),
+ RefererRegions Array(UInt32), IsYandex UInt8, GoalReachesDepth Int32, GoalReachesURL Int32, GoalReachesAny Int32,
+ SocialSourceNetworkID UInt8, SocialSourcePage String, MobilePhoneModel String, ClientEventTime DateTime, RegionID UInt32,
+ ClientIP UInt32, ClientIP6 FixedString(16), RemoteIP UInt32, RemoteIP6 FixedString(16), IPNetworkID UInt32,
+ SilverlightVersion3 UInt32, CodeVersion UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, UserAgentMajor UInt16,
+ UserAgentMinor UInt16, WindowClientWidth UInt16, WindowClientHeight UInt16, SilverlightVersion2 UInt8, SilverlightVersion4 UInt16,
+ FlashVersion3 UInt16, FlashVersion4 UInt16, ClientTimeZone Int16, OS UInt8, UserAgent UInt8, ResolutionDepth UInt8,
+ FlashMajor UInt8, FlashMinor UInt8, NetMajor UInt8, NetMinor UInt8, MobilePhone UInt8, SilverlightVersion1 UInt8,
+ Age UInt8, Sex UInt8, Income UInt8, JavaEnable UInt8, CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8,
+ BrowserLanguage UInt16, BrowserCountry UInt16, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16),
+ Params Array(String), Goals Nested(ID UInt32, Serial UInt32, EventTime DateTime, Price Int64, OrderID String, CurrencyID UInt32),
+ WatchIDs Array(UInt64), ParamSumPrice Int64, ParamCurrency FixedString(3), ParamCurrencyID UInt16, ClickLogID UInt64,
+ ClickEventID Int32, ClickGoodEvent Int32, ClickEventTime DateTime, ClickPriorityID Int32, ClickPhraseID Int32, ClickPageID Int32,
+ ClickPlaceID Int32, ClickTypeID Int32, ClickResourceID Int32, ClickCost UInt32, ClickClientIP UInt32, ClickDomainID UInt32,
+ ClickURL String, ClickAttempt UInt8, ClickOrderID UInt32, ClickBannerID UInt32, ClickMarketCategoryID UInt32, ClickMarketPP UInt32,
+ ClickMarketCategoryName String, ClickMarketPPName String, ClickAWAPSCampaignName String, ClickPageName String, ClickTargetType UInt16,
+ ClickTargetPhraseID UInt64, ClickContextType UInt8, ClickSelectType Int8, ClickOptions String, ClickGroupBannerID Int32,
+ OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String,
+ UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, FirstVisit DateTime,
+ PredLastVisit Date, LastVisit Date, TotalVisits UInt32, TraficSource Nested(ID Int8, SearchEngineID UInt16, AdvEngineID UInt8,
+ PlaceID UInt16, SocialSourceNetworkID UInt8, Domain String, SearchPhrase String, SocialSourcePage String), Attendance FixedString(16),
+ CLID UInt32, YCLID UInt64, NormalizedRefererHash UInt64, SearchPhraseHash UInt64, RefererDomainHash UInt64, NormalizedStartURLHash UInt64,
+ StartURLDomainHash UInt64, NormalizedEndURLHash UInt64, TopLevelDomain UInt64, URLScheme UInt64, OpenstatServiceNameHash UInt64,
+ OpenstatCampaignIDHash UInt64, OpenstatAdIDHash UInt64, OpenstatSourceIDHash UInt64, UTMSourceHash UInt64, UTMMediumHash UInt64,
+ UTMCampaignHash UInt64, UTMContentHash UInt64, UTMTermHash UInt64, FromHash UInt64, WebVisorEnabled UInt8, WebVisorActivity UInt32,
+ ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64),
+ Market Nested(Type UInt8, GoalID UInt32, OrderID String, OrderPrice Int64, PP UInt32, DirectPlaceID UInt32, DirectOrderID UInt32,
+ DirectBannerID UInt32, GoodID String, GoodName String, GoodQuantity Int32, GoodPrice Int64), IslandID FixedString(16))
+ ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
+ SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
clickhouse-client --query "INSERT INTO test.hits_s3 SELECT * FROM datasets.hits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0"
clickhouse-client --query "INSERT INTO test.hits SELECT * FROM datasets.hits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0"
@@ -275,7 +374,9 @@ export ZOOKEEPER_FAULT_INJECTION=1
configure
# But we still need default disk because some tables loaded only into it
-sudo cat /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml | sed "s|s3|s3default|" > /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp
+sudo cat /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml \
+ | sed "s|s3|s3default|" \
+ > /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp
mv /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml.tmp /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
sudo chown clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
sudo chgrp clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_default.xml
@@ -283,8 +384,12 @@ sudo chgrp clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_defau
start
./stress --hung-check --drop-databases --output-folder test_output --skip-func-tests "$SKIP_TESTS_OPTION" --global-time-limit 1200 \
- && echo -e 'Test script exit code\tOK' >> /test_output/test_results.tsv \
- || echo -e 'Test script failed\tFAIL' >> /test_output/test_results.tsv
+ && echo -e "Test script exit code$OK" >> /test_output/test_results.tsv \
+ || echo -e "Test script failed$FAIL script exit code: $?" >> /test_output/test_results.tsv
+
+# NOTE Hung check is implemented in docker/tests/stress/stress
+rg -Fa "No queries hung" /test_output/test_results.tsv | grep -Fa "OK" \
+ || echo -e "Hung check failed, possible deadlock found (see hung_check.log)$FAIL$(head_escaped /test_output/hung_check.log)"
stop
mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.stress.log
@@ -295,9 +400,10 @@ unset "${!THREAD_@}"
start
-clickhouse-client --query "SELECT 'Server successfully started', 'OK'" >> /test_output/test_results.tsv \
- || (echo -e 'Server failed to start (see application_errors.txt and clickhouse-server.clean.log)\tFAIL' >> /test_output/test_results.tsv \
- && rg --text ".*Application" /var/log/clickhouse-server/clickhouse-server.log > /test_output/application_errors.txt)
+clickhouse-client --query "SELECT 'Server successfully started', 'OK', NULL, ''" >> /test_output/test_results.tsv \
+ || (rg --text ".*Application" /var/log/clickhouse-server/clickhouse-server.log > /test_output/application_errors.txt \
+ && echo -e "Server failed to start (see application_errors.txt and clickhouse-server.clean.log)$FAIL$(head_escaped /test_output/application_errors.txt)" \
+ >> /test_output/test_results.tsv)
stop
@@ -310,49 +416,49 @@ stop
rg -Fa "==================" /var/log/clickhouse-server/stderr.log | rg -v "in query:" >> /test_output/tmp
rg -Fa "WARNING" /var/log/clickhouse-server/stderr.log >> /test_output/tmp
rg -Fav -e "ASan doesn't fully support makecontext/swapcontext functions" -e "DB::Exception" /test_output/tmp > /dev/null \
- && echo -e 'Sanitizer assert (in stderr.log)\tFAIL' >> /test_output/test_results.tsv \
- || echo -e 'No sanitizer asserts\tOK' >> /test_output/test_results.tsv
+ && echo -e "Sanitizer assert (in stderr.log)$FAIL$(head_escaped /test_output/tmp)" >> /test_output/test_results.tsv \
+ || echo -e "No sanitizer asserts$OK" >> /test_output/test_results.tsv
rm -f /test_output/tmp
# OOM
rg -Fa " Application: Child process was terminated by signal 9" /var/log/clickhouse-server/clickhouse-server*.log > /dev/null \
- && echo -e 'OOM killer (or signal 9) in clickhouse-server.log\tFAIL' >> /test_output/test_results.tsv \
- || echo -e 'No OOM messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
+ && echo -e "Signal 9 in clickhouse-server.log$FAIL" >> /test_output/test_results.tsv \
+ || echo -e "No OOM messages in clickhouse-server.log$OK" >> /test_output/test_results.tsv
# Logical errors
-rg -Fa "Code: 49, e.displayText() = DB::Exception:" /var/log/clickhouse-server/clickhouse-server*.log > /test_output/logical_errors.txt \
- && echo -e 'Logical error thrown (see clickhouse-server.log or logical_errors.txt)\tFAIL' >> /test_output/test_results.tsv \
- || echo -e 'No logical errors\tOK' >> /test_output/test_results.tsv
+rg -Fa "Code: 49. DB::Exception: " /var/log/clickhouse-server/clickhouse-server*.log > /test_output/logical_errors.txt \
+ && echo -e "Logical error thrown (see clickhouse-server.log or logical_errors.txt)$FAIL$(head_escaped /test_output/logical_errors.txt)" >> /test_output/test_results.tsv \
+ || echo -e "No logical errors$OK" >> /test_output/test_results.tsv
# Remove file logical_errors.txt if it's empty
[ -s /test_output/logical_errors.txt ] || rm /test_output/logical_errors.txt
# No such key errors
rg --text "Code: 499.*The specified key does not exist" /var/log/clickhouse-server/clickhouse-server*.log > /test_output/no_such_key_errors.txt \
- && echo -e 'S3_ERROR No such key thrown (see clickhouse-server.log or no_such_key_errors.txt)\tFAIL' >> /test_output/test_results.tsv \
- || echo -e 'No lost s3 keys\tOK' >> /test_output/test_results.tsv
+ && echo -e "S3_ERROR No such key thrown (see clickhouse-server.log or no_such_key_errors.txt)$FAIL$(head_escaped /test_output/no_such_key_errors.txt)" >> /test_output/test_results.tsv \
+ || echo -e "No lost s3 keys$OK" >> /test_output/test_results.tsv
# Remove file no_such_key_errors.txt if it's empty
[ -s /test_output/no_such_key_errors.txt ] || rm /test_output/no_such_key_errors.txt
# Crash
rg -Fa "########################################" /var/log/clickhouse-server/clickhouse-server*.log > /dev/null \
- && echo -e 'Killed by signal (in clickhouse-server.log)\tFAIL' >> /test_output/test_results.tsv \
- || echo -e 'Not crashed\tOK' >> /test_output/test_results.tsv
+ && echo -e "Killed by signal (in clickhouse-server.log)$FAIL" >> /test_output/test_results.tsv \
+ || echo -e "Not crashed$OK" >> /test_output/test_results.tsv
# It also checks for crash without stacktrace (printed by watchdog)
rg -Fa " " /var/log/clickhouse-server/clickhouse-server*.log > /test_output/fatal_messages.txt \
- && echo -e 'Fatal message in clickhouse-server.log (see fatal_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
- || echo -e 'No fatal messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
+ && echo -e "Fatal message in clickhouse-server.log (see fatal_messages.txt)$FAIL$(head_escaped /test_output/fatal_messages.txt)" >> /test_output/test_results.tsv \
+ || echo -e "No fatal messages in clickhouse-server.log$OK" >> /test_output/test_results.tsv
# Remove file fatal_messages.txt if it's empty
[ -s /test_output/fatal_messages.txt ] || rm /test_output/fatal_messages.txt
rg -Fa "########################################" /test_output/* > /dev/null \
- && echo -e 'Killed by signal (output files)\tFAIL' >> /test_output/test_results.tsv
+ && echo -e "Killed by signal (output files)$FAIL" >> /test_output/test_results.tsv
rg -Fa " received signal " /test_output/gdb.log > /dev/null \
- && echo -e 'Found signal in gdb.log\tFAIL' >> /test_output/test_results.tsv
+ && echo -e "Found signal in gdb.log$FAIL$(rg -A50 -Fa " received signal " /test_output/gdb.log | escaped)" >> /test_output/test_results.tsv
if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
echo -e "Backward compatibility check\n"
@@ -367,8 +473,8 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
echo "Download clickhouse-server from the previous release"
mkdir previous_release_package_folder
- echo $previous_release_tag | download_release_packages && echo -e 'Download script exit code\tOK' >> /test_output/test_results.tsv \
- || echo -e 'Download script failed\tFAIL' >> /test_output/test_results.tsv
+ echo $previous_release_tag | download_release_packages && echo -e "Download script exit code$OK" >> /test_output/test_results.tsv \
+ || echo -e "Download script failed$FAIL" >> /test_output/test_results.tsv
mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.clean.log
for table in query_log trace_log
@@ -381,13 +487,13 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
# Check if we cloned previous release repository successfully
if ! [ "$(ls -A previous_release_repository/tests/queries)" ]
then
- echo -e "Backward compatibility check: Failed to clone previous release tests\tFAIL" >> /test_output/test_results.tsv
+ echo -e "Backward compatibility check: Failed to clone previous release tests$FAIL" >> /test_output/test_results.tsv
elif ! [ "$(ls -A previous_release_package_folder/clickhouse-common-static_*.deb && ls -A previous_release_package_folder/clickhouse-server_*.deb)" ]
then
- echo -e "Backward compatibility check: Failed to download previous release packages\tFAIL" >> /test_output/test_results.tsv
+ echo -e "Backward compatibility check: Failed to download previous release packages$FAIL" >> /test_output/test_results.tsv
else
- echo -e "Successfully cloned previous release tests\tOK" >> /test_output/test_results.tsv
- echo -e "Successfully downloaded previous release packages\tOK" >> /test_output/test_results.tsv
+ echo -e "Successfully cloned previous release tests$OK" >> /test_output/test_results.tsv
+ echo -e "Successfully downloaded previous release packages$OK" >> /test_output/test_results.tsv
# Uninstall current packages
dpkg --remove clickhouse-client
@@ -446,9 +552,10 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
mkdir tmp_stress_output
- ./stress --test-cmd="/usr/bin/clickhouse-test --queries=\"previous_release_repository/tests/queries\"" --backward-compatibility-check --output-folder tmp_stress_output --global-time-limit=1200 \
- && echo -e 'Backward compatibility check: Test script exit code\tOK' >> /test_output/test_results.tsv \
- || echo -e 'Backward compatibility check: Test script failed\tFAIL' >> /test_output/test_results.tsv
+ ./stress --test-cmd="/usr/bin/clickhouse-test --queries=\"previous_release_repository/tests/queries\"" \
+ --backward-compatibility-check --output-folder tmp_stress_output --global-time-limit=1200 \
+ && echo -e "Backward compatibility check: Test script exit code$OK" >> /test_output/test_results.tsv \
+ || echo -e "Backward compatibility check: Test script failed$FAIL" >> /test_output/test_results.tsv
rm -rf tmp_stress_output
# We experienced deadlocks in this command in very rare cases. Let's debug it:
@@ -470,9 +577,9 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
export ZOOKEEPER_FAULT_INJECTION=0
configure
start 500
- clickhouse-client --query "SELECT 'Backward compatibility check: Server successfully started', 'OK'" >> /test_output/test_results.tsv \
- || (echo -e 'Backward compatibility check: Server failed to start\tFAIL' >> /test_output/test_results.tsv \
- && rg --text ".*Application" /var/log/clickhouse-server/clickhouse-server.log >> /test_output/bc_check_application_errors.txt)
+ clickhouse-client --query "SELECT 'Backward compatibility check: Server successfully started', 'OK', NULL, ''" >> /test_output/test_results.tsv \
+ || (rg --text ".*Application" /var/log/clickhouse-server/clickhouse-server.log >> /test_output/bc_check_application_errors.txt \
+ && echo -e "Backward compatibility check: Server failed to start$FAIL$(head_escaped /test_output/bc_check_application_errors.txt)" >> /test_output/test_results.tsv)
clickhouse-client --query="SELECT 'Server version: ', version()"
@@ -488,8 +595,6 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
# FIXME Not sure if it's expected, but some tests from BC check may not be finished yet when we restarting server.
# Let's just ignore all errors from queries ("} TCPHandler: Code:", "} executeQuery: Code:")
# FIXME https://github.com/ClickHouse/ClickHouse/issues/39197 ("Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'")
- # NOTE Incompatibility was introduced in https://github.com/ClickHouse/ClickHouse/pull/39263, it's expected
- # ("This engine is deprecated and is not supported in transactions", "[Queue = DB::MergeMutateRuntimeQueue]: Code: 235. DB::Exception: Part")
# FIXME https://github.com/ClickHouse/ClickHouse/issues/39174 - bad mutation does not indicate backward incompatibility
echo "Check for Error messages in server log:"
rg -Fav -e "Code: 236. DB::Exception: Cancelled merging parts" \
@@ -519,7 +624,6 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
-e "} TCPHandler: Code:" \
-e "} executeQuery: Code:" \
-e "Missing columns: 'v3' while processing query: 'v3, k, v1, v2, p'" \
- -e "This engine is deprecated and is not supported in transactions" \
-e "[Queue = DB::MergeMutateRuntimeQueue]: Code: 235. DB::Exception: Part" \
-e "The set of parts restored in place of" \
-e "(ReplicatedMergeTreeAttachThread): Initialization failed. Error" \
@@ -528,9 +632,11 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
-e "MutateFromLogEntryTask" \
-e "No connection to ZooKeeper, cannot get shared table ID" \
-e "Session expired" \
+ -e "TOO_MANY_PARTS" \
/var/log/clickhouse-server/clickhouse-server.backward.dirty.log | rg -Fa "" > /test_output/bc_check_error_messages.txt \
- && echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
- || echo -e 'Backward compatibility check: No Error messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
+ && echo -e "Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)$FAIL$(head_escaped /test_output/bc_check_error_messages.txt)" \
+ >> /test_output/test_results.tsv \
+ || echo -e "Backward compatibility check: No Error messages in clickhouse-server.log$OK" >> /test_output/test_results.tsv
# Remove file bc_check_error_messages.txt if it's empty
[ -s /test_output/bc_check_error_messages.txt ] || rm /test_output/bc_check_error_messages.txt
@@ -539,34 +645,36 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
rg -Fa "==================" /var/log/clickhouse-server/stderr.log >> /test_output/tmp
rg -Fa "WARNING" /var/log/clickhouse-server/stderr.log >> /test_output/tmp
rg -Fav -e "ASan doesn't fully support makecontext/swapcontext functions" -e "DB::Exception" /test_output/tmp > /dev/null \
- && echo -e 'Backward compatibility check: Sanitizer assert (in stderr.log)\tFAIL' >> /test_output/test_results.tsv \
- || echo -e 'Backward compatibility check: No sanitizer asserts\tOK' >> /test_output/test_results.tsv
+ && echo -e "Backward compatibility check: Sanitizer assert (in stderr.log)$FAIL$(head_escaped /test_output/tmp)" >> /test_output/test_results.tsv \
+ || echo -e "Backward compatibility check: No sanitizer asserts$OK" >> /test_output/test_results.tsv
rm -f /test_output/tmp
# OOM
rg -Fa " Application: Child process was terminated by signal 9" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /dev/null \
- && echo -e 'Backward compatibility check: OOM killer (or signal 9) in clickhouse-server.log\tFAIL' >> /test_output/test_results.tsv \
- || echo -e 'Backward compatibility check: No OOM messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
+ && echo -e "Backward compatibility check: Signal 9 in clickhouse-server.log$FAIL" >> /test_output/test_results.tsv \
+ || echo -e "Backward compatibility check: No OOM messages in clickhouse-server.log$OK" >> /test_output/test_results.tsv
# Logical errors
echo "Check for Logical errors in server log:"
- rg -Fa -A20 "Code: 49, e.displayText() = DB::Exception:" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /test_output/bc_check_logical_errors.txt \
- && echo -e 'Backward compatibility check: Logical error thrown (see clickhouse-server.log or bc_check_logical_errors.txt)\tFAIL' >> /test_output/test_results.tsv \
- || echo -e 'Backward compatibility check: No logical errors\tOK' >> /test_output/test_results.tsv
+ rg -Fa -A20 "Code: 49. DB::Exception:" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /test_output/bc_check_logical_errors.txt \
+ && echo -e "Backward compatibility check: Logical error thrown (see clickhouse-server.log or bc_check_logical_errors.txt)$FAIL$(head_escaped /test_output/bc_check_logical_errors.txt)" \
+ >> /test_output/test_results.tsv \
+ || echo -e "Backward compatibility check: No logical errors$OK" >> /test_output/test_results.tsv
# Remove file bc_check_logical_errors.txt if it's empty
[ -s /test_output/bc_check_logical_errors.txt ] || rm /test_output/bc_check_logical_errors.txt
# Crash
rg -Fa "########################################" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /dev/null \
- && echo -e 'Backward compatibility check: Killed by signal (in clickhouse-server.log)\tFAIL' >> /test_output/test_results.tsv \
- || echo -e 'Backward compatibility check: Not crashed\tOK' >> /test_output/test_results.tsv
+ && echo -e "Backward compatibility check: Killed by signal (in clickhouse-server.log)$FAIL" >> /test_output/test_results.tsv \
+ || echo -e "Backward compatibility check: Not crashed$OK" >> /test_output/test_results.tsv
# It also checks for crash without stacktrace (printed by watchdog)
echo "Check for Fatal message in server log:"
rg -Fa " " /var/log/clickhouse-server/clickhouse-server.backward.*.log > /test_output/bc_check_fatal_messages.txt \
- && echo -e 'Backward compatibility check: Fatal message in clickhouse-server.log (see bc_check_fatal_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
- || echo -e 'Backward compatibility check: No fatal messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
+ && echo -e "Backward compatibility check: Fatal message in clickhouse-server.log (see bc_check_fatal_messages.txt)$FAIL$(head_escaped /test_output/bc_check_fatal_messages.txt)" \
+ >> /test_output/test_results.tsv \
+ || echo -e "Backward compatibility check: No fatal messages in clickhouse-server.log$OK" >> /test_output/test_results.tsv
# Remove file bc_check_fatal_messages.txt if it's empty
[ -s /test_output/bc_check_fatal_messages.txt ] || rm /test_output/bc_check_fatal_messages.txt
@@ -574,7 +682,8 @@ if [ "$DISABLE_BC_CHECK" -ne "1" ]; then
tar -chf /test_output/coordination.backward.tar /var/lib/clickhouse/coordination ||:
for table in query_log trace_log
do
- clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.backward.tsv.zst ||:
+ clickhouse-local --path /var/lib/clickhouse/ --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" \
+ | zstd --threads=0 > /test_output/$table.backward.tsv.zst ||:
done
fi
fi
@@ -583,13 +692,28 @@ dmesg -T > /test_output/dmesg.log
# OOM in dmesg -- those are real
grep -q -F -e 'Out of memory: Killed process' -e 'oom_reaper: reaped process' -e 'oom-kill:constraint=CONSTRAINT_NONE' /test_output/dmesg.log \
- && echo -e 'OOM in dmesg\tFAIL' >> /test_output/test_results.tsv \
- || echo -e 'No OOM in dmesg\tOK' >> /test_output/test_results.tsv
+ && echo -e "OOM in dmesg$FAIL$(head_escaped /test_output/dmesg.log)" >> /test_output/test_results.tsv \
+ || echo -e "No OOM in dmesg$OK" >> /test_output/test_results.tsv
mv /var/log/clickhouse-server/stderr.log /test_output/
# Write check result into check_status.tsv
-clickhouse-local --structure "test String, res String" -q "SELECT 'failure', test FROM table WHERE res != 'OK' order by (lower(test) like '%hung%'), rowNumberInAllBlocks() LIMIT 1" < /test_output/test_results.tsv > /test_output/check_status.tsv
+# Try to choose most specific error for the whole check status
+clickhouse-local --structure "test String, res String" -q "SELECT 'failure', test FROM table WHERE res != 'OK' order by
+(test like 'Backward compatibility check%'), -- BC check goes last
+(test like '%Sanitizer%') DESC,
+(test like '%Killed by signal%') DESC,
+(test like '%gdb.log%') DESC,
+(test ilike '%possible deadlock%') DESC,
+(test like '%start%') DESC,
+(test like '%dmesg%') DESC,
+(test like '%OOM%') DESC,
+(test like '%Signal 9%') DESC,
+(test like '%Fatal message%') DESC,
+(test like '%Error message%') DESC,
+(test like '%previous release%') DESC,
+rowNumberInAllBlocks()
+LIMIT 1" < /test_output/test_results.tsv > /test_output/check_status.tsv
[ -s /test_output/check_status.tsv ] || echo -e "success\tNo errors found" > /test_output/check_status.tsv
# Core dumps
diff --git a/docker/test/stress/stress b/docker/test/stress/stress
index 3fce357cc19..86605b5ce0c 100755
--- a/docker/test/stress/stress
+++ b/docker/test/stress/stress
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from multiprocessing import cpu_count
-from subprocess import Popen, call, check_output, STDOUT
+from subprocess import Popen, call, check_output, STDOUT, PIPE
import os
import argparse
import logging
@@ -299,14 +299,19 @@ if __name__ == "__main__":
"00001_select_1",
]
)
- res = call(cmd, shell=True, stderr=STDOUT)
- hung_check_status = "No queries hung\tOK\n"
+ hung_check_log = os.path.join(args.output_folder, "hung_check.log")
+ tee = Popen(['/usr/bin/tee', hung_check_log], stdin=PIPE)
+ res = call(cmd, shell=True, stdout=tee.stdin, stderr=STDOUT)
+ tee.stdin.close()
if res != 0 and have_long_running_queries:
logging.info("Hung check failed with exit code {}".format(res))
- hung_check_status = "Hung check failed\tFAIL\n"
- with open(
- os.path.join(args.output_folder, "test_results.tsv"), "w+"
- ) as results:
- results.write(hung_check_status)
+ else:
+ hung_check_status = "No queries hung\tOK\t\\N\t\n"
+ with open(
+ os.path.join(args.output_folder, "test_results.tsv"), "w+"
+ ) as results:
+ results.write(hung_check_status)
+ os.remove(hung_check_log)
+
logging.info("Stress test finished")
diff --git a/docs/changelogs/v22.11.5.15-stable.md b/docs/changelogs/v22.11.5.15-stable.md
new file mode 100644
index 00000000000..742a8740514
--- /dev/null
+++ b/docs/changelogs/v22.11.5.15-stable.md
@@ -0,0 +1,22 @@
+---
+sidebar_position: 1
+sidebar_label: 2023
+---
+
+# 2023 Changelog
+
+### ClickHouse release v22.11.5.15-stable (d763e5a9239) FIXME as compared to v22.11.4.3-stable (7f4cf554f69)
+
+#### Bug Fix (user-visible misbehavior in official stable or prestable release)
+
+* Backported in [#44999](https://github.com/ClickHouse/ClickHouse/issues/44999): Another fix for `Cannot read all data` error which could happen while reading `LowCardinality` dictionary from remote fs. Fixes [#44709](https://github.com/ClickHouse/ClickHouse/issues/44709). [#44875](https://github.com/ClickHouse/ClickHouse/pull/44875) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#45552](https://github.com/ClickHouse/ClickHouse/issues/45552): Fix `SELECT ... FROM system.dictionaries` exception when there is a dictionary with a bad structure (e.g. incorrect type in xml config). [#45399](https://github.com/ClickHouse/ClickHouse/pull/45399) ([Aleksei Filatov](https://github.com/aalexfvk)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Automatically merge green backport PRs and green approved PRs [#41110](https://github.com/ClickHouse/ClickHouse/pull/41110) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Improve release scripts [#45074](https://github.com/ClickHouse/ClickHouse/pull/45074) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Fix wrong approved_at, simplify conditions [#45302](https://github.com/ClickHouse/ClickHouse/pull/45302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Get rid of artifactory in favor of r2 + ch-repos-manager [#45421](https://github.com/ClickHouse/ClickHouse/pull/45421) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Trim refs/tags/ from GITHUB_TAG in release workflow [#45636](https://github.com/ClickHouse/ClickHouse/pull/45636) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
diff --git a/docs/changelogs/v22.8.13.20-lts.md b/docs/changelogs/v22.8.13.20-lts.md
new file mode 100644
index 00000000000..d8dd1bd2b1c
--- /dev/null
+++ b/docs/changelogs/v22.8.13.20-lts.md
@@ -0,0 +1,24 @@
+---
+sidebar_position: 1
+sidebar_label: 2023
+---
+
+# 2023 Changelog
+
+### ClickHouse release v22.8.13.20-lts (e4817946d18) FIXME as compared to v22.8.12.45-lts (86b0ecd5d51)
+
+#### Bug Fix (user-visible misbehavior in official stable or prestable release)
+
+* Backported in [#45565](https://github.com/ClickHouse/ClickHouse/issues/45565): Fix positional arguments exception Positional argument out of bounds. Closes [#40634](https://github.com/ClickHouse/ClickHouse/issues/40634). [#41189](https://github.com/ClickHouse/ClickHouse/pull/41189) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Backported in [#44997](https://github.com/ClickHouse/ClickHouse/issues/44997): Another fix for `Cannot read all data` error which could happen while reading `LowCardinality` dictionary from remote fs. Fixes [#44709](https://github.com/ClickHouse/ClickHouse/issues/44709). [#44875](https://github.com/ClickHouse/ClickHouse/pull/44875) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#45550](https://github.com/ClickHouse/ClickHouse/issues/45550): Fix `SELECT ... FROM system.dictionaries` exception when there is a dictionary with a bad structure (e.g. incorrect type in xml config). [#45399](https://github.com/ClickHouse/ClickHouse/pull/45399) ([Aleksei Filatov](https://github.com/aalexfvk)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Automatically merge green backport PRs and green approved PRs [#41110](https://github.com/ClickHouse/ClickHouse/pull/41110) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Improve release scripts [#45074](https://github.com/ClickHouse/ClickHouse/pull/45074) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Fix wrong approved_at, simplify conditions [#45302](https://github.com/ClickHouse/ClickHouse/pull/45302) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Get rid of artifactory in favor of r2 + ch-repos-manager [#45421](https://github.com/ClickHouse/ClickHouse/pull/45421) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Trim refs/tags/ from GITHUB_TAG in release workflow [#45636](https://github.com/ClickHouse/ClickHouse/pull/45636) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Merge pull request [#38262](https://github.com/ClickHouse/ClickHouse/issues/38262) from PolyProgrammist/fix-ordinary-system-un… [#45650](https://github.com/ClickHouse/ClickHouse/pull/45650) ([alesapin](https://github.com/alesapin)).
+
diff --git a/docs/changelogs/v23.1.2.9-stable.md b/docs/changelogs/v23.1.2.9-stable.md
new file mode 100644
index 00000000000..272a2b95a86
--- /dev/null
+++ b/docs/changelogs/v23.1.2.9-stable.md
@@ -0,0 +1,23 @@
+---
+sidebar_position: 1
+sidebar_label: 2023
+---
+
+# 2023 Changelog
+
+### ClickHouse release v23.1.2.9-stable (8dfb1700858) FIXME as compared to v23.1.1.3077-stable (dcaac477025)
+
+#### Performance Improvement
+* Backported in [#45705](https://github.com/ClickHouse/ClickHouse/issues/45705): Fixed performance of short `SELECT` queries that read from tables with large number of`Array`/`Map`/`Nested` columns. [#45630](https://github.com/ClickHouse/ClickHouse/pull/45630) ([Anton Popov](https://github.com/CurtizJ)).
+
+#### Bug Fix
+* Backported in [#45673](https://github.com/ClickHouse/ClickHouse/issues/45673): Fix wiping sensitive info in logs. [#45603](https://github.com/ClickHouse/ClickHouse/pull/45603) ([Vitaly Baranov](https://github.com/vitlibar)).
+
+#### Bug Fix (user-visible misbehavior in official stable or prestable release)
+
+* Backported in [#45730](https://github.com/ClickHouse/ClickHouse/issues/45730): Fix key description when encountering duplicate primary keys. This can happen in projections. See [#45590](https://github.com/ClickHouse/ClickHouse/issues/45590) for details. [#45686](https://github.com/ClickHouse/ClickHouse/pull/45686) ([Amos Bird](https://github.com/amosbird)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Trim refs/tags/ from GITHUB_TAG in release workflow [#45636](https://github.com/ClickHouse/ClickHouse/pull/45636) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+
diff --git a/docs/en/engines/table-engines/mergetree-family/invertedindexes.md b/docs/en/engines/table-engines/mergetree-family/invertedindexes.md
index 4d7a0050c76..2899476b847 100644
--- a/docs/en/engines/table-engines/mergetree-family/invertedindexes.md
+++ b/docs/en/engines/table-engines/mergetree-family/invertedindexes.md
@@ -49,20 +49,20 @@ where `N` specifies the tokenizer:
Being a type of skipping index, inverted indexes can be dropped or added to a column after table creation:
``` sql
-ALTER TABLE tbl DROP INDEX inv_idx;
-ALTER TABLE tbl ADD INDEX inv_idx(s) TYPE inverted(2) GRANULARITY 1;
+ALTER TABLE tab DROP INDEX inv_idx;
+ALTER TABLE tab ADD INDEX inv_idx(s) TYPE inverted(2) GRANULARITY 1;
```
To use the index, no special functions or syntax are required. Typical string search predicates automatically leverage the index. As
examples, consider:
```sql
-SELECT * from tab WHERE s == 'Hello World;
-SELECT * from tab WHERE s IN (‘Hello’, ‘World’);
-SELECT * from tab WHERE s LIKE ‘%Hello%’;
-SELECT * from tab WHERE multiSearchAny(s, ‘Hello’, ‘World’);
-SELECT * from tab WHERE hasToken(s, ‘Hello’);
-SELECT * from tab WHERE multiSearchAll(s, [‘Hello’, ‘World’]);
+INSERT INTO tab(key, str) values (1, 'Hello World');
+SELECT * from tab WHERE str == 'Hello World';
+SELECT * from tab WHERE str IN ('Hello', 'World');
+SELECT * from tab WHERE str LIKE '%Hello%';
+SELECT * from tab WHERE multiSearchAny(str, ['Hello', 'World']);
+SELECT * from tab WHERE hasToken(str, 'Hello');
```
The inverted index also works on columns of type `Array(String)`, `Array(FixedString)`, `Map(String)` and `Map(String)`.
diff --git a/docs/en/engines/table-engines/special/generate.md b/docs/en/engines/table-engines/special/generate.md
index 32fa2cd9b2b..77d90082ddc 100644
--- a/docs/en/engines/table-engines/special/generate.md
+++ b/docs/en/engines/table-engines/special/generate.md
@@ -19,7 +19,7 @@ ENGINE = GenerateRandom([random_seed] [,max_string_length] [,max_array_length])
```
The `max_array_length` and `max_string_length` parameters specify maximum length of all
-array columns and strings correspondingly in generated data.
+array or map columns and strings correspondingly in generated data.
Generate table engine supports only `SELECT` queries.
diff --git a/docs/en/operations/_troubleshooting.md b/docs/en/operations/_troubleshooting.md
index aed63ec4d0f..a5c07ed18bd 100644
--- a/docs/en/operations/_troubleshooting.md
+++ b/docs/en/operations/_troubleshooting.md
@@ -56,6 +56,19 @@ sudo apt-get clean
sudo apt-get autoclean
```
+### You Can't Get Packages With Yum Because Of Wrong Signature
+
+Possible issue: the cache is wrong, maybe it's broken after updated GPG key in 2022-09.
+
+The solution is to clean out the cache and lib directory for yum:
+
+```
+sudo find /var/lib/yum/repos/ /var/cache/yum/ -name 'clickhouse-*' -type d -exec rm -rf {} +
+sudo rm -f /etc/yum.repos.d/clickhouse.repo
+```
+
+After that follow the [install guide](../getting-started/install.md#from-rpm-packages)
+
## Connecting to the Server {#troubleshooting-accepts-no-connections}
Possible issues:
diff --git a/docs/en/operations/backup.md b/docs/en/operations/backup.md
index 4feb434d762..f1a5649cd4c 100644
--- a/docs/en/operations/backup.md
+++ b/docs/en/operations/backup.md
@@ -79,7 +79,7 @@ The BACKUP and RESTORE statements take a list of DATABASE and TABLE names, a des
- ASYNC: backup or restore asynchronously
- PARTITIONS: a list of partitions to restore
- SETTINGS:
- - [`compression_method`](en/sql-reference/statements/create/table/#column-compression-codecs) and compression_level
+ - [`compression_method`](/docs/en/sql-reference/statements/create/table.md/#column-compression-codecs) and compression_level
- `password` for the file on disk
- `base_backup`: the destination of the previous backup of this source. For example, `Disk('backups', '1.zip')`
diff --git a/docs/en/operations/query-cache.md b/docs/en/operations/query-cache.md
new file mode 100644
index 00000000000..1a486de7904
--- /dev/null
+++ b/docs/en/operations/query-cache.md
@@ -0,0 +1,112 @@
+---
+slug: /en/operations/query-cache
+sidebar_position: 65
+sidebar_label: Query Cache [experimental]
+---
+
+# Query Cache [experimental]
+
+The query cache allows to compute `SELECT` queries just once and to serve further executions of the same query directly from the cache.
+Depending on the type of the queries, this can dramatically reduce latency and resource consumption of the ClickHouse server.
+
+## Background, Design and Limitations
+
+Query caches can generally be viewed as transactionally consistent or inconsistent.
+
+- In transactionally consistent caches, the database invalidates (discards) cached query results if the result of the `SELECT` query changes
+ or potentially changes. In ClickHouse, operations which change the data include inserts/updates/deletes in/of/from tables or collapsing
+ merges. Transactionally consistent caching is especially suitable for OLTP databases, for example
+ [MySQL](https://dev.mysql.com/doc/refman/5.6/en/query-cache.html) (which removed query cache after v8.0) and
+ [Oracle](https://docs.oracle.com/database/121/TGDBA/tune_result_cache.htm).
+- In transactionally inconsistent caches, slight inaccuracies in query results are accepted under the assumption that all cache entries are
+ assigned a validity period after which they expire (e.g. 1 minute) and that the underlying data changes only little during this period.
+ This approach is overall more suitable for OLAP databases. As an example where transactionally inconsistent caching is sufficient,
+ consider an hourly sales report in a reporting tool which is simultaneously accessed by multiple users. Sales data changes typically
+ slowly enough that the database only needs to compute the report once (represented by the first `SELECT` query). Further queries can be
+ served directly from the query cache. In this example, a reasonable validity period could be 30 min.
+
+Transactionally inconsistent caching is traditionally provided by client tools or proxy packages interacting with the database. As a result,
+the same caching logic and configuration is often duplicated. With ClickHouse's query cache, the caching logic moves to the server side.
+This reduces maintenance effort and avoids redundancy.
+
+:::warning
+The query cache is an experimental feature that should not be used in production. There are known cases (e.g. in distributed query
+processing) where wrong results are returned.
+:::
+
+## Configuration Settings and Usage
+
+As long as the result cache is experimental it must be activated using the following configuration setting:
+
+```sql
+SET allow_experimental_query_cache = true;
+```
+
+Afterwards, setting [use_query_cache](settings/settings.md#use-query-cache) can be used to control whether a specific query or all queries
+of the current session should utilize the query cache. For example, the first execution of query
+
+```sql
+SELECT some_expensive_calculation(column_1, column_2)
+FROM table
+SETTINGS use_query_cache = true;
+```
+
+will store the query result in the query cache. Subsequent executions of the same query (also with parameter `use_query_cache = true`) will
+read the computed result from the cache and return it immediately.
+
+The way the cache is utilized can be configured in more detail using settings [enable_writes_to_query_cache](settings/settings.md#enable-writes-to-query-cache)
+and [enable_reads_from_query_cache](settings/settings.md#enable-reads-from-query-cache) (both `true` by default). The former setting
+controls whether query results are stored in the cache, whereas the latter setting determines if the database should try to retrieve query
+results from the cache. For example, the following query will use the cache only passively, i.e. attempt to read from it but not store its
+result in it:
+
+```sql
+SELECT some_expensive_calculation(column_1, column_2)
+FROM table
+SETTINGS use_query_cache = true, enable_writes_to_query_cache = false;
+```
+
+For maximum control, it is generally recommended to provide settings "use_query_cache", "enable_writes_to_query_cache" and
+"enable_reads_from_query_cache" only with specific queries. It is also possible to enable caching at user or profile level (e.g. via `SET
+use_query_cache = true`) but one should keep in mind that all `SELECT` queries including monitoring or debugging queries to system tables
+may return cached results then.
+
+The query cache can be cleared using statement `SYSTEM DROP QUERY CACHE`. The content of the query cache is displayed in system table
+`system.query_cache`. The number of query cache hits and misses are shown as events "QueryCacheHits" and "QueryCacheMisses" in system table
+`system.events`. Both counters are only updated for `SELECT` queries which run with setting "use_query_cache = true". Other queries do not
+affect the cache miss counter.
+
+The query cache exists once per ClickHouse server process. However, cache results are by default not shared between users. This can be
+changed (see below) but doing so is not recommended for security reasons.
+
+Query results are referenced in the query cache by the [Abstract Syntax Tree (AST)](https://en.wikipedia.org/wiki/Abstract_syntax_tree) of
+their query. This means that caching is agnostic to upper/lowercase, for example `SELECT 1` and `select 1` are treated as the same query. To
+make the matching more natural, all query-level settings related to the query cache are removed from the AST.
+
+If the query was aborted due to an exception or user cancellation, no entry is written into the query cache.
+
+The size of the query cache, the maximum number of cache entries and the maximum size of cache entries (in bytes and in records) can
+be configured using different [server configuration options](server-configuration-parameters/settings.md#server_configuration_parameters_query-cache).
+
+To define how long a query must run at least such that its result can be cached, you can use setting
+[query_cache_min_query_duration](settings/settings.md#query-cache-min-query-duration). For example, the result of query
+
+``` sql
+SELECT some_expensive_calculation(column_1, column_2)
+FROM table
+SETTINGS use_query_cache = true, query_cache_min_query_duration = 5000;
+```
+
+is only cached if the query runs longer than 5 seconds. It is also possible to specify how often a query needs to run until its result is
+cached - for that use setting [query_cache_min_query_runs](settings/settings.md#query-cache-min-query-runs).
+
+Entries in the query cache become stale after a certain time period (time-to-live). By default, this period is 60 seconds but a different
+value can be specified at session, profile or query level using setting [query_cache_ttl](settings/settings.md#query-cache-ttl).
+
+Also, results of queries with non-deterministic functions such as `rand()` and `now()` are not cached. This can be overruled using
+setting [query_cache_store_results_of_queries_with_nondeterministic_functions](settings/settings.md#query-cache-store-results-of-queries-with-nondeterministic-functions).
+
+Finally, entries in the query cache are not shared between users due to security reasons. For example, user A must not be able to bypass a
+row policy on a table by running the same query as another user B for whom no such policy exists. However, if necessary, cache entries can
+be marked accessible by other users (i.e. shared) by supplying setting
+[query_cache_share_between_users](settings/settings.md#query-cache-share-between-users).
diff --git a/docs/en/operations/query-result-cache.md b/docs/en/operations/query-result-cache.md
index a166cdd143f..046b75ac5c5 100644
--- a/docs/en/operations/query-result-cache.md
+++ b/docs/en/operations/query-result-cache.md
@@ -6,14 +6,14 @@ sidebar_label: Query Result Cache [experimental]
# Query Result Cache [experimental]
-The query result cache allows to compute SELECT queries just once and to serve further executions of the same query directly from the cache.
-Depending on the type of the queries, this can dramatically reduce latency and resource consumption of the ClickHouse server.
+The query result cache allows to compute `SELECT` queries just once and to serve further executions of the same query directly from the
+cache. Depending on the type of the queries, this can dramatically reduce latency and resource consumption of the ClickHouse server.
## Background, Design and Limitations
Query result caches can generally be viewed as transactionally consistent or inconsistent.
-- In transactionally consistent caches, the database invalidates (discards) cached query results if the result of the SELECT query changes
+- In transactionally consistent caches, the database invalidates (discards) cached query results if the result of the `SELECT` query changes
or potentially changes. In ClickHouse, operations which change the data include inserts/updates/deletes in/of/from tables or collapsing
merges. Transactionally consistent caching is especially suitable for OLTP databases, for example
[MySQL](https://dev.mysql.com/doc/refman/5.6/en/query-cache.html) (which removed query result cache after v8.0) and
@@ -22,7 +22,7 @@ Query result caches can generally be viewed as transactionally consistent or inc
assigned a validity period after which they expire (e.g. 1 minute) and that the underlying data changes only little during this period.
This approach is overall more suitable for OLAP databases. As an example where transactionally inconsistent caching is sufficient,
consider an hourly sales report in a reporting tool which is simultaneously accessed by multiple users. Sales data changes typically
- slowly enough that the database only needs to compute the report once (represented by the first SELECT query). Further queries can be
+ slowly enough that the database only needs to compute the report once (represented by the first `SELECT` query). Further queries can be
served directly from the query result cache. In this example, a reasonable validity period could be 30 min.
Transactionally inconsistent caching is traditionally provided by client tools or proxy packages interacting with the database. As a result,
@@ -36,32 +36,45 @@ processing) where wrong results are returned.
## Configuration Settings and Usage
-Parameter [enable_experimental_query_result_cache](settings/settings.md#enable-experimental-query-result-cache) controls whether query
-results are inserted into / retrieved from the cache for the current query or session. For example, the first execution of query
+As long as the result cache is experimental it must be activated using the following configuration setting:
-``` sql
-SELECT some_expensive_calculation(column_1, column_2)
-FROM table
-SETTINGS enable_experimental_query_result_cache = true;
+```sql
+SET allow_experimental_query_result_cache = true;
```
-stores the query result into the query result cache. Subsequent executions of the same query (also with parameter
-`enable_experimental_query_result_cache = true`) will read the computed result directly from the cache.
+Afterwards, setting [use_query_result_cache](settings/settings.md#use-query-result-cache) can be used to control whether a specific query or
+all queries of the current session should utilize the query result cache. For example, the first execution of query
-Sometimes, it is desirable to use the query result cache only passively, i.e. to allow reading from it but not writing into it (if the cache
-result is not stored yet). Parameter [enable_experimental_query_result_cache_passive_usage](settings/settings.md#enable-experimental-query-result-cache-passive-usage)
-instead of 'enable_experimental_query_result_cache' can be used for that.
+```sql
+SELECT some_expensive_calculation(column_1, column_2)
+FROM table
+SETTINGS use_query_result_cache = true;
+```
-For maximum control, it is generally recommended to provide settings "enable_experimental_query_result_cache" or
-"enable_experimental_query_result_cache_passive_usage" only with specific queries. It is also possible to enable caching at user or profile
-level but one should keep in mind that all SELECT queries may return a cached results, including monitoring or debugging queries to system
-tables.
+will store the query result in the query result cache. Subsequent executions of the same query (also with parameter `use_query_result_cache
+= true`) will read the computed result from the cache and return it immediately.
+
+The way the cache is utilized can be configured in more detail using settings [enable_writes_to_query_result_cache](settings/settings.md#enable-writes-to-query-result-cache)
+and [enable_reads_from_query_result_cache](settings/settings.md#enable-reads-from-query-result-cache) (both `true` by default). The first
+settings controls whether query results are stored in the cache, whereas the second parameter determines if the database should try to
+retrieve query results from the cache. For example, the following query will use the cache only passively, i.e. attempt to read from it but
+not store its result in it:
+
+```sql
+SELECT some_expensive_calculation(column_1, column_2)
+FROM table
+SETTINGS use_query_result_cache = true, enable_writes_to_query_result_cache = false;
+```
+
+For maximum control, it is generally recommended to provide settings "use_query_result_cache", "enable_writes_to_query_result_cache" and
+"enable_reads_from_query_result_cache" only with specific queries. It is also possible to enable caching at user or profile level (e.g. via
+`SET use_query_result_cache = true`) but one should keep in mind that all `SELECT` queries including monitoring or debugging queries to
+system tables may return cached results then.
The query result cache can be cleared using statement `SYSTEM DROP QUERY RESULT CACHE`. The content of the query result cache is displayed
-in system table `SYSTEM.QUERY_RESULT_CACHE`. The number of query result cache hits and misses are shown as events "QueryResultCacheHits" and
-"QueryResultCacheMisses" in system table `SYSTEM.EVENTS`. Both counters are only updated for SELECT queries which run with settings
-"enable_experimental_query_result_cache = true" or "enable_experimental_query_result_cache_passive_usage = true". Other queries do not
-affect the cache miss counter.
+in system table `SYSTEM.QUERY_RESULT_CACHE`. The number of query result cache hits and misses are shown as events "QueryCacheHits" and
+"QueryCacheMisses" in system table `SYSTEM.EVENTS`. Both counters are only updated for `SELECT` queries which run with setting
+"use_query_result_cache = true". Other queries do not affect the cache miss counter.
The query result cache exists once per ClickHouse server process. However, cache results are by default not shared between users. This can
be changed (see below) but doing so is not recommended for security reasons.
@@ -81,7 +94,7 @@ To define how long a query must run at least such that its result can be cached,
``` sql
SELECT some_expensive_calculation(column_1, column_2)
FROM table
-SETTINGS enable_experimental_query_result_cache = true, query_result_cache_min_query_duration = 5000;
+SETTINGS use_query_result_cache = true, query_result_cache_min_query_duration = 5000;
```
is only cached if the query runs longer than 5 seconds. It is also possible to specify how often a query needs to run until its result is
@@ -96,4 +109,4 @@ setting [query_result_cache_store_results_of_queries_with_nondeterministic_funct
Finally, entries in the query cache are not shared between users due to security reasons. For example, user A must not be able to bypass a
row policy on a table by running the same query as another user B for whom no such policy exists. However, if necessary, cache entries can
be marked accessible by other users (i.e. shared) by supplying setting
-[query_result_cache_share_between_users]{settings/settings.md#query-result-cache-share-between-users}.
+[query_result_cache_share_between_users](settings/settings.md#query-result-cache-share-between-users).
diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md
index 9a67edd75ca..761d27a889f 100644
--- a/docs/en/operations/server-configuration-parameters/settings.md
+++ b/docs/en/operations/server-configuration-parameters/settings.md
@@ -1270,30 +1270,30 @@ If the table does not exist, ClickHouse will create it. If the structure of the
```
-## query_result_cache {#server_configuration_parameters_query-result-cache}
+## query_cache {#server_configuration_parameters_query-cache}
-[Query result cache](../query-result-cache.md) configuration.
+[Query cache](../query-cache.md) configuration.
The following settings are available:
-- `size`: The maximum cache size in bytes. 0 means the query result cache is disabled. Default value: `1073741824` (1 GiB).
-- `max_entries`: The maximum number of SELECT query results stored in the cache. Default value: `1024`.
-- `max_entry_size`: The maximum size in bytes SELECT query results may have to be saved in the cache. Default value: `1048576` (1 MiB).
-- `max_entry_records`: The maximum number of records SELECT query results may have to be saved in the cache. Default value: `30000000` (30 mil).
+- `size`: The maximum cache size in bytes. 0 means the query cache is disabled. Default value: `1073741824` (1 GiB).
+- `max_entries`: The maximum number of `SELECT` query results stored in the cache. Default value: `1024`.
+- `max_entry_size`: The maximum size in bytes `SELECT` query results may have to be saved in the cache. Default value: `1048576` (1 MiB).
+- `max_entry_records`: The maximum number of records `SELECT` query results may have to be saved in the cache. Default value: `30000000` (30 mil).
:::warning
-Data for the query result cache is allocated in DRAM. If memory is scarce, make sure to set a small value for `size` or disable the query result cache altogether.
+Data for the query cache is allocated in DRAM. If memory is scarce, make sure to set a small value for `size` or disable the query cache altogether.
:::
**Example**
```xml
-
+
1073741824
1024
1048576
30000000
-
+
```
## query_thread_log {#server_configuration_parameters-query_thread_log}
diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md
index b383e0598a1..5bc174727ad 100644
--- a/docs/en/operations/settings/merge-tree-settings.md
+++ b/docs/en/operations/settings/merge-tree-settings.md
@@ -233,7 +233,7 @@ Possible values:
Default value: 100.
-Normally, the `use_async_block_ids_cache` updates as soon as there are updates in the watching keeper path. However, the cache updates might be too frequent and become a heavy burden. This minimum interval prevents the cache from updating too fast. Note that if we set this value too long, the block with duplicated inserts will have a longer retry time.
+Normally, the `use_async_block_ids_cache` updates as soon as there are updates in the watching keeper path. However, the cache updates might be too frequent and become a heavy burden. This minimum interval prevents the cache from updating too fast. Note that if we set this value too long, the block with duplicated inserts will have a longer retry time.
## max_replicated_logs_to_keep
diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index b8376f3449e..81c0531427f 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -1301,9 +1301,43 @@ Possible values:
Default value: `3`.
-## enable_experimental_query_result_cache {#enable-experimental-query-result-cache}
+## use_query_cache {#use-query-cache}
-If turned on, results of SELECT queries are stored in and (if available) retrieved from the [query result cache](../query-result-cache.md).
+If turned on, `SELECT` queries may utilize the [query cache](../query-cache.md). Parameters [enable_reads_from_query_cache](#enable-readsfrom-query-cache)
+and [enable_writes_to_query_cache](#enable-writes-to-query-cache) control in more detail how the cache is used.
+
+Possible values:
+
+- 0 - Yes
+- 1 - No
+
+Default value: `0`.
+
+## enable_reads_from_query_cache {#enable-reads-from-query-cache}
+
+If turned on, results of `SELECT` queries are retrieved from the [query cache](../query-cache.md).
+
+Possible values:
+
+- 0 - Disabled
+- 1 - Enabled
+
+Default value: `1`.
+
+## enable_writes_to_query_cache {#enable-writes-to-query-cache}
+
+If turned on, results of `SELECT` queries are stored in the [query cache](../query-cache.md).
+
+Possible values:
+
+- 0 - Disabled
+- 1 - Enabled
+
+Default value: `1`.
+
+## query_cache_store_results_of_queries_with_nondeterministic_functions {#query--store-results-of-queries-with-nondeterministic-functions}
+
+If turned on, then results of `SELECT` queries with non-deterministic functions (e.g. `rand()`, `now()`) can be cached in the [query cache](../query-cache.md).
Possible values:
@@ -1312,31 +1346,9 @@ Possible values:
Default value: `0`.
-## enable_experimental_query_result_cache_passive_usage {#enable-experimental-query-result-cache-passive-usage}
+## query_cache_min_query_runs {#query-cache-min-query-runs}
-If turned on, results of SELECT queries are (if available) retrieved from the [query result cache](../query-result-cache.md).
-
-Possible values:
-
-- 0 - Disabled
-- 1 - Enabled
-
-Default value: `0`.
-
-## query_result_cache_store_results_of_queries_with_nondeterministic_functions {#query-result-cache-store-results-of-queries-with-nondeterministic-functions}
-
-If turned on, then results of SELECT queries with non-deterministic functions (e.g. `rand()`, `now()`) can be cached in the [query result cache](../query-result-cache.md).
-
-Possible values:
-
-- 0 - Disabled
-- 1 - Enabled
-
-Default value: `0`.
-
-## query_result_cache_min_query_runs {#query-result-cache-min-query-runs}
-
-Minimum number of times a SELECT query must run before its result is stored in the [query result cache](../query-result-cache.md).
+Minimum number of times a `SELECT` query must run before its result is stored in the [query cache](../query-cache.md).
Possible values:
@@ -1344,9 +1356,9 @@ Possible values:
Default value: `0`
-## query_result_cache_min_query_duration {#query-result-cache-min-query-duration}
+## query_cache_min_query_duration {#query-cache-min-query-duration}
-Minimum duration in milliseconds a query needs to run for its result to be stored in the [query result cache](../query-result-cache.md).
+Minimum duration in milliseconds a query needs to run for its result to be stored in the [query cache](../query-cache.md).
Possible values:
@@ -1354,9 +1366,9 @@ Possible values:
Default value: `0`
-## query_result_cache_ttl {#query-result-cache-ttl}
+## query_cache_ttl {#query-cache-ttl}
-After this time in seconds entries in the [query result cache](../query-result-cache.md) become stale.
+After this time in seconds entries in the [query cache](../query-cache.md) become stale.
Possible values:
@@ -1364,9 +1376,9 @@ Possible values:
Default value: `60`
-## query_result_cache_share_between_users {#query-result-cache-share-between-users}
+## query_cache_share_between_users {#query-cache-share-between-users}
-If turned on, the result of SELECT queries cached in the [query result cache](../query-result-cache.md) can be read by other users.
+If turned on, the result of `SELECT` queries cached in the [query cache](../query-cache.md) can be read by other users.
It is not recommended to enable this setting due to security reasons.
Possible values:
@@ -1632,6 +1644,49 @@ SELECT * FROM test_table
└───┘
```
+## insert_keeper_max_retries
+
+The setting sets the maximum number of retries for ClickHouse Keeper (or ZooKeeper) requests during insert into replicated MergeTree. Only Keeper requests which failed due to network error, Keeper session timeout, or request timeout are considered for retries.
+
+Possible values:
+
+- Positive integer.
+- 0 — Retries are disabled
+
+Default value: 0
+
+Keeper request retries are done after some timeout. The timeout is controlled by the following settings: `insert_keeper_retry_initial_backoff_ms`, `insert_keeper_retry_max_backoff_ms`.
+The first retry is done after `insert_keeper_retry_initial_backoff_ms` timeout. The consequent timeouts will be calculated as follows:
+```
+timeout = min(insert_keeper_retry_max_backoff_ms, latest_timeout * 2)
+```
+
+For example, if `insert_keeper_retry_initial_backoff_ms=100`, `insert_keeper_retry_max_backoff_ms=10000` and `insert_keeper_max_retries=8` then timeouts will be `100, 200, 400, 800, 1600, 3200, 6400, 10000`.
+
+Apart from fault tolerance, the retries aim to provide a better user experience - they allow to avoid returning an error during INSERT execution if Keeper is restarted, for example, due to an upgrade.
+
+## insert_keeper_retry_initial_backoff_ms {#insert_keeper_retry_initial_backoff_ms}
+
+Initial timeout(in milliseconds) to retry a failed Keeper request during INSERT query execution
+
+Possible values:
+
+- Positive integer.
+- 0 — No timeout
+
+Default value: 100
+
+## insert_keeper_retry_max_backoff_ms {#insert_keeper_retry_max_backoff_ms}
+
+Maximum timeout (in milliseconds) to retry a failed Keeper request during INSERT query execution
+
+Possible values:
+
+- Positive integer.
+- 0 — Maximum timeout is not limited
+
+Default value: 10000
+
## max_network_bytes {#settings-max-network-bytes}
Limits the data volume (in bytes) that is received or transmitted over the network when executing a query. This setting applies to every individual query.
@@ -3634,6 +3689,30 @@ Default value: `0`.
- [optimize_move_to_prewhere](#optimize_move_to_prewhere) setting
+## optimize_using_constraints
+
+Use [constraints](../../sql-reference/statements/create/table#constraints) for query optimization. The default is `false`.
+
+Possible values:
+
+- true, false
+
+## optimize_append_index
+
+Use [constraints](../../sql-reference/statements/create/table#constraints) in order to append index condition. The default is `false`.
+
+Possible values:
+
+- true, false
+
+## optimize_substitute_columns
+
+Use [constraints](../../sql-reference/statements/create/table#constraints) for column substitution. The default is `false`.
+
+Possible values:
+
+- true, false
+
## describe_include_subcolumns {#describe_include_subcolumns}
Enables describing subcolumns for a [DESCRIBE](../../sql-reference/statements/describe-table.md) query. For example, members of a [Tuple](../../sql-reference/data-types/tuple.md) or subcolumns of a [Map](../../sql-reference/data-types/map.md/#map-subcolumns), [Nullable](../../sql-reference/data-types/nullable.md/#finding-null) or an [Array](../../sql-reference/data-types/array.md/#array-size) data type.
diff --git a/docs/en/sql-reference/aggregate-functions/reference/maxmap.md b/docs/en/sql-reference/aggregate-functions/reference/maxmap.md
index ebb9d054476..4d8c67e1b90 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/maxmap.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/maxmap.md
@@ -19,11 +19,11 @@ Example:
``` sql
SELECT maxMap(a, b)
-FROM values('a Array(Int32), b Array(Int64)', ([1, 2], [2, 2]), ([2, 3], [1, 1]))
+FROM values('a Array(Char), b Array(Int64)', (['x', 'y'], [2, 2]), (['y', 'z'], [3, 1]))
```
``` text
-┌─maxMap(a, b)──────┐
-│ ([1,2,3],[2,2,1]) │
-└───────────────────┘
+┌─maxMap(a, b)───────────┐
+│ [['x','y','z'],[2,3,1]]│
+└────────────────────────┘
```
diff --git a/docs/en/sql-reference/aggregate-functions/reference/topkweighted.md b/docs/en/sql-reference/aggregate-functions/reference/topkweighted.md
index 133de88a07e..32b234fd6b8 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/topkweighted.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/topkweighted.md
@@ -28,15 +28,16 @@ Returns an array of the values with maximum approximate sum of weights.
Query:
``` sql
-SELECT topKWeighted(10)(number, number) FROM numbers(1000)
+SELECT topKWeighted(2)(k, w) FROM
+VALUES('k Char, w UInt64', ('y', 1), ('y', 1), ('x', 5), ('y', 1), ('z', 10))
```
Result:
``` text
-┌─topKWeighted(10)(number, number)──────────┐
-│ [999,998,997,996,995,994,993,992,991,990] │
-└───────────────────────────────────────────┘
+┌─topKWeighted(2)(k, w)──┐
+│ ['z','x'] │
+└────────────────────────┘
```
**See Also**
diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md
index adea1ac0282..4dc6fd33849 100644
--- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md
+++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md
@@ -5,7 +5,7 @@ sidebar_label: Storing Dictionaries in Memory
---
import CloudDetails from '@site/docs/en/sql-reference/dictionaries/external-dictionaries/_snippet_dictionary_in_cloud.md';
-# Storing Dictionaries in Memory
+# Storing Dictionaries in Memory
There are a variety of ways to store dictionaries in memory.
@@ -25,7 +25,7 @@ ClickHouse generates an exception for errors with dictionaries. Examples of erro
You can view the list of dictionaries and their statuses in the [system.dictionaries](../../../operations/system-tables/dictionaries.md) table.
-
+
The configuration looks like this:
@@ -299,11 +299,11 @@ Example: The table contains discounts for each advertiser in the format:
To use a sample for date ranges, define the `range_min` and `range_max` elements in the [structure](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md). These elements must contain elements `name` and `type` (if `type` is not specified, the default type will be used - Date). `type` can be any numeric type (Date / DateTime / UInt64 / Int32 / others).
-:::warning
+:::warning
Values of `range_min` and `range_max` should fit in `Int64` type.
:::
-Example:
+Example:
``` xml
@@ -459,7 +459,7 @@ select dictGet('discounts_dict', 'amount', 1, toDate('2015-01-14')) res;
│ 0.1 │ -- the only one range is matching: 2015-01-01 - Null
└─────┘
-select dictGet('discounts_dict', 'amount', 1, toDate('2015-01-16')) res;
+select dictGet('discounts_dict', 'amount', 1, toDate('2015-01-16')) res;
┌─res─┐
│ 0.2 │ -- two ranges are matching, range_min 2015-01-15 (0.2) is bigger than 2015-01-01 (0.1)
└─────┘
@@ -496,7 +496,7 @@ select dictGet('discounts_dict', 'amount', 1, toDate('2015-01-14')) res;
│ 0.1 │ -- the only one range is matching: 2015-01-01 - Null
└─────┘
-select dictGet('discounts_dict', 'amount', 1, toDate('2015-01-16')) res;
+select dictGet('discounts_dict', 'amount', 1, toDate('2015-01-16')) res;
┌─res─┐
│ 0.1 │ -- two ranges are matching, range_min 2015-01-01 (0.1) is less than 2015-01-15 (0.2)
└─────┘
@@ -588,7 +588,7 @@ Set a large enough cache size. You need to experiment to select the number of ce
3. Assess memory consumption using the `system.dictionaries` table.
4. Increase or decrease the number of cells until the required memory consumption is reached.
-:::warning
+:::warning
Do not use ClickHouse as a source, because it is slow to process queries with random reads.
:::
@@ -660,25 +660,30 @@ This type of storage is for use with composite [keys](../../../sql-reference/dic
This type of storage is for mapping network prefixes (IP addresses) to metadata such as ASN.
-Example: The table contains network prefixes and their corresponding AS number and country code:
+**Example**
-``` text
- +-----------|-----|------+
- | prefix | asn | cca2 |
- +=================+=======+========+
- | 202.79.32.0/20 | 17501 | NP |
- +-----------|-----|------+
- | 2620:0:870::/48 | 3856 | US |
- +-----------|-----|------+
- | 2a02:6b8:1::/48 | 13238 | RU |
- +-----------|-----|------+
- | 2001:db8::/32 | 65536 | ZZ |
- +-----------|-----|------+
+Suppose we have a table in ClickHouse that contains our IP prefixes and mappings:
+
+```sql
+CREATE TABLE my_ip_addresses (
+ prefix String,
+ asn UInt32,
+ cca2 String
+)
+ENGINE = MergeTree
+PRIMARY KEY prefix;
```
-When using this type of layout, the structure must have a composite key.
+```sql
+INSERT INTO my_ip_addresses VALUES
+ ('202.79.32.0/20', 17501, 'NP'),
+ ('2620:0:870::/48', 3856, 'US'),
+ ('2a02:6b8:1::/48', 13238, 'RU'),
+ ('2001:db8::/32', 65536, 'ZZ')
+;
+```
-Example:
+Let's define an `ip_trie` dictionary for this table. The `ip_trie` layout requires a composite key:
``` xml
@@ -712,26 +717,29 @@ Example:
or
``` sql
-CREATE DICTIONARY somedict (
+CREATE DICTIONARY my_ip_trie_dictionary (
prefix String,
asn UInt32,
cca2 String DEFAULT '??'
)
PRIMARY KEY prefix
+SOURCE(CLICKHOUSE(TABLE 'my_ip_addresses'))
+LAYOUT(IP_TRIE)
+LIFETIME(3600);
```
-The key must have only one String type attribute that contains an allowed IP prefix. Other types are not supported yet.
+The key must have only one `String` type attribute that contains an allowed IP prefix. Other types are not supported yet.
-For queries, you must use the same functions (`dictGetT` with a tuple) as for dictionaries with composite keys:
+For queries, you must use the same functions (`dictGetT` with a tuple) as for dictionaries with composite keys. The syntax is:
``` sql
dictGetT('dict_name', 'attr_name', tuple(ip))
```
-The function takes either `UInt32` for IPv4, or `FixedString(16)` for IPv6:
+The function takes either `UInt32` for IPv4, or `FixedString(16)` for IPv6. For example:
``` sql
-dictGetString('prefix', 'asn', tuple(IPv6StringToNum('2001:db8::1')))
+select dictGet('my_ip_trie_dictionary', 'asn', tuple(IPv6StringToNum('2001:db8::1')))
```
Other types are not supported yet. The function returns the attribute for the prefix that corresponds to this IP address. If there are overlapping prefixes, the most specific one is returned.
diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md
index 8ae429973b9..1dc1b60c149 100644
--- a/docs/en/sql-reference/functions/date-time-functions.md
+++ b/docs/en/sql-reference/functions/date-time-functions.md
@@ -1293,31 +1293,31 @@ Similar to formatDateTime, except that it formats datetime in Joda style instead
Using replacement fields, you can define a pattern for the resulting string.
-| Placeholder | Description | Presentation | Examples |
-| ----------- | ----------- | ------------- | -------- |
-| G | era | text | AD |
-| C | century of era (>=0) | number | 20 |
-| Y | year of era (>=0) | year | 1996 |
-| x | weekyear(not supported yet) | year | 1996 |
-| w | week of weekyear(not supported yet) | number | 27 |
-| e | day of week | number | 2 |
-| E | day of week | text | Tuesday; Tue |
-| y | year | year | 1996 |
-| D | day of year | number | 189 |
-| M | month of year | month | July; Jul; 07 |
-| d | day of month | number | 10 |
-| a | halfday of day | text | PM |
-| K | hour of halfday (0~11) | number | 0 |
-| h | clockhour of halfday (1~12) | number | 12 |
-| H | hour of day (0~23) | number | 0 |
-| k | clockhour of day (1~24) | number | 24 |
-| m | minute of hour | number | 30 |
-| s | second of minute | number | 55 |
-| S | fraction of second(not supported yet) | number | 978 |
-| z | time zone(short name not supported yet) | text | Pacific Standard Time; PST |
-| Z | time zone offset/id(not supported yet) | zone | -0800; -08:00; America/Los_Angeles |
-| ' | escape for text | delimiter| |
-| '' | single quote | literal | ' |
+| Placeholder | Description | Presentation | Examples |
+| ----------- | ---------------------------------------- | ------------- | ---------------------------------- |
+| G | era | text | AD |
+| C | century of era (>=0) | number | 20 |
+| Y | year of era (>=0) | year | 1996 |
+| x | weekyear (not supported yet) | year | 1996 |
+| w | week of weekyear (not supported yet) | number | 27 |
+| e | day of week | number | 2 |
+| E | day of week | text | Tuesday; Tue |
+| y | year | year | 1996 |
+| D | day of year | number | 189 |
+| M | month of year | month | July; Jul; 07 |
+| d | day of month | number | 10 |
+| a | halfday of day | text | PM |
+| K | hour of halfday (0~11) | number | 0 |
+| h | clockhour of halfday (1~12) | number | 12 |
+| H | hour of day (0~23) | number | 0 |
+| k | clockhour of day (1~24) | number | 24 |
+| m | minute of hour | number | 30 |
+| s | second of minute | number | 55 |
+| S | fraction of second (not supported yet) | number | 978 |
+| z | time zone (short name not supported yet) | text | Pacific Standard Time; PST |
+| Z | time zone offset/id (not supported yet) | zone | -0800; -08:00; America/Los_Angeles |
+| ' | escape for text | delimiter | |
+| '' | single quote | literal | ' |
**Example**
diff --git a/docs/en/sql-reference/functions/geo/s2.md b/docs/en/sql-reference/functions/geo/s2.md
index ed3c66a0f6f..3cd66cfaaeb 100644
--- a/docs/en/sql-reference/functions/geo/s2.md
+++ b/docs/en/sql-reference/functions/geo/s2.md
@@ -304,7 +304,7 @@ Result:
└──────────────┘
```
-## s2RectUinion
+## s2RectUnion
Returns the smallest rectangle containing the union of this rectangle and the given rectangle. In the S2 system, a rectangle is represented by a type of S2Region called a `S2LatLngRect` that represents a rectangle in latitude-longitude space.
diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md
index 730b494fcb5..ae6cdb7052d 100644
--- a/docs/en/sql-reference/functions/hash-functions.md
+++ b/docs/en/sql-reference/functions/hash-functions.md
@@ -45,37 +45,38 @@ SELECT halfMD5(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')
Calculates the MD4 from a string and returns the resulting set of bytes as FixedString(16).
-## MD5
+## MD5 {#hash_functions-md5}
Calculates the MD5 from a string and returns the resulting set of bytes as FixedString(16).
If you do not need MD5 in particular, but you need a decent cryptographic 128-bit hash, use the ‘sipHash128’ function instead.
If you want to get the same result as output by the md5sum utility, use lower(hex(MD5(s))).
-## sipHash64
+## sipHash64 (#hash_functions-siphash64)
-Produces a 64-bit [SipHash](https://131002.net/siphash/) hash value.
+Produces a 64-bit [SipHash](https://en.wikipedia.org/wiki/SipHash) hash value.
```sql
sipHash64(par1,...)
```
-This is a cryptographic hash function. It works at least three times faster than the [MD5](#hash_functions-md5) function.
+This is a cryptographic hash function. It works at least three times faster than the [MD5](#hash_functions-md5) hash function.
-Function [interprets](/docs/en/sql-reference/functions/type-conversion-functions.md/#type_conversion_functions-reinterpretAsString) all the input parameters as strings and calculates the hash value for each of them. Then combines hashes by the following algorithm:
+The function [interprets](/docs/en/sql-reference/functions/type-conversion-functions.md/#type_conversion_functions-reinterpretAsString) all the input parameters as strings and calculates the hash value for each of them. It then combines the hashes by the following algorithm:
-1. After hashing all the input parameters, the function gets the array of hashes.
-2. Function takes the first and the second elements and calculates a hash for the array of them.
-3. Then the function takes the hash value, calculated at the previous step, and the third element of the initial hash array, and calculates a hash for the array of them.
-4. The previous step is repeated for all the remaining elements of the initial hash array.
+1. The first and the second hash value are concatenated to an array which is hashed.
+2. The previously calculated hash value and the hash of the third input paramter are hashed in a similar way.
+3. This calculation is repeated for all remaining hash values of the original input.
**Arguments**
-The function takes a variable number of input parameters. Arguments can be any of the [supported data types](/docs/en/sql-reference/data-types/index.md). For some data types calculated value of hash function may be the same for the same values even if types of arguments differ (integers of different size, named and unnamed `Tuple` with the same data, `Map` and the corresponding `Array(Tuple(key, value))` type with the same data).
+The function takes a variable number of input parameters of any of the [supported data types](/docs/en/sql-reference/data-types/index.md).
**Returned Value**
A [UInt64](/docs/en/sql-reference/data-types/int-uint.md) data type hash value.
+Note that the calculated hash values may be equal for the same input values of different argument types. This affects for example integer types of different size, named and unnamed `Tuple` with the same data, `Map` and the corresponding `Array(Tuple(key, value))` type with the same data.
+
**Example**
```sql
@@ -84,13 +85,45 @@ SELECT sipHash64(array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00
```response
┌──────────────SipHash─┬─type───┐
-│ 13726873534472839665 │ UInt64 │
+│ 11400366955626497465 │ UInt64 │
└──────────────────────┴────────┘
```
+## sipHash64Keyed
+
+Same as [sipHash64](#hash_functions-siphash64) but additionally takes an explicit key argument instead of using a fixed key.
+
+**Syntax**
+
+```sql
+sipHash64Keyed((k0, k1), par1,...)
+```
+
+**Arguments**
+
+Same as [sipHash64](#hash_functions-siphash64), but the first argument is a tuple of two UInt64 values representing the key.
+
+**Returned value**
+
+A [UInt64](/docs/en/sql-reference/data-types/int-uint.md) data type hash value.
+
+**Example**
+
+Query:
+
+```sql
+SELECT sipHash64Keyed((506097522914230528, 1084818905618843912), array('e','x','a'), 'mple', 10, toDateTime('2019-06-15 23:00:00')) AS SipHash, toTypeName(SipHash) AS type;
+```
+
+```response
+┌─────────────SipHash─┬─type───┐
+│ 8017656310194184311 │ UInt64 │
+└─────────────────────┴────────┘
+```
+
## sipHash128
-Produces a 128-bit [SipHash](https://131002.net/siphash/) hash value. Differs from [sipHash64](#hash_functions-siphash64) in that the final xor-folding state is done up to 128 bits.
+Like [sipHash64](#hash_functions-siphash64) but produces a 128-bit hash value, i.e. the final xor-folding state is done up to 128 bits.
**Syntax**
@@ -100,13 +133,11 @@ sipHash128(par1,...)
**Arguments**
-The function takes a variable number of input parameters. Arguments can be any of the [supported data types](/docs/en/sql-reference/data-types/index.md). For some data types calculated value of hash function may be the same for the same values even if types of arguments differ (integers of different size, named and unnamed `Tuple` with the same data, `Map` and the corresponding `Array(Tuple(key, value))` type with the same data).
+Same as for [sipHash64](#hash_functions-siphash64).
**Returned value**
-A 128-bit `SipHash` hash value.
-
-Type: [FixedString(16)](/docs/en/sql-reference/data-types/fixedstring.md).
+A 128-bit `SipHash` hash value of type [FixedString(16)](/docs/en/sql-reference/data-types/fixedstring.md).
**Example**
@@ -124,6 +155,40 @@ Result:
└──────────────────────────────────┘
```
+## sipHash128Keyed
+
+Same as [sipHash128](#hash_functions-siphash128) but additionally takes an explicit key argument instead of using a fixed key.
+
+**Syntax**
+
+```sql
+sipHash128Keyed((k0, k1), par1,...)
+```
+
+**Arguments**
+
+Same as [sipHash128](#hash_functions-siphash128), but the first argument is a tuple of two UInt64 values representing the key.
+
+**Returned value**
+
+A [UInt64](/docs/en/sql-reference/data-types/int-uint.md) data type hash value.
+
+**Example**
+
+Query:
+
+```sql
+SELECT hex(sipHash128Keyed((506097522914230528, 1084818905618843912),'foo', '\x01', 3));
+```
+
+Result:
+
+```response
+┌─hex(sipHash128Keyed((506097522914230528, 1084818905618843912), 'foo', '', 3))─┐
+│ B8467F65C8B4CFD9A5F8BD733917D9BF │
+└───────────────────────────────────────────────────────────────────────────────┘
+```
+
## cityHash64
Produces a 64-bit [CityHash](https://github.com/google/cityhash) hash value.
diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md
index ca192b685bd..7b14b0e96de 100644
--- a/docs/en/sql-reference/functions/string-search-functions.md
+++ b/docs/en/sql-reference/functions/string-search-functions.md
@@ -95,6 +95,32 @@ Result:
└───────────────────────────────┘
```
+If argument `needle` is empty the following rules apply:
+- if no `start_pos` was specified: return `1`
+- if `start_pos = 0`: return `1`
+- if `start_pos >= 1` and `start_pos <= length(haystack) + 1`: return `start_pos`
+- otherwise: return `0`
+
+The same rules also apply to functions `positionCaseInsensitive`, `positionUTF8` and `positionCaseInsensitiveUTF8`
+
+``` sql
+SELECT
+ position('abc', ''),
+ position('abc', '', 0),
+ position('abc', '', 1),
+ position('abc', '', 2),
+ position('abc', '', 3),
+ position('abc', '', 4),
+ position('abc', '', 5)
+```
+
+``` text
+┌─position('abc', '')─┬─position('abc', '', 0)─┬─position('abc', '', 1)─┬─position('abc', '', 2)─┬─position('abc', '', 3)─┬─position('abc', '', 4)─┬─position('abc', '', 5)─┐
+│ 1 │ 1 │ 1 │ 2 │ 3 │ 4 │ 0 │
+└─────────────────────┴────────────────────────┴────────────────────────┴────────────────────────┴────────────────────────┴────────────────────────┴────────────────────────┘
+```
+
+
**Examples for POSITION(needle IN haystack) syntax**
Query:
diff --git a/docs/en/sql-reference/statements/alter/ttl.md b/docs/en/sql-reference/statements/alter/ttl.md
index a312e8cad91..14865e7bce0 100644
--- a/docs/en/sql-reference/statements/alter/ttl.md
+++ b/docs/en/sql-reference/statements/alter/ttl.md
@@ -6,6 +6,10 @@ sidebar_label: TTL
# Manipulations with Table TTL
+:::note
+If you are looking for details on using TTL for managing old data, check out the [Manage Data with TTL](/docs/en/guides/developer/ttl.md) user guide. The docs below demonstrate how to alter or remove an existing TTL rule.
+:::
+
## MODIFY TTL
You can change [table TTL](../../../engines/table-engines/mergetree-family/mergetree.md#mergetree-table-ttl) with a request of the following form:
diff --git a/docs/en/sql-reference/statements/create/dictionary.md b/docs/en/sql-reference/statements/create/dictionary.md
index a470b071971..e789dd9257f 100644
--- a/docs/en/sql-reference/statements/create/dictionary.md
+++ b/docs/en/sql-reference/statements/create/dictionary.md
@@ -110,7 +110,7 @@ LIFETIME(MIN 0 MAX 1000)
### Create a dictionary from a file available by HTTP(S)
```sql
-statement: CREATE DICTIONARY default.taxi_zone_dictionary
+CREATE DICTIONARY default.taxi_zone_dictionary
(
`LocationID` UInt16 DEFAULT 0,
`Borough` String,
diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md
index a2d363da042..119f25d6d00 100644
--- a/docs/en/sql-reference/statements/create/table.md
+++ b/docs/en/sql-reference/statements/create/table.md
@@ -3,6 +3,7 @@ slug: /en/sql-reference/statements/create/table
sidebar_position: 36
sidebar_label: TABLE
title: "CREATE TABLE"
+keywords: [compression, codec, schema, DDL]
---
Creates a new table. This query can have various syntax forms depending on a use case.
diff --git a/docs/en/sql-reference/statements/delete.md b/docs/en/sql-reference/statements/delete.md
index 0dc6cc0d09a..0acb6637ea6 100644
--- a/docs/en/sql-reference/statements/delete.md
+++ b/docs/en/sql-reference/statements/delete.md
@@ -7,7 +7,7 @@ sidebar_label: DELETE
# DELETE Statement
``` sql
-DELETE FROM [db.]table [WHERE expr]
+DELETE FROM [db.]table [ON CLUSTER cluster] [WHERE expr]
```
`DELETE FROM` removes rows from table `[db.]table` that match expression `expr`. The deleted rows are marked as deleted immediately and will be automatically filtered out of all subsequent queries. Cleanup of data happens asynchronously in background. This feature is only available for MergeTree table engine family.
diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md
index 16fcb7b0c07..18b019dd017 100644
--- a/docs/en/sql-reference/statements/show.md
+++ b/docs/en/sql-reference/statements/show.md
@@ -510,3 +510,15 @@ Result:
**See Also**
- [system.settings](../../operations/system-tables/settings.md) table
+
+## SHOW ENGINES
+
+``` sql
+SHOW ENGINES [INTO OUTFILE filename] [FORMAT format]
+```
+
+Outputs the content of the [system.table_engines](../../operations/system-tables/table_engines.md) table, that contains description of table engines supported by server and their feature support information.
+
+**See Also**
+
+- [system.table_engines](../../operations/system-tables/table_engines.md) table
\ No newline at end of file
diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md
index 6c62229f0d5..300205a7ef4 100644
--- a/docs/en/sql-reference/statements/system.md
+++ b/docs/en/sql-reference/statements/system.md
@@ -103,9 +103,9 @@ Its size can be configured using the server-level setting [uncompressed_cache_si
Reset the compiled expression cache.
The compiled expression cache is enabled/disabled with the query/user/profile-level setting [compile_expressions](../../operations/settings/settings.md#compile-expressions).
-## DROP QUERY RESULT CACHE
+## DROP QUERY CACHE
-Resets the [query result cache](../../operations/query-result-cache.md).
+Resets the [query cache](../../operations/query-cache.md).
## FLUSH LOGS
@@ -362,3 +362,15 @@ Allows to drop filesystem cache.
```sql
SYSTEM DROP FILESYSTEM CACHE
```
+
+### SYNC FILE CACHE
+
+:::note
+It's too heavy and has potential for misuse.
+:::
+
+Will do sync syscall.
+
+```sql
+SYSTEM SYNC FILE CACHE
+```
diff --git a/docs/en/sql-reference/table-functions/generate.md b/docs/en/sql-reference/table-functions/generate.md
index 90d81e4f74e..b53ccdd42b5 100644
--- a/docs/en/sql-reference/table-functions/generate.md
+++ b/docs/en/sql-reference/table-functions/generate.md
@@ -8,7 +8,7 @@ sidebar_label: generateRandom
Generates random data with given schema.
Allows to populate test tables with data.
-Supports all data types that can be stored in table except `LowCardinality` and `AggregateFunction`.
+Not all types are supported.
``` sql
generateRandom('name TypeName[, name TypeName]...', [, 'random_seed'[, 'max_string_length'[, 'max_array_length']]])
@@ -18,7 +18,7 @@ generateRandom('name TypeName[, name TypeName]...', [, 'random_seed'[, 'max_stri
- `name` — Name of corresponding column.
- `TypeName` — Type of corresponding column.
-- `max_array_length` — Maximum array length for all generated arrays. Defaults to `10`.
+- `max_array_length` — Maximum elements for all generated arrays or maps. Defaults to `10`.
- `max_string_length` — Maximum string length for all generated strings. Defaults to `10`.
- `random_seed` — Specify random seed manually to produce stable results. If NULL — seed is randomly generated.
diff --git a/docs/tools/.gitignore b/docs/tools/.gitignore
index 443cee8638c..8d35cb3277f 100644
--- a/docs/tools/.gitignore
+++ b/docs/tools/.gitignore
@@ -1,3 +1,2 @@
-build
__pycache__
*.pyc
diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp
index 1ce29555f2f..d741eb30d4a 100644
--- a/programs/client/Client.cpp
+++ b/programs/client/Client.cpp
@@ -127,6 +127,69 @@ void Client::showWarnings()
}
}
+void Client::parseConnectionsCredentials()
+{
+ /// It is not possible to correctly handle multiple --host --port options.
+ if (hosts_and_ports.size() >= 2)
+ return;
+
+ String host;
+ std::optional port;
+ if (hosts_and_ports.empty())
+ {
+ host = config().getString("host", "localhost");
+ if (config().has("port"))
+ port = config().getInt("port");
+ }
+ else
+ {
+ host = hosts_and_ports.front().host;
+ port = hosts_and_ports.front().port;
+ }
+
+ Strings keys;
+ config().keys("connections_credentials", keys);
+ for (const auto & connection : keys)
+ {
+ const String & prefix = "connections_credentials." + connection;
+
+ const String & connection_name = config().getString(prefix + ".name", "");
+ if (connection_name != host)
+ continue;
+
+ String connection_hostname;
+ if (config().has(prefix + ".hostname"))
+ connection_hostname = config().getString(prefix + ".hostname");
+ else
+ connection_hostname = connection_name;
+
+ /// Set "host" unconditionally (since it is used as a "name"), while
+ /// other options only if they are not set yet (config.xml/cli
+ /// options).
+ config().setString("host", connection_hostname);
+ if (!hosts_and_ports.empty())
+ hosts_and_ports.front().host = connection_hostname;
+
+ if (config().has(prefix + ".port") && !port.has_value())
+ config().setInt("port", config().getInt(prefix + ".port"));
+ if (config().has(prefix + ".secure") && !config().has("secure"))
+ config().setBool("secure", config().getBool(prefix + ".secure"));
+ if (config().has(prefix + ".user") && !config().has("user"))
+ config().setString("user", config().getString(prefix + ".user"));
+ if (config().has(prefix + ".password") && !config().has("password"))
+ config().setString("password", config().getString(prefix + ".password"));
+ if (config().has(prefix + ".database") && !config().has("database"))
+ config().setString("database", config().getString(prefix + ".database"));
+ if (config().has(prefix + ".history_file") && !config().has("history_file"))
+ {
+ String history_file = config().getString(prefix + ".history_file");
+ if (history_file.starts_with("~") && !home_path.empty())
+ history_file = home_path + "/" + history_file.substr(1);
+ config().setString("history_file", history_file);
+ }
+ }
+}
+
/// Make query to get all server warnings
std::vector Client::loadWarningMessages()
{
@@ -216,6 +279,8 @@ void Client::initialize(Poco::Util::Application & self)
if (env_password)
config().setString("password", env_password);
+ parseConnectionsCredentials();
+
// global_context->setApplicationType(Context::ApplicationType::CLIENT);
global_context->setQueryParameters(query_parameters);
diff --git a/programs/client/Client.h b/programs/client/Client.h
index 63f28ca96a2..9f593ea4165 100644
--- a/programs/client/Client.h
+++ b/programs/client/Client.h
@@ -47,6 +47,7 @@ protected:
private:
void printChangedSettings() const;
void showWarnings();
+ void parseConnectionsCredentials();
std::vector loadWarningMessages();
};
}
diff --git a/programs/client/clickhouse-client.xml b/programs/client/clickhouse-client.xml
index 2923de44045..dbfb267d778 100644
--- a/programs/client/clickhouse-client.xml
+++ b/programs/client/clickhouse-client.xml
@@ -57,4 +57,28 @@
The same can be done on user-level configuration, just create & adjust: ~/.clickhouse-client/config.xml
-->
+
+
+
+
+
+
+ default
+
+ 127.0.0.1
+ 9000
+ 1
+ default
+
+
+
+
+
+
+ ]]>
diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp
index 816fa561a6a..256b40414c5 100644
--- a/programs/copier/ClusterCopier.cpp
+++ b/programs/copier/ClusterCopier.cpp
@@ -6,7 +6,6 @@
#include
#include
#include
-#include
#include
#include
#include
diff --git a/programs/odbc-bridge/IdentifierQuoteHandler.cpp b/programs/odbc-bridge/IdentifierQuoteHandler.cpp
index 8157e3b6159..f622995bf15 100644
--- a/programs/odbc-bridge/IdentifierQuoteHandler.cpp
+++ b/programs/odbc-bridge/IdentifierQuoteHandler.cpp
@@ -61,13 +61,18 @@ void IdentifierQuoteHandler::handleRequest(HTTPServerRequest & request, HTTPServ
return;
}
+ bool use_connection_pooling = params.getParsed("use_connection_pooling", true);
+
try
{
std::string connection_string = params.get("connection_string");
- auto connection = ODBCPooledConnectionFactory::instance().get(
- validateODBCConnectionString(connection_string),
- getContext()->getSettingsRef().odbc_bridge_connection_pool_size);
+ nanodbc::ConnectionHolderPtr connection;
+ if (use_connection_pooling)
+ connection = ODBCPooledConnectionFactory::instance().get(
+ validateODBCConnectionString(connection_string), getContext()->getSettingsRef().odbc_bridge_connection_pool_size);
+ else
+ connection = std::make_shared(validateODBCConnectionString(connection_string));
auto identifier = getIdentifierQuote(std::move(connection));
diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp
index 0875cc2e9d9..9130b3e0f47 100644
--- a/programs/odbc-bridge/MainHandler.cpp
+++ b/programs/odbc-bridge/MainHandler.cpp
@@ -102,7 +102,9 @@ void ODBCHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse
std::string format = params.get("format", "RowBinary");
std::string connection_string = params.get("connection_string");
+ bool use_connection_pooling = params.getParsed("use_connection_pooling", true);
LOG_TRACE(log, "Connection string: '{}'", connection_string);
+ LOG_TRACE(log, "Use pooling: {}", use_connection_pooling);
UInt64 max_block_size = DEFAULT_BLOCK_SIZE;
if (params.has("max_block_size"))
@@ -134,7 +136,7 @@ void ODBCHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse
try
{
nanodbc::ConnectionHolderPtr connection_handler;
- if (getContext()->getSettingsRef().odbc_bridge_use_connection_pooling)
+ if (use_connection_pooling)
connection_handler = ODBCPooledConnectionFactory::instance().get(
validateODBCConnectionString(connection_string), getContext()->getSettingsRef().odbc_bridge_connection_pool_size);
else
diff --git a/programs/odbc-bridge/SchemaAllowedHandler.cpp b/programs/odbc-bridge/SchemaAllowedHandler.cpp
index 4d20c8bc3b7..020359f51fd 100644
--- a/programs/odbc-bridge/SchemaAllowedHandler.cpp
+++ b/programs/odbc-bridge/SchemaAllowedHandler.cpp
@@ -70,13 +70,19 @@ void SchemaAllowedHandler::handleRequest(HTTPServerRequest & request, HTTPServer
return;
}
+ bool use_connection_pooling = params.getParsed("use_connection_pooling", true);
+
try
{
std::string connection_string = params.get("connection_string");
- auto connection = ODBCPooledConnectionFactory::instance().get(
- validateODBCConnectionString(connection_string),
- getContext()->getSettingsRef().odbc_bridge_connection_pool_size);
+ nanodbc::ConnectionHolderPtr connection;
+
+ if (use_connection_pooling)
+ connection = ODBCPooledConnectionFactory::instance().get(
+ validateODBCConnectionString(connection_string), getContext()->getSettingsRef().odbc_bridge_connection_pool_size);
+ else
+ connection = std::make_shared(validateODBCConnectionString(connection_string));
bool result = isSchemaAllowed(std::move(connection));
diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index 8b6c43b450e..7b1ab1b8180 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -1517,13 +1517,13 @@ try
global_context->setMMappedFileCache(mmap_cache_size);
/// A cache for query results.
- size_t query_result_cache_size = config().getUInt64("query_result_cache.size", 1_GiB);
- if (query_result_cache_size)
- global_context->setQueryResultCache(
- query_result_cache_size,
- config().getUInt64("query_result_cache.max_entries", 1024),
- config().getUInt64("query_result_cache.max_entry_size", 1_MiB),
- config().getUInt64("query_result_cache.max_entry_records", 30'000'000));
+ size_t query_cache_size = config().getUInt64("query_cache.size", 1_GiB);
+ if (query_cache_size)
+ global_context->setQueryCache(
+ query_cache_size,
+ config().getUInt64("query_cache.max_entries", 1024),
+ config().getUInt64("query_cache.max_entry_size", 1_MiB),
+ config().getUInt64("query_cache.max_entry_records", 30'000'000));
#if USE_EMBEDDED_COMPILER
/// 128 MB
diff --git a/programs/server/config.xml b/programs/server/config.xml
index 0ed8ec48e83..bd46263f851 100644
--- a/programs/server/config.xml
+++ b/programs/server/config.xml
@@ -1466,13 +1466,13 @@
-->
-
-
+
+
-
+
always return 1
+ if (start_pos == nullptr)
+ {
+ for (auto & r : res)
+ r = 1;
+ return;
+ }
+
+ ColumnString::Offset prev_offset = 0;
+ size_t rows = haystack_offsets.size();
+
+ if (const ColumnConst * start_pos_const = typeid_cast(&*start_pos))
+ {
+ /// Needle is empty and start_pos is constant
+ UInt64 start = std::max(start_pos_const->getUInt(0), static_cast(1));
+ for (size_t i = 0; i < rows; ++i)
+ {
+ size_t haystack_size = Impl::countChars(
+ reinterpret_cast(pos), reinterpret_cast(pos + haystack_offsets[i] - prev_offset - 1));
+ res[i] = (start <= haystack_size + 1) ? start : 0;
+
+ pos = begin + haystack_offsets[i];
+ prev_offset = haystack_offsets[i];
+ }
+ return;
+ }
+ else
+ {
+ /// Needle is empty and start_pos is not constant
+ for (size_t i = 0; i < rows; ++i)
+ {
+ size_t haystack_size = Impl::countChars(
+ reinterpret_cast(pos), reinterpret_cast(pos + haystack_offsets[i] - prev_offset - 1));
+ UInt64 start = start_pos->getUInt(i);
+ start = std::max(static_cast(1), start);
+ res[i] = (start <= haystack_size + 1) ? start : 0;
+
+ pos = begin + haystack_offsets[i];
+ prev_offset = haystack_offsets[i];
+ }
+ return;
+ }
+ }
+
/// Current index in the array of strings.
size_t i = 0;
@@ -253,7 +300,7 @@ struct PositionImpl
{
auto start = std::max(start_pos, UInt64(1));
- if (needle.size() == 0)
+ if (needle.empty())
{
size_t haystack_size = Impl::countChars(data.data(), data.data() + data.size());
res = start <= haystack_size + 1 ? start : 0;
diff --git a/src/Functions/array/arrayShuffle.cpp b/src/Functions/array/arrayShuffle.cpp
new file mode 100644
index 00000000000..9cf3ac8f3fe
--- /dev/null
+++ b/src/Functions/array/arrayShuffle.cpp
@@ -0,0 +1,227 @@
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include
+
+#include
+#include
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+ extern const int ILLEGAL_COLUMN;
+ extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+ extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+}
+
+/** Shuffle array elements
+ * arrayShuffle(arr)
+ * arrayShuffle(arr, seed)
+ */
+struct FunctionArrayShuffleTraits
+{
+ static constexpr auto name = "arrayShuffle";
+ static constexpr auto has_limit = false; // Permute the whole array
+ static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {1}; }
+ static constexpr auto max_num_params = 2; // array[, seed]
+ static constexpr auto seed_param_idx = 1; // --------^^^^
+};
+
+/** Partial shuffle array elements
+ * arrayPartialShuffle(arr)
+ * arrayPartialShuffle(arr, limit)
+ * arrayPartialShuffle(arr, limit, seed)
+ */
+struct FunctionArrayPartialShuffleTraits
+{
+ static constexpr auto name = "arrayPartialShuffle";
+ static constexpr auto has_limit = true;
+ static ColumnNumbers getArgumentsThatAreAlwaysConstant() { return {1, 2}; }
+ static constexpr auto max_num_params = 3; // array[, limit[, seed]]
+ static constexpr auto seed_param_idx = 2; // ----------------^^^^
+};
+
+template
+class FunctionArrayShuffleImpl : public IFunction
+{
+public:
+ static constexpr auto name = Traits::name;
+
+ String getName() const override { return name; }
+ bool isVariadic() const override { return true; }
+ size_t getNumberOfArguments() const override { return 0; }
+ ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return Traits::getArgumentsThatAreAlwaysConstant(); }
+ bool useDefaultImplementationForConstants() const override { return true; }
+ bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
+
+ static FunctionPtr create(ContextPtr) { return std::make_shared>(); }
+
+ DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
+ {
+ if (arguments.size() > Traits::max_num_params || arguments.empty())
+ {
+ throw Exception(
+ ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
+ "Function '{}' needs from 1 to {} arguments, passed {}.",
+ getName(),
+ Traits::max_num_params,
+ arguments.size());
+ }
+
+ const DataTypeArray * array_type = checkAndGetDataType(arguments[0].get());
+ if (!array_type)
+ throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument of function '{}' must be array", getName());
+
+ auto check_is_integral = [&](auto param_idx)
+ {
+ WhichDataType which(arguments[param_idx]);
+ if (!which.isUInt() && !which.isInt())
+ throw Exception(
+ ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+ "Illegal type {} of arguments of function {} (must be UInt or Int)",
+ arguments[param_idx]->getName(),
+ getName());
+ };
+
+ for (size_t idx = 1; idx < arguments.size(); ++idx)
+ check_is_integral(idx);
+
+ return arguments[0];
+ }
+
+ ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t) const override;
+
+private:
+ static ColumnPtr executeGeneric(const ColumnArray & array, pcg64_fast & rng, size_t limit);
+};
+
+template
+ColumnPtr FunctionArrayShuffleImpl::executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const
+{
+ const ColumnArray * array = checkAndGetColumn(arguments[0].column.get());
+ if (!array)
+ throw Exception(
+ ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first argument of function {}", arguments[0].column->getName(), getName());
+
+ const auto seed = [&]() -> uint64_t
+ {
+ // If present, seed comes as the last argument
+ if (arguments.size() != Traits::max_num_params)
+ return randomSeed();
+ const auto * val = arguments[Traits::seed_param_idx].column.get();
+ return val->getUInt(0);
+ }();
+ pcg64_fast rng(seed);
+
+ size_t limit = [&]() -> size_t
+ {
+ if constexpr (Traits::has_limit)
+ {
+ if (arguments.size() > 1)
+ {
+ const auto * val = arguments[1].column.get();
+ return val->getUInt(0);
+ }
+ }
+ return 0;
+ }();
+
+ return executeGeneric(*array, rng, limit);
+}
+
+template
+ColumnPtr FunctionArrayShuffleImpl::executeGeneric(const ColumnArray & array, pcg64_fast & rng, size_t limit [[maybe_unused]])
+{
+ const ColumnArray::Offsets & offsets = array.getOffsets();
+
+ size_t size = offsets.size();
+ size_t nested_size = array.getData().size();
+ IColumn::Permutation permutation(nested_size);
+ std::iota(std::begin(permutation), std::end(permutation), 0);
+
+ ColumnArray::Offset current_offset = 0;
+ for (size_t i = 0; i < size; ++i)
+ {
+ auto next_offset = offsets[i];
+ if constexpr (Traits::has_limit)
+ {
+ if (limit)
+ {
+ const auto effective_limit = std::min(limit, next_offset - current_offset);
+ partial_shuffle(&permutation[current_offset], &permutation[next_offset], effective_limit, rng);
+ }
+ else
+ shuffle(&permutation[current_offset], &permutation[next_offset], rng);
+ }
+ else
+ shuffle(&permutation[current_offset], &permutation[next_offset], rng);
+ current_offset = next_offset;
+ }
+ return ColumnArray::create(array.getData().permute(permutation, 0), array.getOffsetsPtr());
+}
+
+REGISTER_FUNCTION(ArrayShuffle)
+{
+ factory.registerFunction>(
+ {
+ R"(
+Returns an array of the same size as the original array containing the elements in shuffled order.
+Elements are being reordered in such a way that each possible permutation of those elements has equal probability of appearance.
+
+Note: this function will not materialize constants:
+[example:materialize]
+
+If no seed is provided a random one will be used:
+[example:random_seed]
+
+It is possible to override the seed to produce stable results:
+[example:explicit_seed]
+)",
+ Documentation::Examples{
+ {"random_seed", "SELECT arrayShuffle([1, 2, 3, 4])"},
+ {"explicit_seed", "SELECT arrayShuffle([1, 2, 3, 4], 41)"},
+ {"materialize", "SELECT arrayShuffle(materialize([1, 2, 3]), 42), arrayShuffle([1, 2, 3], 42) FROM numbers(10)"}},
+ Documentation::Categories{"Array"}},
+ FunctionFactory::CaseInsensitive);
+ factory.registerFunction>(
+ {
+ R"(
+Returns an array of the same size as the original array where elements in range [1..limit] are a random
+subset of the original array. Remaining (limit..n] shall contain the elements not in [1..limit] range in undefined order.
+Value of limit shall be in range [1..n]. Values outside of that range are equivalent to performing full arrayShuffle:
+[example:no_limit1]
+[example:no_limit2]
+
+Note: this function will not materialize constants:
+[example:materialize]
+
+If no seed is provided a random one will be used:
+[example:random_seed]
+
+It is possible to override the seed to produce stable results:
+[example:explicit_seed]
+)",
+ Documentation::Examples{
+ {"no_limit1", "SELECT arrayPartialShuffle([1, 2, 3, 4], 0)"},
+ {"no_limit2", "SELECT arrayPartialShuffle([1, 2, 3, 4])"},
+ {"random_seed", "SELECT arrayPartialShuffle([1, 2, 3, 4], 2)"},
+ {"explicit_seed", "SELECT arrayPartialShuffle([1, 2, 3, 4], 2, 41)"},
+ {"materialize",
+ "SELECT arrayPartialShuffle(materialize([1, 2, 3, 4]), 2, 42), arrayPartialShuffle([1, 2, 3], 2, 42) FROM numbers(10)"}},
+ Documentation::Categories{"Array"}},
+ FunctionFactory::CaseInsensitive);
+}
+
+}
diff --git a/src/Functions/array/length.cpp b/src/Functions/array/length.cpp
index ac1db5c406b..65555a501e8 100644
--- a/src/Functions/array/length.cpp
+++ b/src/Functions/array/length.cpp
@@ -43,6 +43,16 @@ struct LengthImpl
{
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function length to UUID argument");
}
+
+ [[noreturn]] static void ipv6(const ColumnIPv6::Container &, size_t &, PaddedPODArray &)
+ {
+ throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function length to IPv6 argument");
+ }
+
+ [[noreturn]] static void ipv4(const ColumnIPv4::Container &, size_t &, PaddedPODArray &)
+ {
+ throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function length to IPv4 argument");
+ }
};
diff --git a/src/Functions/ascii.cpp b/src/Functions/ascii.cpp
index cb59be55cc1..5632f0d15df 100644
--- a/src/Functions/ascii.cpp
+++ b/src/Functions/ascii.cpp
@@ -60,6 +60,16 @@ struct AsciiImpl
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function {} to UUID argument", AsciiName::name);
}
+ [[noreturn]] static void ipv6(const ColumnIPv6::Container & /*offsets*/, size_t /*n*/, PaddedPODArray & /*res*/)
+ {
+ throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function {} to IPv6 argument", AsciiName::name);
+ }
+
+ [[noreturn]] static void ipv4(const ColumnIPv4::Container & /*offsets*/, size_t /*n*/, PaddedPODArray & /*res*/)
+ {
+ throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function {} to IPv4 argument", AsciiName::name);
+ }
+
private:
static Int32 doAscii(const ColumnString::Chars & buf, size_t offset, size_t size)
{
diff --git a/src/Functions/formatDateTime.cpp b/src/Functions/formatDateTime.cpp
index ea8a490f279..630add20835 100644
--- a/src/Functions/formatDateTime.cpp
+++ b/src/Functions/formatDateTime.cpp
@@ -478,7 +478,7 @@ private:
return res.size();
}
- static size_t jodaCentryOfEra(size_t min_represent_digits, char * dest, Time source, UInt64, UInt32, const DateLUTImpl & timezone)
+ static size_t jodaCenturyOfEra(size_t min_represent_digits, char * dest, Time source, UInt64, UInt32, const DateLUTImpl & timezone)
{
auto year = static_cast(ToYearImpl::execute(source, timezone));
year = (year < 0 ? -year : year);
@@ -1137,7 +1137,7 @@ public:
reserve_size += repetitions <= 3 ? 2 : 13;
break;
case 'C':
- instructions.emplace_back(std::bind_front(&Action::jodaCentryOfEra, repetitions));
+ instructions.emplace_back(std::bind_front(&Action::jodaCenturyOfEra, repetitions));
/// Year range [1900, 2299]
reserve_size += std::max(repetitions, 2);
break;
diff --git a/src/Functions/isValidUTF8.cpp b/src/Functions/isValidUTF8.cpp
index adfca3f6546..0871e82adb8 100644
--- a/src/Functions/isValidUTF8.cpp
+++ b/src/Functions/isValidUTF8.cpp
@@ -248,6 +248,16 @@ SOFTWARE.
{
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function isValidUTF8 to UUID argument");
}
+
+ [[noreturn]] static void ipv6(const ColumnIPv6::Container &, size_t &, PaddedPODArray &)
+ {
+ throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function isValidUTF8 to IPv6 argument");
+ }
+
+ [[noreturn]] static void ipv4(const ColumnIPv4::Container &, size_t &, PaddedPODArray &)
+ {
+ throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function isValidUTF8 to IPv4 argument");
+ }
};
struct NameIsValidUTF8
diff --git a/src/Functions/lengthUTF8.cpp b/src/Functions/lengthUTF8.cpp
index 457725a5775..5a4af4934df 100644
--- a/src/Functions/lengthUTF8.cpp
+++ b/src/Functions/lengthUTF8.cpp
@@ -58,6 +58,16 @@ struct LengthUTF8Impl
{
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function lengthUTF8 to UUID argument");
}
+
+ [[noreturn]] static void ipv6(const ColumnIPv6::Container &, size_t &, PaddedPODArray &)
+ {
+ throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function lengthUTF8 to IPv6 argument");
+ }
+
+ [[noreturn]] static void ipv4(const ColumnIPv4::Container &, size_t &, PaddedPODArray &)
+ {
+ throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function lengthUTF8 to IPv4 argument");
+ }
};
struct NameLengthUTF8
diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp
index 0cf77fbee4f..6cc98464af2 100644
--- a/src/IO/ReadBufferFromS3.cpp
+++ b/src/IO/ReadBufferFromS3.cpp
@@ -5,6 +5,7 @@
#include
#include
+#include
#include
#include
@@ -253,7 +254,7 @@ size_t ReadBufferFromS3::getFileSize()
if (file_size)
return *file_size;
- auto object_size = S3::getObjectSize(*client_ptr, bucket, key, version_id, /* for_disk_s3= */ read_settings.for_object_storage);
+ auto object_size = S3::getObjectSize(*client_ptr, bucket, key, version_id, request_settings, /* for_disk_s3= */ read_settings.for_object_storage);
file_size = object_size;
return *file_size;
diff --git a/src/IO/ReadSettings.h b/src/IO/ReadSettings.h
index 3d8f9a05e53..4a0344bf11b 100644
--- a/src/IO/ReadSettings.h
+++ b/src/IO/ReadSettings.h
@@ -31,6 +31,13 @@ enum class LocalFSReadMethod
*/
mmap,
+ /**
+ * Use the io_uring Linux subsystem for asynchronous reads.
+ * Can use direct IO after specified size.
+ * Can do prefetch with double buffering.
+ */
+ io_uring,
+
/**
* Checks if data is in page cache with 'preadv2' on modern Linux kernels.
* If data is in page cache, read from the same thread.
diff --git a/src/IO/S3/copyS3File.cpp b/src/IO/S3/copyS3File.cpp
index 586768154a7..c4f9718e906 100644
--- a/src/IO/S3/copyS3File.cpp
+++ b/src/IO/S3/copyS3File.cpp
@@ -5,6 +5,7 @@
#include
#include
#include
+#include
#include
#include
@@ -64,9 +65,8 @@ namespace
: client_ptr(client_ptr_)
, dest_bucket(dest_bucket_)
, dest_key(dest_key_)
- , settings(request_settings_.getUploadSettings())
- , check_objects_after_upload(request_settings_.check_objects_after_upload)
- , max_unexpected_write_error_retries(request_settings_.max_unexpected_write_error_retries)
+ , request_settings(request_settings_)
+ , upload_settings(request_settings.getUploadSettings())
, object_metadata(object_metadata_)
, schedule(schedule_)
, for_disk_s3(for_disk_s3_)
@@ -80,9 +80,8 @@ namespace
std::shared_ptr client_ptr;
const String & dest_bucket;
const String & dest_key;
- const S3Settings::RequestSettings::PartUploadSettings & settings;
- bool check_objects_after_upload;
- size_t max_unexpected_write_error_retries;
+ const S3Settings::RequestSettings & request_settings;
+ const S3Settings::RequestSettings::PartUploadSettings & upload_settings;
const std::optional> & object_metadata;
ThreadPoolCallbackRunner schedule;
bool for_disk_s3;
@@ -119,8 +118,9 @@ namespace
if (object_metadata.has_value())
request.SetMetadata(object_metadata.value());
- if (!settings.storage_class_name.empty())
- request.SetStorageClass(Aws::S3::Model::StorageClassMapper::GetStorageClassForName(settings.storage_class_name));
+ const auto & storage_class_name = upload_settings.storage_class_name;
+ if (!storage_class_name.empty())
+ request.SetStorageClass(Aws::S3::Model::StorageClassMapper::GetStorageClassForName(storage_class_name));
ProfileEvents::increment(ProfileEvents::S3CreateMultipartUpload);
if (for_disk_s3)
@@ -161,7 +161,7 @@ namespace
request.SetMultipartUpload(multipart_upload);
- size_t max_retries = std::max(max_unexpected_write_error_retries, 1UL);
+ size_t max_retries = std::max(request_settings.max_unexpected_write_error_retries, 1UL);
for (size_t retries = 1;; ++retries)
{
ProfileEvents::increment(ProfileEvents::S3CompleteMultipartUpload);
@@ -205,7 +205,7 @@ namespace
void checkObjectAfterUpload()
{
LOG_TRACE(log, "Checking object {} exists after upload", dest_key);
- S3::checkObjectExists(*client_ptr, dest_bucket, dest_key, {}, {}, "Immediately after upload");
+ S3::checkObjectExists(*client_ptr, dest_bucket, dest_key, {}, request_settings, {}, "Immediately after upload");
LOG_TRACE(log, "Object {} exists after upload", dest_key);
}
@@ -239,47 +239,49 @@ namespace
if (!total_size)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Chosen multipart upload for an empty file. This must not happen");
- if (!settings.max_part_number)
+ auto max_part_number = upload_settings.max_part_number;
+ auto min_upload_part_size = upload_settings.min_upload_part_size;
+ auto max_upload_part_size = upload_settings.max_upload_part_size;
+
+ if (!max_part_number)
throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "max_part_number must not be 0");
- else if (!settings.min_upload_part_size)
+ else if (!min_upload_part_size)
throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "min_upload_part_size must not be 0");
- else if (settings.max_upload_part_size < settings.min_upload_part_size)
+ else if (max_upload_part_size < min_upload_part_size)
throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "max_upload_part_size must not be less than min_upload_part_size");
- size_t part_size = settings.min_upload_part_size;
+ size_t part_size = min_upload_part_size;
size_t num_parts = (total_size + part_size - 1) / part_size;
- if (num_parts > settings.max_part_number)
+ if (num_parts > max_part_number)
{
- part_size = (total_size + settings.max_part_number - 1) / settings.max_part_number;
+ part_size = (total_size + max_part_number - 1) / max_part_number;
num_parts = (total_size + part_size - 1) / part_size;
}
- if (part_size > settings.max_upload_part_size)
+ if (part_size > max_upload_part_size)
{
- part_size = settings.max_upload_part_size;
+ part_size = max_upload_part_size;
num_parts = (total_size + part_size - 1) / part_size;
}
- if (num_parts < 1 || num_parts > settings.max_part_number || part_size < settings.min_upload_part_size
- || part_size > settings.max_upload_part_size)
+ if (num_parts < 1 || num_parts > max_part_number || part_size < min_upload_part_size || part_size > max_upload_part_size)
{
String msg;
if (num_parts < 1)
msg = "Number of parts is zero";
- else if (num_parts > settings.max_part_number)
- msg = fmt::format("Number of parts exceeds {}", num_parts, settings.max_part_number);
- else if (part_size < settings.min_upload_part_size)
- msg = fmt::format("Size of a part is less than {}", part_size, settings.min_upload_part_size);
+ else if (num_parts > max_part_number)
+ msg = fmt::format("Number of parts exceeds {}", num_parts, max_part_number);
+ else if (part_size < min_upload_part_size)
+ msg = fmt::format("Size of a part is less than {}", part_size, min_upload_part_size);
else
- msg = fmt::format("Size of a part exceeds {}", part_size, settings.max_upload_part_size);
+ msg = fmt::format("Size of a part exceeds {}", part_size, max_upload_part_size);
throw Exception(
ErrorCodes::INVALID_CONFIG_PARAMETER,
"{} while writing {} bytes to S3. Check max_part_number = {}, "
- "min_upload_part_size = {}, max_upload_part_size = {}, max_single_part_upload_size = {}",
- msg, total_size, settings.max_part_number, settings.min_upload_part_size,
- settings.max_upload_part_size, settings.max_single_part_upload_size);
+ "min_upload_part_size = {}, max_upload_part_size = {}",
+ msg, total_size, max_part_number, min_upload_part_size, max_upload_part_size);
}
/// We've calculated the size of a normal part (the final part can be smaller).
@@ -418,12 +420,12 @@ namespace
void performCopy()
{
- if (size <= settings.max_single_part_upload_size)
+ if (size <= upload_settings.max_single_part_upload_size)
performSinglepartUpload();
else
performMultipartUpload();
- if (check_objects_after_upload)
+ if (request_settings.check_objects_after_upload)
checkObjectAfterUpload();
}
@@ -451,8 +453,9 @@ namespace
if (object_metadata.has_value())
request.SetMetadata(object_metadata.value());
- if (!settings.storage_class_name.empty())
- request.SetStorageClass(Aws::S3::Model::StorageClassMapper::GetStorageClassForName(settings.storage_class_name));
+ const auto & storage_class_name = upload_settings.storage_class_name;
+ if (!storage_class_name.empty())
+ request.SetStorageClass(Aws::S3::Model::StorageClassMapper::GetStorageClassForName(storage_class_name));
/// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840
request.SetContentType("binary/octet-stream");
@@ -460,7 +463,7 @@ namespace
void processPutRequest(const Aws::S3::Model::PutObjectRequest & request)
{
- size_t max_retries = std::max(max_unexpected_write_error_retries, 1UL);
+ size_t max_retries = std::max(request_settings.max_unexpected_write_error_retries, 1UL);
for (size_t retries = 1;; ++retries)
{
ProfileEvents::increment(ProfileEvents::S3PutObject);
@@ -582,12 +585,12 @@ namespace
void performCopy()
{
- if (size <= settings.max_single_operation_copy_size)
+ if (size <= upload_settings.max_single_operation_copy_size)
performSingleOperationCopy();
else
performMultipartUploadCopy();
- if (check_objects_after_upload)
+ if (request_settings.check_objects_after_upload)
checkObjectAfterUpload();
}
@@ -616,8 +619,9 @@ namespace
request.SetMetadataDirective(Aws::S3::Model::MetadataDirective::REPLACE);
}
- if (!settings.storage_class_name.empty())
- request.SetStorageClass(Aws::S3::Model::StorageClassMapper::GetStorageClassForName(settings.storage_class_name));
+ const auto & storage_class_name = upload_settings.storage_class_name;
+ if (!storage_class_name.empty())
+ request.SetStorageClass(Aws::S3::Model::StorageClassMapper::GetStorageClassForName(storage_class_name));
/// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840
request.SetContentType("binary/octet-stream");
@@ -625,7 +629,7 @@ namespace
void processCopyRequest(const Aws::S3::Model::CopyObjectRequest & request)
{
- size_t max_retries = std::max(max_unexpected_write_error_retries, 1UL);
+ size_t max_retries = std::max(request_settings.max_unexpected_write_error_retries, 1UL);
for (size_t retries = 1;; ++retries)
{
ProfileEvents::increment(ProfileEvents::S3CopyObject);
diff --git a/src/IO/S3/getObjectInfo.cpp b/src/IO/S3/getObjectInfo.cpp
new file mode 100644
index 00000000000..04871682296
--- /dev/null
+++ b/src/IO/S3/getObjectInfo.cpp
@@ -0,0 +1,218 @@
+#include
+
+#if USE_AWS_S3
+#include
+#include
+#include
+#include
+
+
+namespace ErrorCodes
+{
+ extern const int S3_ERROR;
+}
+
+
+namespace ProfileEvents
+{
+ extern const Event S3GetObject;
+ extern const Event S3GetObjectAttributes;
+ extern const Event S3HeadObject;
+ extern const Event DiskS3GetObject;
+ extern const Event DiskS3GetObjectAttributes;
+ extern const Event DiskS3HeadObject;
+}
+
+
+namespace DB::S3
+{
+
+namespace
+{
+ Aws::S3::Model::HeadObjectOutcome headObject(
+ const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3)
+ {
+ ProfileEvents::increment(ProfileEvents::S3HeadObject);
+ if (for_disk_s3)
+ ProfileEvents::increment(ProfileEvents::DiskS3HeadObject);
+
+ Aws::S3::Model::HeadObjectRequest req;
+ req.SetBucket(bucket);
+ req.SetKey(key);
+
+ if (!version_id.empty())
+ req.SetVersionId(version_id);
+
+ return client.HeadObject(req);
+ }
+
+ Aws::S3::Model::GetObjectAttributesOutcome getObjectAttributes(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3)
+ {
+ ProfileEvents::increment(ProfileEvents::S3GetObjectAttributes);
+ if (for_disk_s3)
+ ProfileEvents::increment(ProfileEvents::DiskS3GetObjectAttributes);
+
+ Aws::S3::Model::GetObjectAttributesRequest req;
+ req.SetBucket(bucket);
+ req.SetKey(key);
+
+ if (!version_id.empty())
+ req.SetVersionId(version_id);
+
+ req.SetObjectAttributes({Aws::S3::Model::ObjectAttributes::ObjectSize});
+
+ return client.GetObjectAttributes(req);
+ }
+
+ Aws::S3::Model::GetObjectOutcome getObjectDummy(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3)
+ {
+ ProfileEvents::increment(ProfileEvents::S3GetObject);
+ if (for_disk_s3)
+ ProfileEvents::increment(ProfileEvents::DiskS3GetObject);
+
+ Aws::S3::Model::GetObjectRequest req;
+ req.SetBucket(bucket);
+ req.SetKey(key);
+
+ if (!version_id.empty())
+ req.SetVersionId(version_id);
+
+ /// Only the first byte will be read.
+ /// We don't need that first byte but the range should be set otherwise the entire object will be read.
+ req.SetRange("bytes=0-0");
+
+ return client.GetObject(req);
+ }
+
+
+ /// Performs a request to get the size and last modification time of an object.
+ /// The function performs either HeadObject or GetObjectAttributes request depending on the endpoint.
+ std::pair, Aws::S3::S3Error> tryGetObjectInfo(
+ const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id,
+ const S3Settings::RequestSettings & request_settings, bool with_metadata, bool for_disk_s3)
+ {
+ if (request_settings.allow_head_object_request)
+ {
+ auto outcome = headObject(client, bucket, key, version_id, for_disk_s3);
+ if (!outcome.IsSuccess())
+ return {std::nullopt, outcome.GetError()};
+
+ const auto & result = outcome.GetResult();
+ ObjectInfo object_info;
+ object_info.size = static_cast(result.GetContentLength());
+ object_info.last_modification_time = result.GetLastModified().Millis() / 1000;
+
+ if (with_metadata)
+ object_info.metadata = result.GetMetadata();
+
+ return {object_info, {}};
+ }
+ else
+ {
+ ObjectInfo object_info;
+
+ {
+ auto outcome = getObjectAttributes(client, bucket, key, version_id, for_disk_s3);
+ if (!outcome.IsSuccess())
+ return {std::nullopt, outcome.GetError()};
+
+ const auto & result = outcome.GetResult();
+ object_info.size = static_cast(result.GetObjectSize());
+ object_info.last_modification_time = result.GetLastModified().Millis() / 1000;
+ }
+
+ if (with_metadata)
+ {
+ auto outcome = getObjectDummy(client, bucket, key, version_id, for_disk_s3);
+ if (!outcome.IsSuccess())
+ return {std::nullopt, outcome.GetError()};
+
+ const auto & result = outcome.GetResult();
+ object_info.metadata = result.GetMetadata();
+ }
+
+ return {object_info, {}};
+ }
+ }
+}
+
+
+bool isNotFoundError(Aws::S3::S3Errors error)
+{
+ return error == Aws::S3::S3Errors::RESOURCE_NOT_FOUND || error == Aws::S3::S3Errors::NO_SUCH_KEY;
+}
+
+ObjectInfo getObjectInfo(
+ const Aws::S3::S3Client & client,
+ const String & bucket,
+ const String & key,
+ const String & version_id,
+ const S3Settings::RequestSettings & request_settings,
+ bool with_metadata,
+ bool for_disk_s3,
+ bool throw_on_error)
+{
+ auto [object_info, error] = tryGetObjectInfo(client, bucket, key, version_id, request_settings, with_metadata, for_disk_s3);
+ if (object_info)
+ {
+ return *object_info;
+ }
+ else if (throw_on_error)
+ {
+ throw DB::Exception(ErrorCodes::S3_ERROR,
+ "Failed to get object attributes: {}. HTTP response code: {}",
+ error.GetMessage(), static_cast(error.GetResponseCode()));
+ }
+ return {};
+}
+
+size_t getObjectSize(
+ const Aws::S3::S3Client & client,
+ const String & bucket,
+ const String & key,
+ const String & version_id,
+ const S3Settings::RequestSettings & request_settings,
+ bool for_disk_s3,
+ bool throw_on_error)
+{
+ return getObjectInfo(client, bucket, key, version_id, request_settings, {}, for_disk_s3, throw_on_error).size;
+}
+
+bool objectExists(
+ const Aws::S3::S3Client & client,
+ const String & bucket,
+ const String & key,
+ const String & version_id,
+ const S3Settings::RequestSettings & request_settings,
+ bool for_disk_s3)
+{
+ auto [object_info, error] = tryGetObjectInfo(client, bucket, key, version_id, request_settings, {}, for_disk_s3);
+ if (object_info)
+ return true;
+
+ if (isNotFoundError(error.GetErrorType()))
+ return false;
+
+ throw S3Exception(error.GetErrorType(),
+ "Failed to check existence of key {} in bucket {}: {}",
+ key, bucket, error.GetMessage());
+}
+
+void checkObjectExists(
+ const Aws::S3::S3Client & client,
+ const String & bucket,
+ const String & key,
+ const String & version_id,
+ const S3Settings::RequestSettings & request_settings,
+ bool for_disk_s3,
+ std::string_view description)
+{
+ auto [object_info, error] = tryGetObjectInfo(client, bucket, key, version_id, request_settings, {}, for_disk_s3);
+ if (object_info)
+ return;
+ throw S3Exception(error.GetErrorType(), "{}Object {} in bucket {} suddenly disappeared: {}",
+ (description.empty() ? "" : (String(description) + ": ")), key, bucket, error.GetMessage());
+}
+}
+
+#endif
diff --git a/src/IO/S3/getObjectInfo.h b/src/IO/S3/getObjectInfo.h
new file mode 100644
index 00000000000..363e0601f32
--- /dev/null
+++ b/src/IO/S3/getObjectInfo.h
@@ -0,0 +1,63 @@
+#pragma once
+
+#include "config.h"
+
+#if USE_AWS_S3
+#include
+#include
+#include
+
+
+namespace DB::S3
+{
+
+struct ObjectInfo
+{
+ size_t size = 0;
+ time_t last_modification_time = 0;
+
+ std::map metadata; /// Set only if getObjectInfo() is called with `with_metadata = true`.
+};
+
+ObjectInfo getObjectInfo(
+ const Aws::S3::S3Client & client,
+ const String & bucket,
+ const String & key,
+ const String & version_id = {},
+ const S3Settings::RequestSettings & request_settings = {},
+ bool with_metadata = false,
+ bool for_disk_s3 = false,
+ bool throw_on_error = true);
+
+size_t getObjectSize(
+ const Aws::S3::S3Client & client,
+ const String & bucket,
+ const String & key,
+ const String & version_id = {},
+ const S3Settings::RequestSettings & request_settings = {},
+ bool for_disk_s3 = false,
+ bool throw_on_error = true);
+
+bool objectExists(
+ const Aws::S3::S3Client & client,
+ const String & bucket,
+ const String & key,
+ const String & version_id = {},
+ const S3Settings::RequestSettings & request_settings = {},
+ bool for_disk_s3 = false);
+
+/// Throws an exception if a specified object doesn't exist. `description` is used as a part of the error message.
+void checkObjectExists(
+ const Aws::S3::S3Client & client,
+ const String & bucket,
+ const String & key,
+ const String & version_id = {},
+ const S3Settings::RequestSettings & request_settings = {},
+ bool for_disk_s3 = false,
+ std::string_view description = {});
+
+bool isNotFoundError(Aws::S3::S3Errors error);
+
+}
+
+#endif
diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp
index bf1ce849f82..e590cbdcf31 100644
--- a/src/IO/S3Common.cpp
+++ b/src/IO/S3Common.cpp
@@ -705,92 +705,6 @@ public:
}
};
-/// Extracts the endpoint from a constructed S3 client.
-String getEndpoint(const Aws::S3::S3Client & client)
-{
- const auto * endpoint_provider = dynamic_cast(const_cast(client).accessEndpointProvider().get());
- if (!endpoint_provider)
- return {};
- String endpoint;
- endpoint_provider->GetBuiltInParameters().GetParameter("Endpoint").GetString(endpoint);
- return endpoint;
-}
-
-/// Performs a request to get the size and last modification time of an object.
-/// The function performs either HeadObject or GetObjectAttributes request depending on the endpoint.
-std::pair, Aws::S3::S3Error> tryGetObjectInfo(
- const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3)
-{
- auto endpoint = getEndpoint(client);
- bool use_get_object_attributes_request = (endpoint.find(".amazonaws.com") != String::npos);
-
- if (use_get_object_attributes_request)
- {
- /// It's better not to use `HeadObject` requests for AWS S3 because they don't work well with the global region.
- /// Details: `HeadObject` request never returns a response body (even if there is an error) however
- /// if the request was sent without specifying a region in the endpoint (i.e. for example "https://test.s3.amazonaws.com/mydata.csv"
- /// instead of "https://test.s3-us-west-2.amazonaws.com/mydata.csv") then that response body is one of the main ways
- /// to determine the correct region and try to repeat the request again with the correct region.
- /// For any other request type (`GetObject`, `ListObjects`, etc.) AWS SDK does that because they have response bodies,
- /// but for `HeadObject` there is no response body so this way doesn't work. That's why we use `GetObjectAttributes` request instead.
- /// See https://github.com/aws/aws-sdk-cpp/issues/1558 and also the function S3ErrorMarshaller::ExtractRegion() for more information.
-
- ProfileEvents::increment(ProfileEvents::S3GetObjectAttributes);
- if (for_disk_s3)
- ProfileEvents::increment(ProfileEvents::DiskS3GetObjectAttributes);
-
- Aws::S3::Model::GetObjectAttributesRequest req;
- req.SetBucket(bucket);
- req.SetKey(key);
-
- if (!version_id.empty())
- req.SetVersionId(version_id);
-
- req.SetObjectAttributes({Aws::S3::Model::ObjectAttributes::ObjectSize});
-
- auto outcome = client.GetObjectAttributes(req);
- if (outcome.IsSuccess())
- {
- const auto & result = outcome.GetResult();
- DB::S3::ObjectInfo object_info;
- object_info.size = static_cast(result.GetObjectSize());
- object_info.last_modification_time = result.GetLastModified().Millis() / 1000;
- return {object_info, {}};
- }
-
- return {std::nullopt, outcome.GetError()};
- }
- else
- {
- /// By default we use `HeadObject` requests.
- /// We cannot just use `GetObjectAttributes` requests always because some S3 providers (e.g. Minio)
- /// don't support `GetObjectAttributes` requests.
-
- ProfileEvents::increment(ProfileEvents::S3HeadObject);
- if (for_disk_s3)
- ProfileEvents::increment(ProfileEvents::DiskS3HeadObject);
-
- Aws::S3::Model::HeadObjectRequest req;
- req.SetBucket(bucket);
- req.SetKey(key);
-
- if (!version_id.empty())
- req.SetVersionId(version_id);
-
- auto outcome = client.HeadObject(req);
- if (outcome.IsSuccess())
- {
- const auto & result = outcome.GetResult();
- DB::S3::ObjectInfo object_info;
- object_info.size = static_cast(result.GetContentLength());
- object_info.last_modification_time = result.GetLastModified().Millis() / 1000;
- return {object_info, {}};
- }
-
- return {std::nullopt, outcome.GetError()};
- }
-}
-
}
@@ -982,88 +896,6 @@ namespace S3
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bucket name length is out of bounds in virtual hosted style S3 URI: {}{}",
quoteString(bucket), !uri.empty() ? " (" + uri.toString() + ")" : "");
}
-
- bool isNotFoundError(Aws::S3::S3Errors error)
- {
- return error == Aws::S3::S3Errors::RESOURCE_NOT_FOUND || error == Aws::S3::S3Errors::NO_SUCH_KEY;
- }
-
- ObjectInfo getObjectInfo(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3, bool throw_on_error)
- {
- auto [object_info, error] = tryGetObjectInfo(client, bucket, key, version_id, for_disk_s3);
- if (object_info)
- {
- return *object_info;
- }
- else if (throw_on_error)
- {
- throw DB::Exception(ErrorCodes::S3_ERROR,
- "Failed to get object attributes: {}. HTTP response code: {}",
- error.GetMessage(), static_cast(error.GetResponseCode()));
- }
- return {};
- }
-
- size_t getObjectSize(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3, bool throw_on_error)
- {
- return getObjectInfo(client, bucket, key, version_id, for_disk_s3, throw_on_error).size;
- }
-
- bool objectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3)
- {
- auto [object_info, error] = tryGetObjectInfo(client, bucket, key, version_id, for_disk_s3);
- if (object_info)
- return true;
-
- if (isNotFoundError(error.GetErrorType()))
- return false;
-
- throw S3Exception(error.GetErrorType(),
- "Failed to check existence of key {} in bucket {}: {}",
- key, bucket, error.GetMessage());
- }
-
- void checkObjectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3, std::string_view description)
- {
- auto [object_info, error] = tryGetObjectInfo(client, bucket, key, version_id, for_disk_s3);
- if (object_info)
- return;
- throw S3Exception(error.GetErrorType(), "{}Object {} in bucket {} suddenly disappeared: {}",
- (description.empty() ? "" : (String(description) + ": ")), key, bucket, error.GetMessage());
- }
-
- std::map getObjectMetadata(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id, bool for_disk_s3, bool throw_on_error)
- {
- ProfileEvents::increment(ProfileEvents::S3GetObjectMetadata);
- if (for_disk_s3)
- ProfileEvents::increment(ProfileEvents::DiskS3GetObjectMetadata);
-
- /// We must not use the `HeadObject` request, see the comment about `HeadObjectRequest` in S3Common.h.
-
- Aws::S3::Model::GetObjectRequest req;
- req.SetBucket(bucket);
- req.SetKey(key);
-
- /// Only the first byte will be read.
- /// We don't need that first byte but the range should be set otherwise the entire object will be read.
- req.SetRange("bytes=0-0");
-
- if (!version_id.empty())
- req.SetVersionId(version_id);
-
- auto outcome = client.GetObject(req);
-
- if (outcome.IsSuccess())
- return outcome.GetResult().GetMetadata();
-
- if (!throw_on_error)
- return {};
-
- const auto & error = outcome.GetError();
- throw S3Exception(error.GetErrorType(),
- "Failed to get metadata of key {} in bucket {}: {}",
- key, bucket, error.GetMessage());
- }
}
}
diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h
index 69ae1cbb4f4..19e660a338d 100644
--- a/src/IO/S3Common.h
+++ b/src/IO/S3Common.h
@@ -121,29 +121,6 @@ struct URI
static void validateBucket(const String & bucket, const Poco::URI & uri);
};
-/// WARNING: Don't use `HeadObjectRequest`! Use the functions below instead.
-/// For explanation see the comment about `HeadObject` request in the function tryGetObjectInfo().
-
-struct ObjectInfo
-{
- size_t size = 0;
- time_t last_modification_time = 0;
-};
-
-ObjectInfo getObjectInfo(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false, bool throw_on_error = true);
-
-size_t getObjectSize(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false, bool throw_on_error = true);
-
-bool objectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false);
-
-/// Throws an exception if a specified object doesn't exist. `description` is used as a part of the error message.
-void checkObjectExists(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false, std::string_view description = {});
-
-bool isNotFoundError(Aws::S3::S3Errors error);
-
-/// Returns the object's metadata.
-std::map getObjectMetadata(const Aws::S3::S3Client & client, const String & bucket, const String & key, const String & version_id = "", bool for_disk_s3 = false, bool throw_on_error = true);
-
}
#endif
diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp
index bc887b8ca78..c71a0e6a252 100644
--- a/src/IO/WriteBufferFromS3.cpp
+++ b/src/IO/WriteBufferFromS3.cpp
@@ -9,7 +9,7 @@
#include
#include
-#include
+#include
#include
#include
@@ -76,7 +76,7 @@ WriteBufferFromS3::WriteBufferFromS3(
std::shared_ptr client_ptr_,
const String & bucket_,
const String & key_,
- const S3Settings::RequestSettings & request_settings,
+ const S3Settings::RequestSettings & request_settings_,
std::optional> object_metadata_,
size_t buffer_size_,
ThreadPoolCallbackRunner schedule_,
@@ -84,12 +84,11 @@ WriteBufferFromS3::WriteBufferFromS3(
: BufferWithOwnMemory(buffer_size_, nullptr, 0)
, bucket(bucket_)
, key(key_)
- , settings(request_settings.getUploadSettings())
- , check_objects_after_upload(request_settings.check_objects_after_upload)
- , max_unexpected_write_error_retries(request_settings.max_unexpected_write_error_retries)
+ , request_settings(request_settings_)
+ , upload_settings(request_settings.getUploadSettings())
, client_ptr(std::move(client_ptr_))
, object_metadata(std::move(object_metadata_))
- , upload_part_size(settings.min_upload_part_size)
+ , upload_part_size(upload_settings.min_upload_part_size)
, schedule(std::move(schedule_))
, write_settings(write_settings_)
{
@@ -114,7 +113,7 @@ void WriteBufferFromS3::nextImpl()
write_settings.remote_throttler->add(offset(), ProfileEvents::RemoteWriteThrottlerBytes, ProfileEvents::RemoteWriteThrottlerSleepMicroseconds);
/// Data size exceeds singlepart upload threshold, need to use multipart upload.
- if (multipart_upload_id.empty() && last_part_size > settings.max_single_part_upload_size)
+ if (multipart_upload_id.empty() && last_part_size > upload_settings.max_single_part_upload_size)
createMultipartUpload();
chassert(upload_part_size > 0);
@@ -182,10 +181,10 @@ void WriteBufferFromS3::finalizeImpl()
if (!multipart_upload_id.empty())
completeMultipartUpload();
- if (check_objects_after_upload)
+ if (request_settings.check_objects_after_upload)
{
LOG_TRACE(log, "Checking object {} exists after upload", key);
- S3::checkObjectExists(*client_ptr, bucket, key, {}, /* for_disk_s3= */ write_settings.for_object_storage, "Immediately after upload");
+ S3::checkObjectExists(*client_ptr, bucket, key, {}, request_settings, /* for_disk_s3= */ write_settings.for_object_storage, "Immediately after upload");
LOG_TRACE(log, "Object {} exists after upload", key);
}
}
@@ -303,7 +302,10 @@ void WriteBufferFromS3::fillUploadRequest(Aws::S3::Model::UploadPartRequest & re
{
/// Increase part number.
++part_number;
- if (!multipart_upload_id.empty() && (part_number > settings.max_part_number))
+
+ auto max_part_number = upload_settings.max_part_number;
+
+ if (!multipart_upload_id.empty() && (part_number > max_part_number))
{
throw Exception(
ErrorCodes::INVALID_CONFIG_PARAMETER,
@@ -311,10 +313,11 @@ void WriteBufferFromS3::fillUploadRequest(Aws::S3::Model::UploadPartRequest & re
"Check min_upload_part_size = {}, max_upload_part_size = {}, "
"upload_part_size_multiply_factor = {}, upload_part_size_multiply_parts_count_threshold = {}, "
"max_single_part_upload_size = {}",
- settings.max_part_number, count(), settings.min_upload_part_size, settings.max_upload_part_size,
- settings.upload_part_size_multiply_factor,
- settings.upload_part_size_multiply_parts_count_threshold,
- settings.max_single_part_upload_size);
+ max_part_number, count(),
+ upload_settings.min_upload_part_size, upload_settings.max_upload_part_size,
+ upload_settings.upload_part_size_multiply_factor,
+ upload_settings.upload_part_size_multiply_parts_count_threshold,
+ upload_settings.max_single_part_upload_size);
}
/// Setup request.
@@ -329,10 +332,13 @@ void WriteBufferFromS3::fillUploadRequest(Aws::S3::Model::UploadPartRequest & re
req.SetContentType("binary/octet-stream");
/// Maybe increase `upload_part_size` (we need to increase it sometimes to keep `part_number` less or equal than `max_part_number`).
- if (!multipart_upload_id.empty() && (part_number % settings.upload_part_size_multiply_parts_count_threshold == 0))
+ auto threshold = upload_settings.upload_part_size_multiply_parts_count_threshold;
+ if (!multipart_upload_id.empty() && (part_number % threshold == 0))
{
- upload_part_size *= settings.upload_part_size_multiply_factor;
- upload_part_size = std::min(upload_part_size, settings.max_upload_part_size);
+ auto max_upload_part_size = upload_settings.max_upload_part_size;
+ auto upload_part_size_multiply_factor = upload_settings.upload_part_size_multiply_factor;
+ upload_part_size *= upload_part_size_multiply_factor;
+ upload_part_size = std::min(upload_part_size, max_upload_part_size);
}
}
@@ -377,7 +383,7 @@ void WriteBufferFromS3::completeMultipartUpload()
req.SetMultipartUpload(multipart_upload);
- size_t max_retry = std::max(max_unexpected_write_error_retries, 1UL);
+ size_t max_retry = std::max(request_settings.max_unexpected_write_error_retries, 1UL);
for (size_t i = 0; i < max_retry; ++i)
{
ProfileEvents::increment(ProfileEvents::S3CompleteMultipartUpload);
@@ -476,8 +482,8 @@ void WriteBufferFromS3::fillPutRequest(Aws::S3::Model::PutObjectRequest & req)
req.SetBody(temporary_buffer);
if (object_metadata.has_value())
req.SetMetadata(object_metadata.value());
- if (!settings.storage_class_name.empty())
- req.SetStorageClass(Aws::S3::Model::StorageClassMapper::GetStorageClassForName(settings.storage_class_name));
+ if (!upload_settings.storage_class_name.empty())
+ req.SetStorageClass(Aws::S3::Model::StorageClassMapper::GetStorageClassForName(upload_settings.storage_class_name));
/// If we don't do it, AWS SDK can mistakenly set it to application/xml, see https://github.com/aws/aws-sdk-cpp/issues/1840
req.SetContentType("binary/octet-stream");
@@ -485,7 +491,7 @@ void WriteBufferFromS3::fillPutRequest(Aws::S3::Model::PutObjectRequest & req)
void WriteBufferFromS3::processPutRequest(const PutObjectTask & task)
{
- size_t max_retry = std::max(max_unexpected_write_error_retries, 1UL);
+ size_t max_retry = std::max(request_settings.max_unexpected_write_error_retries, 1UL);
for (size_t i = 0; i < max_retry; ++i)
{
ProfileEvents::increment(ProfileEvents::S3PutObject);
diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h
index 41ed009bcf9..8c81c49d32c 100644
--- a/src/IO/WriteBufferFromS3.h
+++ b/src/IO/WriteBufferFromS3.h
@@ -50,7 +50,7 @@ public:
std::shared_ptr client_ptr_,
const String & bucket_,
const String & key_,
- const S3Settings::RequestSettings & request_settings,
+ const S3Settings::RequestSettings & request_settings_,
std::optional> object_metadata_ = std::nullopt,
size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE,
ThreadPoolCallbackRunner schedule_ = {},
@@ -88,9 +88,8 @@ private:
const String bucket;
const String key;
- const S3Settings::RequestSettings::PartUploadSettings settings;
- const bool check_objects_after_upload = false;
- const size_t max_unexpected_write_error_retries = 4;
+ const S3Settings::RequestSettings request_settings;
+ const S3Settings::RequestSettings::PartUploadSettings & upload_settings;
const std::shared_ptr client_ptr;
const std::optional> object_metadata;
diff --git a/src/IO/ZstdDeflatingAppendableWriteBuffer.cpp b/src/IO/ZstdDeflatingAppendableWriteBuffer.cpp
index c99f8e50baf..f8c4d0e2bac 100644
--- a/src/IO/ZstdDeflatingAppendableWriteBuffer.cpp
+++ b/src/IO/ZstdDeflatingAppendableWriteBuffer.cpp
@@ -142,8 +142,17 @@ void ZstdDeflatingAppendableWriteBuffer::finalizeBefore()
ZSTD_getErrorName(remaining), ZSTD_VERSION_STRING);
remaining = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end);
+
+ out->position() = out->buffer().begin() + output.pos;
+
+ if (!out->hasPendingData())
+ {
+ out->next();
+ output.dst = reinterpret_cast(out->buffer().begin());
+ output.size = out->buffer().size();
+ output.pos = out->offset();
+ }
}
- out->position() = out->buffer().begin() + output.pos;
}
void ZstdDeflatingAppendableWriteBuffer::finalizeAfter()
diff --git a/src/IO/ZstdInflatingReadBuffer.cpp b/src/IO/ZstdInflatingReadBuffer.cpp
index ff754025440..6956bdb6710 100644
--- a/src/IO/ZstdInflatingReadBuffer.cpp
+++ b/src/IO/ZstdInflatingReadBuffer.cpp
@@ -58,9 +58,9 @@ bool ZstdInflatingReadBuffer::nextImpl()
size_t ret = ZSTD_decompressStream(dctx, &output, &input);
if (ZSTD_isError(ret))
throw Exception(
- ErrorCodes::ZSTD_DECODER_FAILED,
- "Zstd stream encoding failed: error '{}'; zstd version: {}",
- ZSTD_getErrorName(ret), ZSTD_VERSION_STRING);
+ ErrorCodes::ZSTD_DECODER_FAILED,
+ "Zstd stream encoding failed: error '{}'; zstd version: {}",
+ ZSTD_getErrorName(ret), ZSTD_VERSION_STRING);
/// Check that something has changed after decompress (input or output position)
assert(in->eof() || output.pos > 0 || in->position() < in->buffer().begin() + input.pos);
diff --git a/src/Interpreters/Cache/QueryResultCache.cpp b/src/Interpreters/Cache/QueryCache.cpp
similarity index 69%
rename from src/Interpreters/Cache/QueryResultCache.cpp
rename to src/Interpreters/Cache/QueryCache.cpp
index 894b2b77d6c..06fd0689993 100644
--- a/src/Interpreters/Cache/QueryResultCache.cpp
+++ b/src/Interpreters/Cache/QueryCache.cpp
@@ -1,4 +1,4 @@
-#include "Interpreters/Cache/QueryResultCache.h"
+#include "Interpreters/Cache/QueryCache.h"
#include
#include
@@ -16,8 +16,8 @@
namespace ProfileEvents
{
- extern const Event QueryResultCacheHits;
- extern const Event QueryResultCacheMisses;
+ extern const Event QueryCacheHits;
+ extern const Event QueryCacheMisses;
};
namespace DB
@@ -64,7 +64,7 @@ bool astContainsNonDeterministicFunctions(ASTPtr ast, ContextPtr context)
namespace
{
-class RemoveQueryResultCacheSettingsMatcher
+class RemoveQueryCacheSettingsMatcher
{
public:
struct Data {};
@@ -77,63 +77,65 @@ public:
{
chassert(!set_clause->is_standalone);
- auto is_query_result_cache_related_setting = [](const auto & change)
+ auto is_query_cache_related_setting = [](const auto & change)
{
- return change.name.starts_with("enable_experimental_query_result_cache")
- || change.name.starts_with("query_result_cache");
+ return change.name == "allow_experimental_query_cache"
+ || change.name.starts_with("query_cache")
+ || change.name.ends_with("query_cache");
};
- std::erase_if(set_clause->changes, is_query_result_cache_related_setting);
+ std::erase_if(set_clause->changes, is_query_cache_related_setting);
}
}
/// TODO further improve AST cleanup, e.g. remove SETTINGS clause completely if it is empty
- /// E.g. SELECT 1 SETTINGS enable_experimental_query_result_cache = true
+ /// E.g. SELECT 1 SETTINGS use_query_cache = true
/// and SELECT 1;
/// currently don't match.
};
-using RemoveQueryResultCacheSettingsVisitor = InDepthNodeVisitor;
+using RemoveQueryCacheSettingsVisitor = InDepthNodeVisitor;
/// Consider
-/// (1) SET enable_experimental_query_result_cache = true;
-/// SELECT expensiveComputation(...) SETTINGS max_threads = 64, query_result_cache_ttl = 300;
-/// SET enable_experimental_query_result_cache = false;
+/// (1) SET use_query_cache = true;
+/// SELECT expensiveComputation(...) SETTINGS max_threads = 64, query_cache_ttl = 300;
+/// SET use_query_cache = false;
/// and
-/// (2) SELECT expensiveComputation(...) SETTINGS max_threads = 64, enable_experimental_query_result_cache_passive_usage = true;
-/// The SELECT queries in (1) and (2) are basically the same and the user expects that the second invocation is served from the query result
+/// (2) SELECT expensiveComputation(...) SETTINGS max_threads = 64, use_query_cache = true;
+///
+/// The SELECT queries in (1) and (2) are basically the same and the user expects that the second invocation is served from the query
/// cache. However, query results are indexed by their query ASTs and therefore no result will be found. Insert and retrieval behave overall
-/// more natural if settings related to the query result cache are erased from the AST key. Note that at this point the settings themselves
+/// more natural if settings related to the query cache are erased from the AST key. Note that at this point the settings themselves
/// have been parsed already, they are not lost or discarded.
-ASTPtr removeQueryResultCacheSettings(ASTPtr ast)
+ASTPtr removeQueryCacheSettings(ASTPtr ast)
{
ASTPtr transformed_ast = ast->clone();
- RemoveQueryResultCacheSettingsMatcher::Data visitor_data;
- RemoveQueryResultCacheSettingsVisitor(visitor_data).visit(transformed_ast);
+ RemoveQueryCacheSettingsMatcher::Data visitor_data;
+ RemoveQueryCacheSettingsVisitor(visitor_data).visit(transformed_ast);
return transformed_ast;
}
}
-QueryResultCache::Key::Key(
+QueryCache::Key::Key(
ASTPtr ast_,
Block header_, const std::optional & username_,
std::chrono::time_point expires_at_)
- : ast(removeQueryResultCacheSettings(ast_))
+ : ast(removeQueryCacheSettings(ast_))
, header(header_)
, username(username_)
, expires_at(expires_at_)
{
}
-bool QueryResultCache::Key::operator==(const Key & other) const
+bool QueryCache::Key::operator==(const Key & other) const
{
return ast->getTreeHash() == other.ast->getTreeHash();
}
-String QueryResultCache::Key::queryStringFromAst() const
+String QueryCache::Key::queryStringFromAst() const
{
WriteBufferFromOwnString buf;
IAST::FormatSettings format_settings(buf, /*one_line*/ true);
@@ -142,7 +144,7 @@ String QueryResultCache::Key::queryStringFromAst() const
return buf.str();
}
-size_t QueryResultCache::KeyHasher::operator()(const Key & key) const
+size_t QueryCache::KeyHasher::operator()(const Key & key) const
{
SipHash hash;
hash.update(key.ast->getTreeHash());
@@ -150,7 +152,7 @@ size_t QueryResultCache::KeyHasher::operator()(const Key & key) const
return res;
}
-size_t QueryResultCache::QueryResult::sizeInBytes() const
+size_t QueryCache::QueryResult::sizeInBytes() const
{
size_t res = 0;
for (const auto & chunk : *chunks)
@@ -161,14 +163,14 @@ size_t QueryResultCache::QueryResult::sizeInBytes() const
namespace
{
-auto is_stale = [](const QueryResultCache::Key & key)
+auto is_stale = [](const QueryCache::Key & key)
{
return (key.expires_at < std::chrono::system_clock::now());
};
}
-QueryResultCache::Writer::Writer(std::mutex & mutex_, Cache & cache_, const Key & key_,
+QueryCache::Writer::Writer(std::mutex & mutex_, Cache & cache_, const Key & key_,
size_t & cache_size_in_bytes_, size_t max_cache_size_in_bytes_,
size_t max_cache_entries_,
size_t max_entry_size_in_bytes_, size_t max_entry_size_in_rows_,
@@ -187,7 +189,7 @@ QueryResultCache::Writer::Writer(std::mutex & mutex_, Cache & cache_, const Key
skip_insert = true; /// Key already contained in cache and did not expire yet --> don't replace it
}
-void QueryResultCache::Writer::buffer(Chunk && partial_query_result)
+void QueryCache::Writer::buffer(Chunk && partial_query_result)
{
if (skip_insert)
return;
@@ -206,7 +208,7 @@ void QueryResultCache::Writer::buffer(Chunk && partial_query_result)
}
}
-void QueryResultCache::Writer::finalizeWrite()
+void QueryCache::Writer::finalizeWrite()
{
if (skip_insert)
return;
@@ -237,7 +239,7 @@ void QueryResultCache::Writer::finalizeWrite()
}
else
++it;
- LOG_TRACE(&Poco::Logger::get("QueryResultCache"), "Removed {} stale entries", removed_items);
+ LOG_TRACE(&Poco::Logger::get("QueryCache"), "Removed {} stale entries", removed_items);
}
/// Insert or replace if enough space
@@ -248,23 +250,23 @@ void QueryResultCache::Writer::finalizeWrite()
cache_size_in_bytes -= it->second.sizeInBytes(); // key replacement
cache[key] = std::move(query_result);
- LOG_TRACE(&Poco::Logger::get("QueryResultCache"), "Stored result of query {}", key.queryStringFromAst());
+ LOG_TRACE(&Poco::Logger::get("QueryCache"), "Stored result of query {}", key.queryStringFromAst());
}
}
-QueryResultCache::Reader::Reader(const Cache & cache_, const Key & key, size_t & cache_size_in_bytes_, const std::lock_guard &)
+QueryCache::Reader::Reader(const Cache & cache_, const Key & key, size_t & cache_size_in_bytes_, const std::lock_guard &)
{
auto it = cache_.find(key);
if (it == cache_.end())
{
- LOG_TRACE(&Poco::Logger::get("QueryResultCache"), "No entry found for query {}", key.queryStringFromAst());
+ LOG_TRACE(&Poco::Logger::get("QueryCache"), "No entry found for query {}", key.queryStringFromAst());
return;
}
if (it->first.username.has_value() && it->first.username != key.username)
{
- LOG_TRACE(&Poco::Logger::get("QueryResultCache"), "Inaccessible entry found for query {}", key.queryStringFromAst());
+ LOG_TRACE(&Poco::Logger::get("QueryCache"), "Inaccessible entry found for query {}", key.queryStringFromAst());
return;
}
@@ -272,33 +274,33 @@ QueryResultCache::Reader::Reader(const Cache & cache_, const Key & key, size_t &
{
cache_size_in_bytes_ -= it->second.sizeInBytes();
const_cast(cache_).erase(it);
- LOG_TRACE(&Poco::Logger::get("QueryResultCache"), "Stale entry found and removed for query {}", key.queryStringFromAst());
+ LOG_TRACE(&Poco::Logger::get("QueryCache"), "Stale entry found and removed for query {}", key.queryStringFromAst());
return;
}
pipe = Pipe(std::make_shared(it->first.header, it->second.chunks));
- LOG_TRACE(&Poco::Logger::get("QueryResultCache"), "Entry found for query {}", key.queryStringFromAst());
+ LOG_TRACE(&Poco::Logger::get("QueryCache"), "Entry found for query {}", key.queryStringFromAst());
}
-bool QueryResultCache::Reader::hasCacheEntryForKey() const
+bool QueryCache::Reader::hasCacheEntryForKey() const
{
bool res = !pipe.empty();
if (res)
- ProfileEvents::increment(ProfileEvents::QueryResultCacheHits);
+ ProfileEvents::increment(ProfileEvents::QueryCacheHits);
else
- ProfileEvents::increment(ProfileEvents::QueryResultCacheMisses);
+ ProfileEvents::increment(ProfileEvents::QueryCacheMisses);
return res;
}
-Pipe && QueryResultCache::Reader::getPipe()
+Pipe && QueryCache::Reader::getPipe()
{
chassert(!pipe.empty()); // cf. hasCacheEntryForKey()
return std::move(pipe);
}
-QueryResultCache::QueryResultCache(size_t max_cache_size_in_bytes_, size_t max_cache_entries_, size_t max_cache_entry_size_in_bytes_, size_t max_cache_entry_size_in_rows_)
+QueryCache::QueryCache(size_t max_cache_size_in_bytes_, size_t max_cache_entries_, size_t max_cache_entry_size_in_bytes_, size_t max_cache_entry_size_in_rows_)
: max_cache_size_in_bytes(max_cache_size_in_bytes_)
, max_cache_entries(max_cache_entries_)
, max_cache_entry_size_in_bytes(max_cache_entry_size_in_bytes_)
@@ -306,19 +308,19 @@ QueryResultCache::QueryResultCache(size_t max_cache_size_in_bytes_, size_t max_c
{
}
-QueryResultCache::Reader QueryResultCache::createReader(const Key & key)
+QueryCache::Reader QueryCache::createReader(const Key & key)
{
std::lock_guard lock(mutex);
return Reader(cache, key, cache_size_in_bytes, lock);
}
-QueryResultCache::Writer QueryResultCache::createWriter(const Key & key, std::chrono::milliseconds min_query_runtime)
+QueryCache::Writer QueryCache::createWriter(const Key & key, std::chrono::milliseconds min_query_runtime)
{
std::lock_guard lock(mutex);
return Writer(mutex, cache, key, cache_size_in_bytes, max_cache_size_in_bytes, max_cache_entries, max_cache_entry_size_in_bytes, max_cache_entry_size_in_rows, min_query_runtime);
}
-void QueryResultCache::reset()
+void QueryCache::reset()
{
std::lock_guard lock(mutex);
cache.clear();
@@ -326,7 +328,7 @@ void QueryResultCache::reset()
cache_size_in_bytes = 0;
}
-size_t QueryResultCache::recordQueryRun(const Key & key)
+size_t QueryCache::recordQueryRun(const Key & key)
{
static constexpr size_t TIMES_EXECUTED_MAX_SIZE = 10'000;
diff --git a/src/Interpreters/Cache/QueryResultCache.h b/src/Interpreters/Cache/QueryCache.h
similarity index 89%
rename from src/Interpreters/Cache/QueryResultCache.h
rename to src/Interpreters/Cache/QueryCache.h
index 65cab854a45..45f48c7a558 100644
--- a/src/Interpreters/Cache/QueryResultCache.h
+++ b/src/Interpreters/Cache/QueryCache.h
@@ -18,7 +18,7 @@ bool astContainsNonDeterministicFunctions(ASTPtr ast, ContextPtr context);
/// returned. In order to still obtain sufficiently up-to-date query results, a expiry time (TTL) must be specified for each cache entry
/// after which it becomes stale and is ignored. Stale entries are removed opportunistically from the cache, they are only evicted when a
/// new entry is inserted and the cache has insufficient capacity.
-class QueryResultCache
+class QueryCache
{
public:
/// Represents a query result in the cache.
@@ -82,9 +82,9 @@ public:
/// Buffers multiple partial query result chunks (buffer()) and eventually stores them as cache entry (finalizeWrite()).
///
/// Implementation note: Queries may throw exceptions during runtime, e.g. out-of-memory errors. In this case, no query result must be
- /// written into the query result cache. Unfortunately, neither the Writer nor the special transform added on top of the query pipeline
- /// which holds the Writer know whether they are destroyed because the query ended successfully or because of an exception (otherwise,
- /// we could simply implement a check in their destructors). To handle exceptions correctly nevertheless, we do the actual insert in
+ /// written into the query cache. Unfortunately, neither the Writer nor the special transform added on top of the query pipeline which
+ /// holds the Writer know whether they are destroyed because the query ended successfully or because of an exception (otherwise, we
+ /// could simply implement a check in their destructors). To handle exceptions correctly nevertheless, we do the actual insert in
/// finalizeWrite() as opposed to the Writer destructor. This function is then called only for successful queries in finish_callback()
/// which runs before the transform and the Writer are destroyed, whereas for unsuccessful queries we do nothing (the Writer is
/// destroyed w/o inserting anything).
@@ -117,7 +117,7 @@ public:
size_t max_entry_size_in_bytes_, size_t max_entry_size_in_rows_,
std::chrono::milliseconds min_query_runtime_);
- friend class QueryResultCache; /// for createWriter()
+ friend class QueryCache; /// for createWriter()
};
/// Looks up a query result for a key in the cache and (if found) constructs a pipe with the query result chunks as source.
@@ -129,10 +129,10 @@ public:
private:
Reader(const Cache & cache_, const Key & key, size_t & cache_size_in_bytes_, const std::lock_guard &);
Pipe pipe;
- friend class QueryResultCache; /// for createReader()
+ friend class QueryCache; /// for createReader()
};
- QueryResultCache(size_t max_cache_size_in_bytes_, size_t max_cache_entries_, size_t max_cache_entry_size_in_bytes_, size_t max_cache_entry_size_in_rows_);
+ QueryCache(size_t max_cache_size_in_bytes_, size_t max_cache_entries_, size_t max_cache_entry_size_in_bytes_, size_t max_cache_entry_size_in_rows_);
Reader createReader(const Key & key);
Writer createWriter(const Key & key, std::chrono::milliseconds min_query_runtime);
@@ -160,9 +160,9 @@ private:
const size_t max_cache_entry_size_in_bytes;
const size_t max_cache_entry_size_in_rows;
- friend class StorageSystemQueryResultCache;
+ friend class StorageSystemQueryCache;
};
-using QueryResultCachePtr = std::shared_ptr;
+using QueryCachePtr = std::shared_ptr;
}
diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
index e228dcc1f4a..5c781c531ed 100644
--- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
+++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
@@ -7,7 +7,6 @@
#include
#include
#include
-#include
#include
#include
diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp
index 2a8734596c5..d9a7aa2e677 100644
--- a/src/Interpreters/Context.cpp
+++ b/src/Interpreters/Context.cpp
@@ -40,7 +40,7 @@
#include
#include
#include
-#include
+#include
#include
#include
#include
@@ -236,7 +236,7 @@ struct ContextSharedPart : boost::noncopyable
mutable std::unique_ptr load_marks_threadpool; /// Threadpool for loading marks cache.
mutable UncompressedCachePtr index_uncompressed_cache; /// The cache of decompressed blocks for MergeTree indices.
mutable MarkCachePtr index_mark_cache; /// Cache of marks in compressed files of MergeTree indices.
- mutable QueryResultCachePtr query_result_cache; /// Cache of query results.
+ mutable QueryCachePtr query_cache; /// Cache of query results.
mutable MMappedFileCachePtr mmap_cache; /// Cache of mmapped files to avoid frequent open/map/unmap/close and to reuse from several threads.
ProcessList process_list; /// Executing queries at the moment.
GlobalOvercommitTracker global_overcommit_tracker;
@@ -2041,27 +2041,27 @@ void Context::dropIndexMarkCache() const
shared->index_mark_cache->reset();
}
-void Context::setQueryResultCache(size_t max_size_in_bytes, size_t max_entries, size_t max_entry_size_in_bytes, size_t max_entry_size_in_records)
+void Context::setQueryCache(size_t max_size_in_bytes, size_t max_entries, size_t max_entry_size_in_bytes, size_t max_entry_size_in_records)
{
auto lock = getLock();
- if (shared->query_result_cache)
- throw Exception(ErrorCodes::LOGICAL_ERROR, "Query result cache has been already created.");
+ if (shared->query_cache)
+ throw Exception(ErrorCodes::LOGICAL_ERROR, "Query cache has been already created.");
- shared->query_result_cache = std::make_shared(max_size_in_bytes, max_entries, max_entry_size_in_bytes, max_entry_size_in_records);
+ shared->query_cache = std::make_shared(max_size_in_bytes, max_entries, max_entry_size_in_bytes, max_entry_size_in_records);
}
-QueryResultCachePtr Context::getQueryResultCache() const
+QueryCachePtr Context::getQueryCache() const
{
auto lock = getLock();
- return shared->query_result_cache;
+ return shared->query_cache;
}
-void Context::dropQueryResultCache() const
+void Context::dropQueryCache() const
{
auto lock = getLock();
- if (shared->query_result_cache)
- shared->query_result_cache->reset();
+ if (shared->query_cache)
+ shared->query_cache->reset();
}
void Context::setMMappedFileCache(size_t cache_size_in_num_entries)
@@ -2104,8 +2104,8 @@ void Context::dropCaches() const
if (shared->index_mark_cache)
shared->index_mark_cache->reset();
- if (shared->query_result_cache)
- shared->query_result_cache->reset();
+ if (shared->query_cache)
+ shared->query_cache->reset();
if (shared->mmap_cache)
shared->mmap_cache->reset();
diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h
index 10983b15d7f..00dc4204496 100644
--- a/src/Interpreters/Context.h
+++ b/src/Interpreters/Context.h
@@ -81,8 +81,8 @@ class Macros;
struct Progress;
struct FileProgress;
class Clusters;
+class QueryCache;
class QueryLog;
-class QueryResultCache;
class QueryThreadLog;
class QueryViewsLog;
class PartLog;
@@ -861,9 +861,9 @@ public:
void dropMMappedFileCache() const;
/// Create a cache of query results for statements which run repeatedly.
- void setQueryResultCache(size_t max_size_in_bytes, size_t max_entries, size_t max_entry_size_in_bytes, size_t max_entry_size_in_records);
- std::shared_ptr getQueryResultCache() const;
- void dropQueryResultCache() const;
+ void setQueryCache(size_t max_size_in_bytes, size_t max_entries, size_t max_entry_size_in_bytes, size_t max_entry_size_in_records);
+ std::shared_ptr getQueryCache() const;
+ void dropQueryCache() const;
/** Clear the caches of the uncompressed blocks and marks.
* This is usually done when renaming tables, changing the type of columns, deleting a table.
diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp
index ee5aad3d18e..a87308f0cef 100644
--- a/src/Interpreters/InterpreterAlterQuery.cpp
+++ b/src/Interpreters/InterpreterAlterQuery.cpp
@@ -147,7 +147,7 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter)
{
table->checkMutationIsPossible(mutation_commands, getContext()->getSettingsRef());
MutationsInterpreter(table, metadata_snapshot, mutation_commands, getContext(), false).validate();
- table->mutate(mutation_commands, getContext(), false);
+ table->mutate(mutation_commands, getContext());
}
if (!partition_commands.empty())
diff --git a/src/Interpreters/InterpreterDeleteQuery.cpp b/src/Interpreters/InterpreterDeleteQuery.cpp
index abccc313e14..f8974a19f45 100644
--- a/src/Interpreters/InterpreterDeleteQuery.cpp
+++ b/src/Interpreters/InterpreterDeleteQuery.cpp
@@ -5,15 +5,16 @@
#include
#include
#include
+#include
#include
+#include
+#include
+#include
#include
-#include
-#include
#include
#include
#include
#include
-#include
namespace DB
@@ -72,7 +73,7 @@ BlockIO InterpreterDeleteQuery::execute()
table->checkMutationIsPossible(mutation_commands, getContext()->getSettingsRef());
MutationsInterpreter(table, metadata_snapshot, mutation_commands, getContext(), false).validate();
- table->mutate(mutation_commands, getContext(), false);
+ table->mutate(mutation_commands, getContext());
return {};
}
else if (table->supportsLightweightDelete())
@@ -82,35 +83,25 @@ BlockIO InterpreterDeleteQuery::execute()
"Lightweight delete mutate is experimental. "
"Set `allow_experimental_lightweight_delete` setting to enable it");
- /// Convert to MutationCommand
- MutationCommands mutation_commands;
- MutationCommand mut_command;
+ /// Build "ALTER ... UPDATE _row_exists = 0 WHERE predicate" query
+ String alter_query =
+ "ALTER TABLE " + table->getStorageID().getFullTableName()
+ + (delete_query.cluster.empty() ? "" : " ON CLUSTER " + backQuoteIfNeed(delete_query.cluster))
+ + " UPDATE `_row_exists` = 0 WHERE " + serializeAST(*delete_query.predicate);
- /// Build "UPDATE _row_exists = 0 WHERE predicate" query
- mut_command.type = MutationCommand::Type::UPDATE;
- mut_command.predicate = delete_query.predicate;
+ ParserAlterQuery parser;
+ ASTPtr alter_ast = parseQuery(
+ parser,
+ alter_query.data(),
+ alter_query.data() + alter_query.size(),
+ "ALTER query",
+ 0,
+ DBMS_DEFAULT_MAX_PARSER_DEPTH);
- auto command = std::make_shared();
- command->type = ASTAlterCommand::UPDATE;
- command->predicate = delete_query.predicate;
- command->update_assignments = std::make_shared();
- auto set_row_does_not_exist = std::make_shared();
- set_row_does_not_exist->column_name = LightweightDeleteDescription::FILTER_COLUMN.name;
- auto zero_value = std::make_shared(DB::Field(UInt8(0)));
- set_row_does_not_exist->children.push_back(zero_value);
- command->update_assignments->children.push_back(set_row_does_not_exist);
- command->children.push_back(command->predicate);
- command->children.push_back(command->update_assignments);
- mut_command.column_to_update_expression[set_row_does_not_exist->column_name] = zero_value;
- mut_command.ast = command->ptr();
-
- mutation_commands.emplace_back(mut_command);
-
- table->checkMutationIsPossible(mutation_commands, getContext()->getSettingsRef());
- MutationsInterpreter(table, metadata_snapshot, mutation_commands, getContext(), false).validate();
- table->mutate(mutation_commands, getContext(), true);
-
- return {};
+ auto context = Context::createCopy(getContext());
+ context->setSetting("mutations_sync", 2); /// Lightweight delete is always synchronous
+ InterpreterAlterQuery alter_interpreter(alter_ast, context);
+ return alter_interpreter.execute();
}
else
{
diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp
index efcdde46e49..502de459156 100644
--- a/src/Interpreters/InterpreterFactory.cpp
+++ b/src/Interpreters/InterpreterFactory.cpp
@@ -17,6 +17,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -73,6 +74,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -167,6 +169,10 @@ std::unique_ptr InterpreterFactory::get(ASTPtr & query, ContextMut
{
return std::make_unique(query, context);
}
+ else if (query->as())
+ {
+ return std::make_unique(query, context);
+ }
else if (query->as())
{
return std::make_unique(query, context);
diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp
index 3b90ac8d284..74fe351daaf 100644
--- a/src/Interpreters/InterpreterInsertQuery.cpp
+++ b/src/Interpreters/InterpreterInsertQuery.cpp
@@ -6,7 +6,6 @@
#include
#include
#include
-#include
#include
#include
#include
diff --git a/src/Interpreters/InterpreterShowEngineQuery.cpp b/src/Interpreters/InterpreterShowEngineQuery.cpp
new file mode 100644
index 00000000000..5aae6ad5d28
--- /dev/null
+++ b/src/Interpreters/InterpreterShowEngineQuery.cpp
@@ -0,0 +1,18 @@
+#include
+
+#include
+#include
+#include
+
+#include
+
+
+namespace DB
+{
+
+BlockIO InterpreterShowEnginesQuery::execute()
+{
+ return executeQuery("SELECT * FROM system.table_engines", getContext(), true);
+}
+
+}
diff --git a/src/Interpreters/InterpreterShowEngineQuery.h b/src/Interpreters/InterpreterShowEngineQuery.h
new file mode 100644
index 00000000000..3c451e9b071
--- /dev/null
+++ b/src/Interpreters/InterpreterShowEngineQuery.h
@@ -0,0 +1,29 @@
+#pragma once
+
+#include
+#include
+
+
+namespace DB
+{
+
+/** Return list of all engines
+ */
+class InterpreterShowEnginesQuery : public IInterpreter, WithMutableContext
+{
+public:
+ InterpreterShowEnginesQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_)
+ : WithMutableContext(context_), query_ptr(query_ptr_) {}
+
+ BlockIO execute() override;
+
+ /// We ignore the quota and limits here because execute() will rewrite a show query as a SELECT query and then
+ /// the SELECT query will check the quota and limits.
+ bool ignoreQuota() const override { return true; }
+ bool ignoreLimits() const override { return true; }
+
+private:
+ ASTPtr query_ptr;
+};
+
+}
diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp
index 5fa5e20be18..abd0ecd6ea1 100644
--- a/src/Interpreters/InterpreterSystemQuery.cpp
+++ b/src/Interpreters/InterpreterSystemQuery.cpp
@@ -56,6 +56,7 @@
#include
#include
#include
+#include
#include "config.h"
@@ -292,6 +293,12 @@ BlockIO InterpreterSystemQuery::execute()
res->wait();
break;
}
+ case Type::SYNC_FILE_CACHE:
+ {
+ LOG_DEBUG(log, "Will perform 'sync' syscall (it can take time).");
+ sync();
+ break;
+ }
case Type::DROP_DNS_CACHE:
{
getContext()->checkAccess(AccessType::SYSTEM_DROP_DNS_CACHE);
@@ -320,9 +327,9 @@ BlockIO InterpreterSystemQuery::execute()
getContext()->checkAccess(AccessType::SYSTEM_DROP_MMAP_CACHE);
system_context->dropMMappedFileCache();
break;
- case Type::DROP_QUERY_RESULT_CACHE:
- getContext()->checkAccess(AccessType::SYSTEM_DROP_QUERY_RESULT_CACHE);
- getContext()->dropQueryResultCache();
+ case Type::DROP_QUERY_CACHE:
+ getContext()->checkAccess(AccessType::SYSTEM_DROP_QUERY_CACHE);
+ getContext()->dropQueryCache();
break;
#if USE_EMBEDDED_COMPILER
case Type::DROP_COMPILED_EXPRESSION_CACHE:
@@ -962,7 +969,7 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster()
case Type::DROP_DNS_CACHE:
case Type::DROP_MARK_CACHE:
case Type::DROP_MMAP_CACHE:
- case Type::DROP_QUERY_RESULT_CACHE:
+ case Type::DROP_QUERY_CACHE:
#if USE_EMBEDDED_COMPILER
case Type::DROP_COMPILED_EXPRESSION_CACHE:
#endif
@@ -1133,6 +1140,11 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster()
required_access.emplace_back(AccessType::SYSTEM_UNFREEZE);
break;
}
+ case Type::SYNC_FILE_CACHE:
+ {
+ required_access.emplace_back(AccessType::SYSTEM_SYNC_FILE_CACHE);
+ break;
+ }
case Type::STOP_LISTEN_QUERIES:
case Type::START_LISTEN_QUERIES:
case Type::STOP_THREAD_FUZZER:
diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp
index 5964c816514..f46adf91ee0 100644
--- a/src/Interpreters/executeQuery.cpp
+++ b/src/Interpreters/executeQuery.cpp
@@ -14,7 +14,7 @@
#include
#include
#include
-#include
+#include
#include
#include
@@ -714,43 +714,50 @@ static std::tuple executeQueryImpl(
res = interpreter->execute();
- /// Try to read (SELECT) query result from query result cache (if it is enabled)
- auto query_result_cache = context->getQueryResultCache();
- bool read_result_from_query_result_cache = false; /// a query must not read from *and* write to the query result cache at the same time
- if (query_result_cache != nullptr
- && (settings.enable_experimental_query_result_cache || settings.enable_experimental_query_result_cache_passive_usage)
+ /// If
+ /// - it is a SELECT query,
+ /// - passive (read) use of the query cache is enabled, and
+ /// - the query cache knows the query result
+ /// then replace the pipeline by a new pipeline with a single source that is populated from the query cache
+ auto query_cache = context->getQueryCache();
+ bool read_result_from_query_cache = false; /// a query must not read from *and* write to the query cache at the same time
+ if (query_cache != nullptr
+ && (settings.allow_experimental_query_cache && settings.use_query_cache && settings.enable_reads_from_query_cache)
&& res.pipeline.pulling())
{
- QueryResultCache::Key key(
+ QueryCache::Key key(
ast, res.pipeline.getHeader(),
std::make_optional(context->getUserName()),
- std::chrono::system_clock::now() + std::chrono::seconds(settings.query_result_cache_ttl));
- QueryResultCache::Reader reader = query_result_cache->createReader(key);
+ std::chrono::system_clock::now() + std::chrono::seconds(settings.query_cache_ttl));
+ QueryCache::Reader reader = query_cache->createReader(key);
if (reader.hasCacheEntryForKey())
{
res.pipeline = QueryPipeline(reader.getPipe());
- read_result_from_query_result_cache = true;
+ read_result_from_query_cache = true;
}
}
- /// Try to write (SELECT) query result into query result cache (if it is enabled)
- if (!read_result_from_query_result_cache
- && query_result_cache != nullptr
- && settings.enable_experimental_query_result_cache
+ /// If
+ /// - it is a SELECT query, and
+ /// - active (write) use of the query cache is enabled
+ /// then add a processor on top of the pipeline which stores the result in the query cache.
+ if (!read_result_from_query_cache
+ && query_cache != nullptr
+ && settings.allow_experimental_query_cache && settings.use_query_cache && settings.enable_writes_to_query_cache
&& res.pipeline.pulling()
- && (!astContainsNonDeterministicFunctions(ast, context) || settings.query_result_cache_store_results_of_queries_with_nondeterministic_functions))
+ && (!astContainsNonDeterministicFunctions(ast, context) || settings.query_cache_store_results_of_queries_with_nondeterministic_functions))
{
- QueryResultCache::Key key(
+ QueryCache::Key key(
ast, res.pipeline.getHeader(),
- settings.query_result_cache_share_between_users ? std::nullopt : std::make_optional(context->getUserName()),
- std::chrono::system_clock::now() + std::chrono::seconds(settings.query_result_cache_ttl));
+ settings.query_cache_share_between_users ? std::nullopt : std::make_optional(context->getUserName()),
+ std::chrono::system_clock::now() + std::chrono::seconds(settings.query_cache_ttl));
- const size_t num_query_runs = query_result_cache->recordQueryRun(key);
- if (num_query_runs > settings.query_result_cache_min_query_runs)
+ const size_t num_query_runs = query_cache->recordQueryRun(key);
+ if (num_query_runs > settings.query_cache_min_query_runs)
{
- auto stream_in_query_result_cache_transform = std::make_shared(res.pipeline.getHeader(), query_result_cache, key,
- std::chrono::milliseconds(context->getSettings().query_result_cache_min_query_duration.totalMilliseconds()));
- res.pipeline.streamIntoQueryResultCache(stream_in_query_result_cache_transform);
+ auto stream_in_query_cache_transform = std::make_shared(res.pipeline.getHeader(), query_cache, key,
+ std::chrono::milliseconds(context->getSettings().query_cache_min_query_duration.totalMilliseconds()));
+ res.pipeline.streamIntoQueryCache(stream_in_query_cache_transform);
}
}
@@ -901,8 +908,10 @@ static std::tuple executeQueryImpl(
auto finish_callback = [elem,
context,
ast,
- enable_experimental_query_result_cache = settings.enable_experimental_query_result_cache,
- query_result_cache_store_results_of_queries_with_nondeterministic_functions = settings.query_result_cache_store_results_of_queries_with_nondeterministic_functions,
+ allow_experimental_query_cache = settings.allow_experimental_query_cache,
+ use_query_cache = settings.use_query_cache,
+ enable_writes_to_query_cache = settings.enable_writes_to_query_cache,
+ query_cache_store_results_of_queries_with_nondeterministic_functions = settings.query_cache_store_results_of_queries_with_nondeterministic_functions,
log_queries,
log_queries_min_type = settings.log_queries_min_type,
log_queries_min_query_duration_ms = settings.log_queries_min_query_duration_ms.totalMilliseconds(),
@@ -912,14 +921,15 @@ static std::tuple executeQueryImpl(
pulling_pipeline = pipeline.pulling(),
query_span](QueryPipeline & query_pipeline) mutable
{
- /// Write query result into query result cache (if enabled)
- auto query_result_cache = context->getQueryResultCache();
- if (query_result_cache != nullptr
+ /// If active (write) use of the query cache is enabled and the query is eligible for result caching, then store the query
+ /// result buffered in the special-purpose cache processor (added on top of the pipeline) into the cache.
+ auto query_cache = context->getQueryCache();
+ if (query_cache != nullptr
&& pulling_pipeline
- && enable_experimental_query_result_cache
- && (!astContainsNonDeterministicFunctions(ast, context) || query_result_cache_store_results_of_queries_with_nondeterministic_functions))
+ && allow_experimental_query_cache && use_query_cache && enable_writes_to_query_cache
+ && (!astContainsNonDeterministicFunctions(ast, context) || query_cache_store_results_of_queries_with_nondeterministic_functions))
{
- query_pipeline.finalizeWriteInQueryResultCache();
+ query_pipeline.finalizeWriteInQueryCache();
}
QueryStatusPtr process_list_elem = context->getProcessListElement();
diff --git a/src/Parsers/ASTDeleteQuery.cpp b/src/Parsers/ASTDeleteQuery.cpp
index 08b40f65121..09dc4b936ae 100644
--- a/src/Parsers/ASTDeleteQuery.cpp
+++ b/src/Parsers/ASTDeleteQuery.cpp
@@ -41,6 +41,8 @@ void ASTDeleteQuery::formatQueryImpl(const FormatSettings & settings, FormatStat
}
settings.ostr << backQuoteIfNeed(getTable());
+ formatOnCluster(settings);
+
settings.ostr << (settings.hilite ? hilite_keyword : "") << " WHERE " << (settings.hilite ? hilite_none : "");
predicate->formatImpl(settings, state, frame);
}
diff --git a/src/Parsers/ASTDeleteQuery.h b/src/Parsers/ASTDeleteQuery.h
index bcb97639b64..1dab684ffc9 100644
--- a/src/Parsers/ASTDeleteQuery.h
+++ b/src/Parsers/ASTDeleteQuery.h
@@ -2,15 +2,20 @@
#include
#include
+#include
namespace DB
{
/// DELETE FROM [db.]name WHERE ...
-class ASTDeleteQuery : public ASTQueryWithTableAndOutput
+class ASTDeleteQuery : public ASTQueryWithTableAndOutput, public ASTQueryWithOnCluster
{
public:
String getID(char delim) const final;
ASTPtr clone() const final;
+ ASTPtr getRewrittenASTWithoutOnCluster(const WithoutOnClusterASTRewriteParams & params) const override
+ {
+ return removeOnCluster(clone(), params.default_database);
+ }
ASTPtr predicate;
diff --git a/src/Parsers/ASTQualifiedAsterisk.h b/src/Parsers/ASTQualifiedAsterisk.h
index e67b4cd82dd..079b83ae171 100644
--- a/src/Parsers/ASTQualifiedAsterisk.h
+++ b/src/Parsers/ASTQualifiedAsterisk.h
@@ -17,8 +17,13 @@ public:
ASTPtr clone() const override
{
auto clone = std::make_shared(*this);
+ clone->children.clear();
- if (transformers) { clone->transformers = transformers->clone(); clone->children.push_back(clone->transformers); }
+ if (transformers)
+ {
+ clone->transformers = transformers->clone();
+ clone->children.push_back(clone->transformers);
+ }
clone->qualifier = qualifier->clone();
clone->children.push_back(clone->qualifier);
diff --git a/src/Parsers/ASTShowEngineQuery.h b/src/Parsers/ASTShowEngineQuery.h
new file mode 100644
index 00000000000..7a447a4f24b
--- /dev/null
+++ b/src/Parsers/ASTShowEngineQuery.h
@@ -0,0 +1,17 @@
+#pragma once
+
+#include
+
+
+namespace DB
+{
+
+struct ASTShowEngineAndQueryNames
+{
+ static constexpr auto ID = "ShowEngineQuery";
+ static constexpr auto Query = "SHOW ENGINES";
+};
+
+using ASTShowEnginesQuery = ASTQueryWithOutputImpl;
+
+}
diff --git a/src/Parsers/ASTSystemQuery.cpp b/src/Parsers/ASTSystemQuery.cpp
index bfc7c5e6a45..ffb018f23fe 100644
--- a/src/Parsers/ASTSystemQuery.cpp
+++ b/src/Parsers/ASTSystemQuery.cpp
@@ -206,6 +206,10 @@ void ASTSystemQuery::formatImpl(const FormatSettings & settings, FormatState &,
{
settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(backup_name);
}
+ else if (type == Type::SYNC_FILE_CACHE)
+ {
+ settings.ostr << (settings.hilite ? hilite_none : "");
+ }
}
diff --git a/src/Parsers/ASTSystemQuery.h b/src/Parsers/ASTSystemQuery.h
index 499807df7df..02ddbc7dcd2 100644
--- a/src/Parsers/ASTSystemQuery.h
+++ b/src/Parsers/ASTSystemQuery.h
@@ -25,7 +25,7 @@ public:
DROP_INDEX_MARK_CACHE,
DROP_INDEX_UNCOMPRESSED_CACHE,
DROP_MMAP_CACHE,
- DROP_QUERY_RESULT_CACHE,
+ DROP_QUERY_CACHE,
#if USE_EMBEDDED_COMPILER
DROP_COMPILED_EXPRESSION_CACHE,
#endif
@@ -42,6 +42,7 @@ public:
SYNC_REPLICA,
SYNC_DATABASE_REPLICA,
SYNC_TRANSACTION_LOG,
+ SYNC_FILE_CACHE,
RELOAD_DICTIONARY,
RELOAD_DICTIONARIES,
RELOAD_MODEL,
diff --git a/src/Parsers/ParserDeleteQuery.cpp b/src/Parsers/ParserDeleteQuery.cpp
index 7b8057d227e..7b27651d82d 100644
--- a/src/Parsers/ParserDeleteQuery.cpp
+++ b/src/Parsers/ParserDeleteQuery.cpp
@@ -18,6 +18,7 @@ bool ParserDeleteQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
ParserKeyword s_where("WHERE");
ParserExpression parser_exp_elem;
ParserKeyword s_settings("SETTINGS");
+ ParserKeyword s_on{"ON"};
if (s_delete.ignore(pos, expected))
{
@@ -27,6 +28,14 @@ bool ParserDeleteQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
if (!parseDatabaseAndTableAsAST(pos, expected, query->database, query->table))
return false;
+ if (s_on.ignore(pos, expected))
+ {
+ String cluster_str;
+ if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected))
+ return false;
+ query->cluster = cluster_str;
+ }
+
if (!s_where.ignore(pos, expected))
return false;
diff --git a/src/Parsers/ParserQueryWithOutput.cpp b/src/Parsers/ParserQueryWithOutput.cpp
index 163e71e3201..7024d8cbe11 100644
--- a/src/Parsers/ParserQueryWithOutput.cpp
+++ b/src/Parsers/ParserQueryWithOutput.cpp
@@ -15,6 +15,7 @@
#include
#include
#include