diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 4b201802cae..db170c3e28f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,7 +2,7 @@ A technical comment, you are free to remove or leave it as it is when PR is created The following categories are used in the next scripts, update them accordingly utils/changelog/changelog.py -tests/ci/run_check.py +tests/ci/cancel_and_rerun_workflow_lambda/app.py --> ### Changelog category (leave one): - New Feature diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 782cfad43f8..506ed451b6d 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -1308,6 +1308,40 @@ jobs: docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestReleaseAnalyzer: + needs: [BuilderDebRelease] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_analyzer + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (release, analyzer) + REPO_COPY=${{runner.temp}}/stateless_analyzer/ClickHouse + KILL_TIMEOUT=10800 + EOF + - name: Download json reports + uses: actions/download-artifact@v3 + with: + path: ${{ env.REPORTS_PATH }} + - name: Check out repository code + uses: ClickHouse/checkout@v1 + with: + clear-repository: true + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker ps --quiet | xargs --no-run-if-empty docker kill ||: + docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: + sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestReleaseS3_0: needs: [BuilderDebRelease] runs-on: [self-hosted, func-tester] @@ -4755,6 +4789,7 @@ jobs: - FunctionalStatelessTestReleaseDatabaseReplicated2 - FunctionalStatelessTestReleaseDatabaseReplicated3 - FunctionalStatelessTestReleaseWideParts + - FunctionalStatelessTestReleaseAnalyzer - FunctionalStatelessTestAarch64 - FunctionalStatelessTestAsan0 - FunctionalStatelessTestAsan1 diff --git a/.gitmodules b/.gitmodules index ca55281e643..ed61ddb96ba 100644 --- a/.gitmodules +++ b/.gitmodules @@ -253,9 +253,6 @@ [submodule "contrib/qpl"] path = contrib/qpl url = https://github.com/intel/qpl -[submodule "contrib/idxd-config"] - path = contrib/idxd-config - url = https://github.com/intel/idxd-config [submodule "contrib/wyhash"] path = contrib/wyhash url = https://github.com/wangyi-fudan/wyhash @@ -296,6 +293,9 @@ [submodule "contrib/libdivide"] path = contrib/libdivide url = https://github.com/ridiculousfish/libdivide +[submodule "contrib/libbcrypt"] + path = contrib/libbcrypt + url = https://github.com/rg3/libbcrypt.git [submodule "contrib/ulid-c"] path = contrib/ulid-c url = https://github.com/ClickHouse/ulid-c.git @@ -335,3 +335,6 @@ [submodule "contrib/liburing"] path = contrib/liburing url = https://github.com/axboe/liburing +[submodule "contrib/isa-l"] + path = contrib/isa-l + url = https://github.com/ClickHouse/isa-l.git diff --git a/CHANGELOG.md b/CHANGELOG.md index 47320208f02..1ccd4f9846d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ### Table of Contents +**[ClickHouse release v23.4, 2023-04-26](#234)**
**[ClickHouse release v23.3 LTS, 2023-03-30](#233)**
**[ClickHouse release v23.2, 2023-02-23](#232)**
**[ClickHouse release v23.1, 2023-01-25](#231)**
@@ -6,6 +7,153 @@ # 2023 Changelog +### ClickHouse release 23.4, 2023-04-26 + +#### Backward Incompatible Change +* Formatter '%M' in function formatDateTime() now prints the month name instead of the minutes. This makes the behavior consistent with MySQL. The previous behavior can be restored using setting "formatdatetime_parsedatetime_m_is_month_name = 0". [#47246](https://github.com/ClickHouse/ClickHouse/pull/47246) ([Robert Schulze](https://github.com/rschu1ze)). +* This change makes sense only if you are using the virtual filesystem cache. If `path` in the virtual filesystem cache configuration is not empty and is not an absolute path, then it will be put in `/caches/`. [#48784](https://github.com/ClickHouse/ClickHouse/pull/48784) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Primary/secondary indices and sorting keys with identical expressions are now rejected. This behavior can be disabled using setting `allow_suspicious_indices`. [#48536](https://github.com/ClickHouse/ClickHouse/pull/48536) ([凌涛](https://github.com/lingtaolf)). + +#### New Feature +* Support new aggregate function `quantileGK`/`quantilesGK`, like [approx_percentile](https://spark.apache.org/docs/latest/api/sql/index.html#approx_percentile) in spark. Greenwald-Khanna algorithm refer to http://infolab.stanford.edu/~datar/courses/cs361a/papers/quantiles.pdf. [#46428](https://github.com/ClickHouse/ClickHouse/pull/46428) ([李扬](https://github.com/taiyang-li)). +* Add a statement `SHOW COLUMNS` which shows distilled information from system.columns. [#48017](https://github.com/ClickHouse/ClickHouse/pull/48017) ([Robert Schulze](https://github.com/rschu1ze)). +* Added `LIGHTWEIGHT` and `PULL` modifiers for `SYSTEM SYNC REPLICA` query. `LIGHTWEIGHT` version waits for fetches and drop-ranges only (merges and mutations are ignored). `PULL` version pulls new entries from ZooKeeper and does not wait for them. Fixes [#47794](https://github.com/ClickHouse/ClickHouse/issues/47794). [#48085](https://github.com/ClickHouse/ClickHouse/pull/48085) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add `kafkaMurmurHash` function for compatibility with Kafka DefaultPartitioner. Closes [#47834](https://github.com/ClickHouse/ClickHouse/issues/47834). [#48185](https://github.com/ClickHouse/ClickHouse/pull/48185) ([Nikolay Degterinsky](https://github.com/evillique)). +* Allow to easily create a user with the same grants as the current user by using `GRANT CURRENT GRANTS`. [#48262](https://github.com/ClickHouse/ClickHouse/pull/48262) ([pufit](https://github.com/pufit)). +* Add statistical aggregate function `kolmogorovSmirnovTest`. Close [#48228](https://github.com/ClickHouse/ClickHouse/issues/48228). [#48325](https://github.com/ClickHouse/ClickHouse/pull/48325) ([FFFFFFFHHHHHHH](https://github.com/FFFFFFFHHHHHHH)). +* Added a `lost_part_count` column to the `system.replicas` table. The column value shows the total number of lost parts in the corresponding table. Value is stored in zookeeper and can be used instead of not persistent `ReplicatedDataLoss` profile event for monitoring. [#48526](https://github.com/ClickHouse/ClickHouse/pull/48526) ([Sergei Trifonov](https://github.com/serxa)). +* Add `soundex` function for compatibility. Closes [#39880](https://github.com/ClickHouse/ClickHouse/issues/39880). [#48567](https://github.com/ClickHouse/ClickHouse/pull/48567) ([FriendLey](https://github.com/FriendLey)). +* Support `Map` type for JSONExtract. [#48629](https://github.com/ClickHouse/ClickHouse/pull/48629) ([李扬](https://github.com/taiyang-li)). +* Add `PrettyJSONEachRow` format to output pretty JSON with new line delimiters and 4 space indents. [#48898](https://github.com/ClickHouse/ClickHouse/pull/48898) ([Kruglov Pavel](https://github.com/Avogar)). +* Add `ParquetMetadata` input format to read Parquet file metadata. [#48911](https://github.com/ClickHouse/ClickHouse/pull/48911) ([Kruglov Pavel](https://github.com/Avogar)). +* Add `extractKeyValuePairs` function to extract key value pairs from strings. Input strings might contain noise (i.e. log files / do not need to be 100% formatted in key-value-pair format), the algorithm will look for key value pairs matching the arguments passed to the function. As of now, function accepts the following arguments: `data_column` (mandatory), `key_value_pair_delimiter` (defaults to `:`), `pair_delimiters` (defaults to `\space \, \;`) and `quoting_character` (defaults to double quotes). [#43606](https://github.com/ClickHouse/ClickHouse/pull/43606) ([Arthur Passos](https://github.com/arthurpassos)). +* Functions replaceOne(), replaceAll(), replaceRegexpOne() and replaceRegexpAll() can now be called with non-const pattern and replacement arguments. [#46589](https://github.com/ClickHouse/ClickHouse/pull/46589) ([Robert Schulze](https://github.com/rschu1ze)). +* Added functions to work with columns of type `Map`: `mapConcat`, `mapSort`, `mapExists`. [#48071](https://github.com/ClickHouse/ClickHouse/pull/48071) ([Anton Popov](https://github.com/CurtizJ)). + +#### Performance Improvement +* Reading files in `Parquet` format is now much faster. IO and decoding are parallelized (controlled by `max_threads` setting), and only required data ranges are read. [#47964](https://github.com/ClickHouse/ClickHouse/pull/47964) ([Michael Kolupaev](https://github.com/al13n321)). +* If we run a mutation with IN (subquery) like this: `ALTER TABLE t UPDATE col='new value' WHERE id IN (SELECT id FROM huge_table)` and the table `t` has multiple parts than for each part a set for subquery `SELECT id FROM huge_table` is built in memory. And if there are many parts then this might consume a lot of memory (and lead to an OOM) and CPU. The solution is to introduce a short-lived cache of sets that are currently being built by mutation tasks. If another task of the same mutation is executed concurrently it can look up the set in the cache, wait for it to be built and reuse it. [#46835](https://github.com/ClickHouse/ClickHouse/pull/46835) ([Alexander Gololobov](https://github.com/davenger)). +* Only check dependencies if necessary when applying `ALTER TABLE` queries. [#48062](https://github.com/ClickHouse/ClickHouse/pull/48062) ([Raúl Marín](https://github.com/Algunenano)). +* Optimize function `mapUpdate`. [#48118](https://github.com/ClickHouse/ClickHouse/pull/48118) ([Anton Popov](https://github.com/CurtizJ)). +* Now an internal query to local replica is sent explicitly and data from it received through loopback interface. Setting `prefer_localhost_replica` is not respected for parallel replicas. This is needed for better scheduling and makes the code cleaner: the initiator is only responsible for coordinating of the reading process and merging results, continuously answering for requests while all the secondary queries read the data. Note: Using loopback interface is not so performant, otherwise some replicas could starve for tasks which could lead to even slower query execution and not utilizing all possible resources. The initialization of the coordinator is now even more lazy. All incoming requests contain the information about the reading algorithm we initialize the coordinator with it when first request comes. If any replica decides to read with a different algorithm–an exception will be thrown and a query will be aborted. [#48246](https://github.com/ClickHouse/ClickHouse/pull/48246) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Do not build set for the right side of `IN` clause with subquery when it is used only for analysis of skip indexes, and they are disabled by setting (`use_skip_indexes=0`). Previously it might affect the performance of queries. [#48299](https://github.com/ClickHouse/ClickHouse/pull/48299) ([Anton Popov](https://github.com/CurtizJ)). +* Query processing is parallelized right after reading `FROM file(...)`. Related to [#38755](https://github.com/ClickHouse/ClickHouse/issues/38755). [#48525](https://github.com/ClickHouse/ClickHouse/pull/48525) ([Igor Nikonov](https://github.com/devcrafter)). Query processing is parallelized right after reading from any data source. Affected data sources are mostly simple or external storages like table functions `url`, `file`. [#48727](https://github.com/ClickHouse/ClickHouse/pull/48727) ([Igor Nikonov](https://github.com/devcrafter)). This is controlled by the setting `parallelize_output_from_storages` which is not enabled by default. +* Lowered contention of ThreadPool mutex (may increase performance for a huge amount of small jobs). [#48750](https://github.com/ClickHouse/ClickHouse/pull/48750) ([Sergei Trifonov](https://github.com/serxa)). +* Reduce memory usage for multiple `ALTER DELETE` mutations. [#48522](https://github.com/ClickHouse/ClickHouse/pull/48522) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Remove the excessive connection attempts if the `skip_unavailable_shards` setting is enabled. [#48771](https://github.com/ClickHouse/ClickHouse/pull/48771) ([Azat Khuzhin](https://github.com/azat)). + +#### Experimental Feature +* Entries in the query cache are now squashed to max_block_size and compressed. [#45912](https://github.com/ClickHouse/ClickHouse/pull/45912) ([Robert Schulze](https://github.com/rschu1ze)). +* It is now possible to define per-user quotas in the query cache. [#48284](https://github.com/ClickHouse/ClickHouse/pull/48284) ([Robert Schulze](https://github.com/rschu1ze)). +* Some fixes for parallel replicas [#48433](https://github.com/ClickHouse/ClickHouse/pull/48433) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Implement zero-copy-replication (an experimental feature) on encrypted disks. [#48741](https://github.com/ClickHouse/ClickHouse/pull/48741) ([Vitaly Baranov](https://github.com/vitlibar)). + +#### Improvement +* Increase default value for `connect_timeout_with_failover_ms` to 1000 ms (because of adding async connections in https://github.com/ClickHouse/ClickHouse/pull/47229) . Closes [#5188](https://github.com/ClickHouse/ClickHouse/issues/5188). [#49009](https://github.com/ClickHouse/ClickHouse/pull/49009) ([Kruglov Pavel](https://github.com/Avogar)). +* Several improvements around data lakes: - Make `Iceberg` work with non-partitioned data. - Support `Iceberg` format version v2 (previously only v1 was supported) - Support reading partitioned data for `DeltaLake`/`Hudi` - Faster reading of `DeltaLake` metadata by using Delta's checkpoint files - Fixed incorrect `Hudi` reads: previously it incorrectly chose which data to read and therefore was able to read correctly only small size tables - Made these engines to pickup updates of changed data (previously the state was set on table creation) - Make proper testing for `Iceberg`/`DeltaLake`/`Hudi` using spark. [#47307](https://github.com/ClickHouse/ClickHouse/pull/47307) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add async connection to socket and async writing to socket. Make creating connections and sending query/external tables async across shards. Refactor code with fibers. Closes [#46931](https://github.com/ClickHouse/ClickHouse/issues/46931). We will be able to increase `connect_timeout_with_failover_ms` by default after this PR (https://github.com/ClickHouse/ClickHouse/issues/5188). [#47229](https://github.com/ClickHouse/ClickHouse/pull/47229) ([Kruglov Pavel](https://github.com/Avogar)). +* Support config sections `keeper`/`keeper_server` as an alternative to `zookeeper`. Close [#34766](https://github.com/ClickHouse/ClickHouse/issues/34766) , [#34767](https://github.com/ClickHouse/ClickHouse/issues/34767). [#35113](https://github.com/ClickHouse/ClickHouse/pull/35113) ([李扬](https://github.com/taiyang-li)). +* It is possible to set _secure_ flag in named_collections for a dictionary with a ClickHouse table source. Addresses [#38450](https://github.com/ClickHouse/ClickHouse/issues/38450) . [#46323](https://github.com/ClickHouse/ClickHouse/pull/46323) ([Ilya Golshtein](https://github.com/ilejn)). +* `bitCount` function support `FixedString` and `String` data type. [#49044](https://github.com/ClickHouse/ClickHouse/pull/49044) ([flynn](https://github.com/ucasfl)). +* Added configurable retries for all operations with [Zoo]Keeper for Backup queries. [#47224](https://github.com/ClickHouse/ClickHouse/pull/47224) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Enable `use_environment_credentials` for S3 by default, so the entire provider chain is constructed by default. [#47397](https://github.com/ClickHouse/ClickHouse/pull/47397) ([Antonio Andelic](https://github.com/antonio2368)). +* Currently, the JSON_VALUE function is similar as spark's get_json_object function, which support to get value from JSON string by a path like '$.key'. But still has something different - 1. in spark's get_json_object will return null while the path is not exist, but in JSON_VALUE will return empty string; - 2. in spark's get_json_object will return a complex type value, such as a JSON object/array value, but in JSON_VALUE will return empty string. [#47494](https://github.com/ClickHouse/ClickHouse/pull/47494) ([KevinyhZou](https://github.com/KevinyhZou)). +* For `use_structure_from_insertion_table_in_table_functions` more flexible insert table structure propagation to table function. Fixed an issue with name mapping and using virtual columns. No more need for 'auto' setting. [#47962](https://github.com/ClickHouse/ClickHouse/pull/47962) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Do not continue retrying to connect to Keeper if the query is killed or over limits. [#47985](https://github.com/ClickHouse/ClickHouse/pull/47985) ([Raúl Marín](https://github.com/Algunenano)). +* Support Enum output/input in `BSONEachRow`, allow all map key types and avoid extra calculations on output. [#48122](https://github.com/ClickHouse/ClickHouse/pull/48122) ([Kruglov Pavel](https://github.com/Avogar)). +* Support more ClickHouse types in `ORC`/`Arrow`/`Parquet` formats: Enum(8|16), (U)Int(128|256), Decimal256 (for ORC), allow reading IPv4 from Int32 values (ORC outputs IPv4 as Int32, and we couldn't read it back), fix reading Nullable(IPv6) from binary data for `ORC`. [#48126](https://github.com/ClickHouse/ClickHouse/pull/48126) ([Kruglov Pavel](https://github.com/Avogar)). +* Add columns `perform_ttl_move_on_insert`, `load_balancing` for table `system.storage_policies`, modify column `volume_type` type to `Enum8`. [#48167](https://github.com/ClickHouse/ClickHouse/pull/48167) ([lizhuoyu5](https://github.com/lzydmxy)). +* Added support for `BACKUP ALL` command which backups all tables and databases, including temporary and system ones. [#48189](https://github.com/ClickHouse/ClickHouse/pull/48189) ([Vitaly Baranov](https://github.com/vitlibar)). +* Function mapFromArrays supports `Map` type as an input. [#48207](https://github.com/ClickHouse/ClickHouse/pull/48207) ([李扬](https://github.com/taiyang-li)). +* The output of some SHOW PROCESSLIST is now sorted. [#48241](https://github.com/ClickHouse/ClickHouse/pull/48241) ([Robert Schulze](https://github.com/rschu1ze)). +* Per-query/per-server throttling for remote IO/local IO/BACKUPs (server settings: `max_remote_read_network_bandwidth_for_server`, `max_remote_write_network_bandwidth_for_server`, `max_local_read_bandwidth_for_server`, `max_local_write_bandwidth_for_server`, `max_backup_bandwidth_for_server`, settings: `max_remote_read_network_bandwidth`, `max_remote_write_network_bandwidth`, `max_local_read_bandwidth`, `max_local_write_bandwidth`, `max_backup_bandwidth`). [#48242](https://github.com/ClickHouse/ClickHouse/pull/48242) ([Azat Khuzhin](https://github.com/azat)). +* Support more types in `CapnProto` format: Map, (U)Int(128|256), Decimal(128|256). Allow integer conversions during input/output. [#48257](https://github.com/ClickHouse/ClickHouse/pull/48257) ([Kruglov Pavel](https://github.com/Avogar)). +* Don't throw CURRENT_WRITE_BUFFER_IS_EXHAUSTED for normal behaviour. [#48288](https://github.com/ClickHouse/ClickHouse/pull/48288) ([Raúl Marín](https://github.com/Algunenano)). +* Add new setting `keeper_map_strict_mode` which enforces extra guarantees on operations made on top of `KeeperMap` tables. [#48293](https://github.com/ClickHouse/ClickHouse/pull/48293) ([Antonio Andelic](https://github.com/antonio2368)). +* Check primary key type for simple dictionary is native unsigned integer type Add setting `check_dictionary_primary_key ` for compatibility(set `check_dictionary_primary_key =false` to disable checking). [#48335](https://github.com/ClickHouse/ClickHouse/pull/48335) ([lizhuoyu5](https://github.com/lzydmxy)). +* Don't replicate mutations for `KeeperMap` because it's unnecessary. [#48354](https://github.com/ClickHouse/ClickHouse/pull/48354) ([Antonio Andelic](https://github.com/antonio2368)). +* Allow to write/read unnamed tuple as nested Message in Protobuf format. Tuple elements and Message fields are matched by position. [#48390](https://github.com/ClickHouse/ClickHouse/pull/48390) ([Kruglov Pavel](https://github.com/Avogar)). +* Support `additional_table_filters` and `additional_result_filter` settings in the new planner. Also, add a documentation entry for `additional_result_filter`. [#48405](https://github.com/ClickHouse/ClickHouse/pull/48405) ([Dmitry Novik](https://github.com/novikd)). +* `parseDateTime` now understands format string '%f' (fractional seconds). [#48420](https://github.com/ClickHouse/ClickHouse/pull/48420) ([Robert Schulze](https://github.com/rschu1ze)). +* Format string "%f" in formatDateTime() now prints "000000" if the formatted value has no fractional seconds, the previous behavior (single zero) can be restored using setting "formatdatetime_f_prints_single_zero = 1". [#48422](https://github.com/ClickHouse/ClickHouse/pull/48422) ([Robert Schulze](https://github.com/rschu1ze)). +* Don't replicate DELETE and TRUNCATE for KeeperMap. [#48434](https://github.com/ClickHouse/ClickHouse/pull/48434) ([Antonio Andelic](https://github.com/antonio2368)). +* Generate valid Decimals and Bools in generateRandom function. [#48436](https://github.com/ClickHouse/ClickHouse/pull/48436) ([Kruglov Pavel](https://github.com/Avogar)). +* Allow trailing commas in expression list of SELECT query, for example `SELECT a, b, c, FROM table`. Closes [#37802](https://github.com/ClickHouse/ClickHouse/issues/37802). [#48438](https://github.com/ClickHouse/ClickHouse/pull/48438) ([Nikolay Degterinsky](https://github.com/evillique)). +* Override `CLICKHOUSE_USER` and `CLICKHOUSE_PASSWORD` environment variables with `--user` and `--password` client parameters. Closes [#38909](https://github.com/ClickHouse/ClickHouse/issues/38909). [#48440](https://github.com/ClickHouse/ClickHouse/pull/48440) ([Nikolay Degterinsky](https://github.com/evillique)). +* Added retries to loading of data parts in `MergeTree` tables in case of retryable errors. [#48442](https://github.com/ClickHouse/ClickHouse/pull/48442) ([Anton Popov](https://github.com/CurtizJ)). +* Add support for `Date`, `Date32`, `DateTime`, `DateTime64` data types to `arrayMin`, `arrayMax`, `arrayDifference` functions. Closes [#21645](https://github.com/ClickHouse/ClickHouse/issues/21645). [#48445](https://github.com/ClickHouse/ClickHouse/pull/48445) ([Nikolay Degterinsky](https://github.com/evillique)). +* Add support for `{server_uuid}` macro. It is useful for identifying replicas in autoscaled clusters when new replicas are constantly added and removed in runtime. This closes [#48554](https://github.com/ClickHouse/ClickHouse/issues/48554). [#48563](https://github.com/ClickHouse/ClickHouse/pull/48563) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The installation script will create a hard link instead of copying if it is possible. [#48578](https://github.com/ClickHouse/ClickHouse/pull/48578) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support `SHOW TABLE` syntax meaning the same as `SHOW CREATE TABLE`. Closes [#48580](https://github.com/ClickHouse/ClickHouse/issues/48580). [#48591](https://github.com/ClickHouse/ClickHouse/pull/48591) ([flynn](https://github.com/ucasfl)). +* HTTP temporary buffers now support working by evicting data from the virtual filesystem cache. [#48664](https://github.com/ClickHouse/ClickHouse/pull/48664) ([Vladimir C](https://github.com/vdimir)). +* Make Schema inference works for `CREATE AS SELECT`. Closes [#47599](https://github.com/ClickHouse/ClickHouse/issues/47599). [#48679](https://github.com/ClickHouse/ClickHouse/pull/48679) ([flynn](https://github.com/ucasfl)). +* Added a `replicated_max_mutations_in_one_entry` setting for `ReplicatedMergeTree` that allows limiting the number of mutation commands per one `MUTATE_PART` entry (default is 10000). [#48731](https://github.com/ClickHouse/ClickHouse/pull/48731) ([Alexander Tokmakov](https://github.com/tavplubix)). +* In AggregateFunction types, don't count unused arena bytes as `read_bytes`. [#48745](https://github.com/ClickHouse/ClickHouse/pull/48745) ([Raúl Marín](https://github.com/Algunenano)). +* Fix some MySQL-related settings not being handled with the MySQL dictionary source + named collection. Closes [#48402](https://github.com/ClickHouse/ClickHouse/issues/48402). [#48759](https://github.com/ClickHouse/ClickHouse/pull/48759) ([Kseniia Sumarokova](https://github.com/kssenii)). +* If a user set `max_single_part_upload_size` to a very large value, it can lead to a crash due to a bug in the AWS S3 SDK. This fixes [#47679](https://github.com/ClickHouse/ClickHouse/issues/47679). [#48816](https://github.com/ClickHouse/ClickHouse/pull/48816) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix data race in `RabbitMQ` ([report](https://pastila.nl/?004f7100/de1505289ab5bb355e67ebe6c7cc8707)), refactor the code. [#48845](https://github.com/ClickHouse/ClickHouse/pull/48845) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add aliases `name` and `part_name` form `system.parts` and `system.part_log`. Closes [#48718](https://github.com/ClickHouse/ClickHouse/issues/48718). [#48850](https://github.com/ClickHouse/ClickHouse/pull/48850) ([sichenzhao](https://github.com/sichenzhao)). +* Functions "arrayDifferenceSupport()", "arrayCumSum()" and "arrayCumSumNonNegative()" now support input arrays of wide integer types (U)Int128/256. [#48866](https://github.com/ClickHouse/ClickHouse/pull/48866) ([cluster](https://github.com/infdahai)). +* Multi-line history in clickhouse-client is now no longer padded. This makes pasting more natural. [#48870](https://github.com/ClickHouse/ClickHouse/pull/48870) ([Joanna Hulboj](https://github.com/jh0x)). +* Implement a slight improvement for the rare case when ClickHouse is run inside LXC and LXCFS is used. The LXCFS has an issue: sometimes it returns an error "Transport endpoint is not connected" on reading from the file inside `/proc`. This error was correctly logged into ClickHouse's server log. We have additionally workaround this issue by reopening a file. This is a minuscule change. [#48922](https://github.com/ClickHouse/ClickHouse/pull/48922) ([Real](https://github.com/RunningXie)). +* Improve memory accounting for prefetches. Randomise prefetch settings In CI. [#48973](https://github.com/ClickHouse/ClickHouse/pull/48973) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Correctly set headers for native copy operations on GCS. [#48981](https://github.com/ClickHouse/ClickHouse/pull/48981) ([Antonio Andelic](https://github.com/antonio2368)). +* Add support for specifying setting names in the command line with dashes instead of underscores, for example, `--max-threads` instead of `--max_threads`. Additionally, support Unicode dash characters like `—` instead of `--` - this is useful when you communicate with a team in another company, and a manager from that team copy-pasted code from MS Word. [#48985](https://github.com/ClickHouse/ClickHouse/pull/48985) ([alekseygolub](https://github.com/alekseygolub)). +* Add fallback to password authentication when authentication with SSL user certificate has failed. Closes [#48974](https://github.com/ClickHouse/ClickHouse/issues/48974). [#48989](https://github.com/ClickHouse/ClickHouse/pull/48989) ([Nikolay Degterinsky](https://github.com/evillique)). +* Improve the embedded dashboard. Close [#46671](https://github.com/ClickHouse/ClickHouse/issues/46671). [#49036](https://github.com/ClickHouse/ClickHouse/pull/49036) ([Kevin Zhang](https://github.com/Kinzeng)). +* Add profile events for log messages, so you can easily see the count of log messages by severity. [#49042](https://github.com/ClickHouse/ClickHouse/pull/49042) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* In previous versions, the `LineAsString` format worked inconsistently when the parallel parsing was enabled or not, in presence of DOS or macOS Classic line breaks. This closes [#49039](https://github.com/ClickHouse/ClickHouse/issues/49039). [#49052](https://github.com/ClickHouse/ClickHouse/pull/49052) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The exception message about the unparsed query parameter will also tell about the name of the parameter. Reimplement [#48878](https://github.com/ClickHouse/ClickHouse/issues/48878). Close [#48772](https://github.com/ClickHouse/ClickHouse/issues/48772). [#49061](https://github.com/ClickHouse/ClickHouse/pull/49061) ([Alexey Milovidov](https://github.com/alexey-milovidov)). + +#### Build/Testing/Packaging Improvement +* Update time zones. The following were updated: Africa/Cairo, Africa/Casablanca, Africa/El_Aaiun, America/Bogota, America/Cambridge_Bay, America/Ciudad_Juarez, America/Godthab, America/Inuvik, America/Iqaluit, America/Nuuk, America/Ojinaga, America/Pangnirtung, America/Rankin_Inlet, America/Resolute, America/Whitehorse, America/Yellowknife, Asia/Gaza, Asia/Hebron, Asia/Kuala_Lumpur, Asia/Singapore, Canada/Yukon, Egypt, Europe/Kirov, Europe/Volgograd, Singapore. [#48572](https://github.com/ClickHouse/ClickHouse/pull/48572) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Reduce the number of dependencies in the header files to speed up the build. [#47984](https://github.com/ClickHouse/ClickHouse/pull/47984) ([Dmitry Novik](https://github.com/novikd)). +* Randomize compression of marks and indices in tests. [#48286](https://github.com/ClickHouse/ClickHouse/pull/48286) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Bump internal ZSTD from 1.5.4 to 1.5.5. [#46797](https://github.com/ClickHouse/ClickHouse/pull/46797) ([Robert Schulze](https://github.com/rschu1ze)). +* Randomize vertical merges from compact to wide parts in tests. [#48287](https://github.com/ClickHouse/ClickHouse/pull/48287) ([Raúl Marín](https://github.com/Algunenano)). +* Support for CRC32 checksum in HDFS. Fix performance issues. [#48614](https://github.com/ClickHouse/ClickHouse/pull/48614) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove remainders of GCC support. [#48671](https://github.com/ClickHouse/ClickHouse/pull/48671) ([Robert Schulze](https://github.com/rschu1ze)). +* Add CI run with new analyzer infrastructure enabled. [#48719](https://github.com/ClickHouse/ClickHouse/pull/48719) ([Dmitry Novik](https://github.com/novikd)). + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Fix system.query_views_log for MVs that are pushed from background threads [#46668](https://github.com/ClickHouse/ClickHouse/pull/46668) ([Azat Khuzhin](https://github.com/azat)). +* Fix several `RENAME COLUMN` bugs [#46946](https://github.com/ClickHouse/ClickHouse/pull/46946) ([alesapin](https://github.com/alesapin)). +* Fix minor hiliting issues in clickhouse-format [#47610](https://github.com/ClickHouse/ClickHouse/pull/47610) ([Natasha Murashkina](https://github.com/murfel)). +* Fix a bug in LLVM's libc++ leading to a crash for uploading parts to S3 which size is greater than INT_MAX [#47693](https://github.com/ClickHouse/ClickHouse/pull/47693) ([Azat Khuzhin](https://github.com/azat)). +* Fix overflow in the `sparkbar` function [#48121](https://github.com/ClickHouse/ClickHouse/pull/48121) ([Vladimir C](https://github.com/vdimir)). +* Fix race in S3 [#48190](https://github.com/ClickHouse/ClickHouse/pull/48190) ([Anton Popov](https://github.com/CurtizJ)). +* Disable JIT for aggregate functions due to inconsistent behavior [#48195](https://github.com/ClickHouse/ClickHouse/pull/48195) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix alter formatting (minor) [#48289](https://github.com/ClickHouse/ClickHouse/pull/48289) ([Natasha Murashkina](https://github.com/murfel)). +* Fix CPU usage in RabbitMQ (was worsened in 23.2 after [#44404](https://github.com/ClickHouse/ClickHouse/issues/44404)) [#48311](https://github.com/ClickHouse/ClickHouse/pull/48311) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix crash in EXPLAIN PIPELINE for Merge over Distributed [#48320](https://github.com/ClickHouse/ClickHouse/pull/48320) ([Azat Khuzhin](https://github.com/azat)). +* Fix serializing LowCardinality as Arrow dictionary [#48361](https://github.com/ClickHouse/ClickHouse/pull/48361) ([Kruglov Pavel](https://github.com/Avogar)). +* Reset downloader for cache file segment in TemporaryFileStream [#48386](https://github.com/ClickHouse/ClickHouse/pull/48386) ([Vladimir C](https://github.com/vdimir)). +* Fix possible SYSTEM SYNC REPLICA stuck in case of DROP/REPLACE PARTITION [#48391](https://github.com/ClickHouse/ClickHouse/pull/48391) ([Azat Khuzhin](https://github.com/azat)). +* Fix a startup error when loading a distributed table that depends on a dictionary [#48419](https://github.com/ClickHouse/ClickHouse/pull/48419) ([MikhailBurdukov](https://github.com/MikhailBurdukov)). +* Don't check dependencies when renaming system tables automatically [#48431](https://github.com/ClickHouse/ClickHouse/pull/48431) ([Raúl Marín](https://github.com/Algunenano)). +* Update only affected rows in KeeperMap storage [#48435](https://github.com/ClickHouse/ClickHouse/pull/48435) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix possible segfault in the VFS cache [#48469](https://github.com/ClickHouse/ClickHouse/pull/48469) ([Kseniia Sumarokova](https://github.com/kssenii)). +* `toTimeZone` function throws an error when no constant string is provided [#48471](https://github.com/ClickHouse/ClickHouse/pull/48471) ([Jordi Villar](https://github.com/jrdi)). +* Fix logical error with IPv4 in Protobuf, add support for Date32 [#48486](https://github.com/ClickHouse/ClickHouse/pull/48486) ([Kruglov Pavel](https://github.com/Avogar)). +* "changed" flag in system.settings was calculated incorrectly for settings with multiple values [#48516](https://github.com/ClickHouse/ClickHouse/pull/48516) ([MikhailBurdukov](https://github.com/MikhailBurdukov)). +* Fix storage `Memory` with enabled compression [#48517](https://github.com/ClickHouse/ClickHouse/pull/48517) ([Anton Popov](https://github.com/CurtizJ)). +* Fix bracketed-paste mode messing up password input in the event of client reconnection [#48528](https://github.com/ClickHouse/ClickHouse/pull/48528) ([Michael Kolupaev](https://github.com/al13n321)). +* Fix nested map for keys of IP and UUID types [#48556](https://github.com/ClickHouse/ClickHouse/pull/48556) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Fix an uncaught exception in case of parallel loader for hashed dictionaries [#48571](https://github.com/ClickHouse/ClickHouse/pull/48571) ([Azat Khuzhin](https://github.com/azat)). +* The `groupArray` aggregate function correctly works for empty result over nullable types [#48593](https://github.com/ClickHouse/ClickHouse/pull/48593) ([lgbo](https://github.com/lgbo-ustc)). +* Fix bug in Keeper when a node is not created with scheme `auth` in ACL sometimes. [#48595](https://github.com/ClickHouse/ClickHouse/pull/48595) ([Aleksei Filatov](https://github.com/aalexfvk)). +* Allow IPv4 comparison operators with UInt [#48611](https://github.com/ClickHouse/ClickHouse/pull/48611) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Fix possible error from cache [#48636](https://github.com/ClickHouse/ClickHouse/pull/48636) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Async inserts with empty data will no longer throw exception. [#48663](https://github.com/ClickHouse/ClickHouse/pull/48663) ([Anton Popov](https://github.com/CurtizJ)). +* Fix table dependencies in case of failed RENAME TABLE [#48683](https://github.com/ClickHouse/ClickHouse/pull/48683) ([Azat Khuzhin](https://github.com/azat)). +* If the primary key has duplicate columns (which is only possible for projections), in previous versions it might lead to a bug [#48838](https://github.com/ClickHouse/ClickHouse/pull/48838) ([Amos Bird](https://github.com/amosbird)). +* Fix for a race condition in ZooKeeper when joining send_thread/receive_thread [#48849](https://github.com/ClickHouse/ClickHouse/pull/48849) ([Alexander Gololobov](https://github.com/davenger)). +* Fix unexpected part name error when trying to drop a ignored detached part with zero copy replication [#48862](https://github.com/ClickHouse/ClickHouse/pull/48862) ([Michael Lex](https://github.com/mlex)). +* Fix reading `Date32` Parquet/Arrow column into not a `Date32` column [#48864](https://github.com/ClickHouse/ClickHouse/pull/48864) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix `UNKNOWN_IDENTIFIER` error while selecting from table with row policy and column with dots [#48976](https://github.com/ClickHouse/ClickHouse/pull/48976) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix aggregation by empty nullable strings [#48999](https://github.com/ClickHouse/ClickHouse/pull/48999) ([LiuNeng](https://github.com/liuneng1994)). + ### ClickHouse release 23.3 LTS, 2023-03-30 #### Upgrade Notes diff --git a/CMakeLists.txt b/CMakeLists.txt index cc1a64a9e96..263b202049b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -170,12 +170,6 @@ else () set(NO_WHOLE_ARCHIVE --no-whole-archive) endif () -option(ENABLE_CURL_BUILD "Enable curl, azure, sentry build on by default except MacOS." ON) -if (OS_DARWIN) - # Disable the curl, azure, senry build on MacOS - set (ENABLE_CURL_BUILD OFF) -endif () - if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE") # Can be lld or ld-lld or lld-13 or /path/to/lld. if (LINKER_NAME MATCHES "lld") @@ -393,9 +387,9 @@ else() endif () option (ENABLE_GWP_ASAN "Enable Gwp-Asan" ON) -# We use mmap for allocations more heavily in debug builds, -# but GWP-ASan also wants to use mmap frequently, -# and due to a large number of memory mappings, +# We use mmap for allocations more heavily in debug builds, +# but GWP-ASan also wants to use mmap frequently, +# and due to a large number of memory mappings, # it does not work together well. if ((NOT OS_LINUX AND NOT OS_ANDROID) OR (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")) set(ENABLE_GWP_ASAN OFF) @@ -421,8 +415,11 @@ endif () set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX") -set (CMAKE_POSITION_INDEPENDENT_CODE OFF) -if (OS_LINUX AND NOT (ARCH_AARCH64 OR ARCH_S390X)) +if (NOT SANITIZE) + set (CMAKE_POSITION_INDEPENDENT_CODE OFF) +endif() + +if (OS_LINUX AND NOT (ARCH_AARCH64 OR ARCH_S390X) AND NOT SANITIZE) # Slightly more efficient code can be generated # It's disabled for ARM because otherwise ClickHouse cannot run on Android. set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-pie") diff --git a/SECURITY.md b/SECURITY.md index 566a1820834..75c1a9d7d6a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -13,21 +13,16 @@ The following versions of ClickHouse server are currently being supported with s | Version | Supported | |:-|:-| +| 23.4 | ✔️ | | 23.3 | ✔️ | | 23.2 | ✔️ | -| 23.1 | ✔️ | +| 23.1 | ❌ | | 22.12 | ❌ | | 22.11 | ❌ | | 22.10 | ❌ | | 22.9 | ❌ | | 22.8 | ✔️ | -| 22.7 | ❌ | -| 22.6 | ❌ | -| 22.5 | ❌ | -| 22.4 | ❌ | -| 22.3 | ❌ | -| 22.2 | ❌ | -| 22.1 | ❌ | +| 22.* | ❌ | | 21.* | ❌ | | 20.* | ❌ | | 19.* | ❌ | diff --git a/base/glibc-compatibility/glibc-compatibility.c b/base/glibc-compatibility/glibc-compatibility.c index 7e8ea5051d7..49bb81a58be 100644 --- a/base/glibc-compatibility/glibc-compatibility.c +++ b/base/glibc-compatibility/glibc-compatibility.c @@ -235,6 +235,17 @@ ssize_t getrandom(void *buf, size_t buflen, unsigned flags) return syscall(SYS_getrandom, buf, buflen, flags); } +/* Structure for scatter/gather I/O. */ +struct iovec +{ + void *iov_base; /* Pointer to data. */ + size_t iov_len; /* Length of data. */ +}; + +ssize_t preadv(int __fd, const struct iovec *__iovec, int __count, __off_t __offset) +{ + return syscall(SYS_preadv, __fd, __iovec, __count, (long)(__offset), (long)(__offset>>32)); +} #include #include diff --git a/base/harmful/harmful.c b/base/harmful/harmful.c index 6112f9a339c..78796ca0c05 100644 --- a/base/harmful/harmful.c +++ b/base/harmful/harmful.c @@ -31,7 +31,8 @@ TRAP(argp_state_help) TRAP(argp_usage) TRAP(asctime) TRAP(clearenv) -TRAP(crypt) +// Redefined at contrib/libbcrypt/crypt_blowfish/wrapper.c:186 +// TRAP(crypt) TRAP(ctime) TRAP(cuserid) TRAP(drand48) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 9bb148c12a9..462529fbc13 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54473) +SET(VERSION_REVISION 54474) SET(VERSION_MAJOR 23) -SET(VERSION_MINOR 4) +SET(VERSION_MINOR 5) SET(VERSION_PATCH 1) -SET(VERSION_GITHASH 46e85357ce2da2a99f56ee83a079e892d7ec3726) -SET(VERSION_DESCRIBE v23.4.1.1-testing) -SET(VERSION_STRING 23.4.1.1) +SET(VERSION_GITHASH 3920eb987f7ed837ada5de8907284adf123f0583) +SET(VERSION_DESCRIBE v23.5.1.1-testing) +SET(VERSION_STRING 23.5.1.1) # end of autochange diff --git a/cmake/fuzzer.cmake b/cmake/fuzzer.cmake index 578a9757270..52f301ab8ad 100644 --- a/cmake/fuzzer.cmake +++ b/cmake/fuzzer.cmake @@ -7,10 +7,6 @@ if (FUZZER) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=fuzzer-no-link") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=fuzzer-no-link") - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=fuzzer-no-link") - endif() - # NOTE: oss-fuzz can change LIB_FUZZING_ENGINE variable if (NOT LIB_FUZZING_ENGINE) set (LIB_FUZZING_ENGINE "-fsanitize=fuzzer") diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake index 13aaa414b93..bf5eddf09f5 100644 --- a/cmake/sanitize.cmake +++ b/cmake/sanitize.cmake @@ -16,50 +16,24 @@ if (SANITIZE) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${ASAN_FLAGS}") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${ASAN_FLAGS}") - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${ASAN_FLAGS}") - endif() - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libasan") - endif () - elseif (SANITIZE STREQUAL "memory") # MemorySanitizer flags are set according to the official documentation: # https://clang.llvm.org/docs/MemorySanitizer.html#usage - # - # For now, it compiles with `cmake -DSANITIZE=memory -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_CXX_FLAGS_ADD="-O1" -DCMAKE_C_FLAGS_ADD="-O1"` - # Compiling with -DCMAKE_BUILD_TYPE=Debug leads to ld.lld failures because - # of large files (was not tested with ld.gold). This is why we compile with - # RelWithDebInfo, and downgrade optimizations to -O1 but not to -Og, to - # keep the binary size down. - # TODO: try compiling with -Og and with ld.gold. - set (MSAN_FLAGS "-fsanitize=memory -fsanitize-memory-use-after-dtor -fsanitize-memory-track-origins -fno-optimize-sibling-calls -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/msan_suppressions.txt") + # Linking can fail due to relocation overflows (see #49145), caused by too big object files / libraries. + # Work around this with position-independent builds (-fPIC and -fpie), this is slightly slower than non-PIC/PIE but that's okay. + set (MSAN_FLAGS "-fsanitize=memory -fsanitize-memory-use-after-dtor -fsanitize-memory-track-origins -fno-optimize-sibling-calls -fPIC -fpie -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/msan_suppressions.txt") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${MSAN_FLAGS}") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${MSAN_FLAGS}") - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=memory") - endif() - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libmsan") - endif () - elseif (SANITIZE STREQUAL "thread") set (TSAN_FLAGS "-fsanitize=thread") if (COMPILER_CLANG) set (TSAN_FLAGS "${TSAN_FLAGS} -fsanitize-blacklist=${CMAKE_SOURCE_DIR}/tests/tsan_suppressions.txt") endif() - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${TSAN_FLAGS}") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${TSAN_FLAGS}") - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread") - endif() - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libtsan") - endif () elseif (SANITIZE STREQUAL "undefined") set (UBSAN_FLAGS "-fsanitize=undefined -fno-sanitize-recover=all -fno-sanitize=float-divide-by-zero") @@ -78,12 +52,6 @@ if (SANITIZE) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} ${UBSAN_FLAGS}") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} ${UBSAN_FLAGS}") - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined") - endif() - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libubsan") - endif () # llvm-tblgen, that is used during LLVM build, doesn't work with UBSan. set (ENABLE_EMBEDDED_COMPILER 0 CACHE BOOL "") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index be3563d2c61..0c92ff17f11 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -141,20 +141,19 @@ add_contrib (libuv-cmake libuv) add_contrib (liburing-cmake liburing) add_contrib (amqpcpp-cmake AMQP-CPP) # requires: libuv add_contrib (cassandra-cmake cassandra) # requires: libuv - -if (ENABLE_CURL_BUILD) +if (NOT OS_DARWIN) add_contrib (curl-cmake curl) add_contrib (azure-cmake azure) add_contrib (sentry-native-cmake sentry-native) # requires: curl endif() - add_contrib (fmtlib-cmake fmtlib) add_contrib (krb5-cmake krb5) add_contrib (cyrus-sasl-cmake cyrus-sasl) # for krb5 add_contrib (libgsasl-cmake libgsasl) # requires krb5 add_contrib (librdkafka-cmake librdkafka) # requires: libgsasl add_contrib (nats-io-cmake nats-io) -add_contrib (libhdfs3-cmake libhdfs3) # requires: protobuf, krb5 +add_contrib (isa-l-cmake isa-l) +add_contrib (libhdfs3-cmake libhdfs3) # requires: protobuf, krb5, isa-l add_contrib (hive-metastore-cmake hive-metastore) # requires: thrift/avro/arrow/libhdfs3 add_contrib (cppkafka-cmake cppkafka) add_contrib (libpqxx-cmake libpqxx) @@ -178,17 +177,15 @@ add_contrib (s2geometry-cmake s2geometry) add_contrib (c-ares-cmake c-ares) add_contrib (qpl-cmake qpl) add_contrib (morton-nd-cmake morton-nd) - if (ARCH_S390X) add_contrib(crc32-s390x-cmake crc32-s390x) endif() - add_contrib (annoy-cmake annoy) - add_contrib (xxHash-cmake xxHash) -add_contrib (google-benchmark-cmake google-benchmark) +add_contrib (libbcrypt-cmake libbcrypt) +add_contrib (google-benchmark-cmake google-benchmark) add_contrib (ulid-c-cmake ulid-c) # Put all targets defined here and in subdirectories under "contrib/" folders in GUI-based IDEs. diff --git a/contrib/curl b/contrib/curl index c12fb3ddaf4..b0edf0b7dae 160000 --- a/contrib/curl +++ b/contrib/curl @@ -1 +1 @@ -Subproject commit c12fb3ddaf48e709a7a4deaa55ec485e4df163ee +Subproject commit b0edf0b7dae44d9e66f270a257cf654b35d5263d diff --git a/contrib/curl-cmake/CMakeLists.txt b/contrib/curl-cmake/CMakeLists.txt index 8a570bd267c..70d9c2816dc 100644 --- a/contrib/curl-cmake/CMakeLists.txt +++ b/contrib/curl-cmake/CMakeLists.txt @@ -12,6 +12,9 @@ set (SRCS "${LIBRARY_DIR}/lib/noproxy.c" "${LIBRARY_DIR}/lib/idn.c" "${LIBRARY_DIR}/lib/cfilters.c" + "${LIBRARY_DIR}/lib/cf-socket.c" + "${LIBRARY_DIR}/lib/cf-haproxy.c" + "${LIBRARY_DIR}/lib/cf-https-connect.c" "${LIBRARY_DIR}/lib/file.c" "${LIBRARY_DIR}/lib/timeval.c" "${LIBRARY_DIR}/lib/base64.c" @@ -37,8 +40,8 @@ set (SRCS "${LIBRARY_DIR}/lib/strcase.c" "${LIBRARY_DIR}/lib/easy.c" "${LIBRARY_DIR}/lib/curl_fnmatch.c" + "${LIBRARY_DIR}/lib/curl_log.c" "${LIBRARY_DIR}/lib/fileinfo.c" - "${LIBRARY_DIR}/lib/wildcard.c" "${LIBRARY_DIR}/lib/krb5.c" "${LIBRARY_DIR}/lib/memdebug.c" "${LIBRARY_DIR}/lib/http_chunks.c" @@ -96,6 +99,7 @@ set (SRCS "${LIBRARY_DIR}/lib/rand.c" "${LIBRARY_DIR}/lib/curl_multibyte.c" "${LIBRARY_DIR}/lib/conncache.c" + "${LIBRARY_DIR}/lib/cf-h1-proxy.c" "${LIBRARY_DIR}/lib/http2.c" "${LIBRARY_DIR}/lib/smb.c" "${LIBRARY_DIR}/lib/curl_endian.c" @@ -113,12 +117,13 @@ set (SRCS "${LIBRARY_DIR}/lib/altsvc.c" "${LIBRARY_DIR}/lib/socketpair.c" "${LIBRARY_DIR}/lib/bufref.c" + "${LIBRARY_DIR}/lib/bufq.c" "${LIBRARY_DIR}/lib/dynbuf.c" + "${LIBRARY_DIR}/lib/dynhds.c" "${LIBRARY_DIR}/lib/hsts.c" "${LIBRARY_DIR}/lib/http_aws_sigv4.c" "${LIBRARY_DIR}/lib/mqtt.c" "${LIBRARY_DIR}/lib/rename.c" - "${LIBRARY_DIR}/lib/h2h3.c" "${LIBRARY_DIR}/lib/headers.c" "${LIBRARY_DIR}/lib/timediff.c" "${LIBRARY_DIR}/lib/vauth/vauth.c" @@ -133,6 +138,7 @@ set (SRCS "${LIBRARY_DIR}/lib/vauth/oauth2.c" "${LIBRARY_DIR}/lib/vauth/spnego_gssapi.c" "${LIBRARY_DIR}/lib/vauth/spnego_sspi.c" + "${LIBRARY_DIR}/lib/vquic/vquic.c" "${LIBRARY_DIR}/lib/vtls/openssl.c" "${LIBRARY_DIR}/lib/vtls/gtls.c" "${LIBRARY_DIR}/lib/vtls/vtls.c" @@ -147,9 +153,6 @@ set (SRCS "${LIBRARY_DIR}/lib/vtls/keylog.c" "${LIBRARY_DIR}/lib/vtls/x509asn1.c" "${LIBRARY_DIR}/lib/vtls/hostcheck.c" - "${LIBRARY_DIR}/lib/vquic/ngtcp2.c" - "${LIBRARY_DIR}/lib/vquic/quiche.c" - "${LIBRARY_DIR}/lib/vquic/msh3.c" "${LIBRARY_DIR}/lib/vssh/libssh2.c" "${LIBRARY_DIR}/lib/vssh/libssh.c" ) diff --git a/contrib/idxd-config b/contrib/idxd-config deleted file mode 160000 index f6605c41a73..00000000000 --- a/contrib/idxd-config +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f6605c41a735e3fdfef2d2d18655a33af6490b99 diff --git a/contrib/isa-l b/contrib/isa-l new file mode 160000 index 00000000000..9f2b68f0575 --- /dev/null +++ b/contrib/isa-l @@ -0,0 +1 @@ +Subproject commit 9f2b68f05752097f0f16632fc4a9a86950831efd diff --git a/contrib/isa-l-cmake/CMakeLists.txt b/contrib/isa-l-cmake/CMakeLists.txt new file mode 100644 index 00000000000..d4d6d648268 --- /dev/null +++ b/contrib/isa-l-cmake/CMakeLists.txt @@ -0,0 +1,203 @@ +option(ENABLE_ISAL_LIBRARY "Enable ISA-L library" ${ENABLE_LIBRARIES}) +if (ARCH_AARCH64) + # Disable ISA-L libray on aarch64. + set (ENABLE_ISAL_LIBRARY OFF) +endif () + +if (NOT ENABLE_ISAL_LIBRARY) + message(STATUS "Not using isa-l") + return() +endif() + +set(ISAL_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/isa-l") + +# The YASM and NASM assembers are somewhat mutually compatible. ISAL specifically needs NASM. If only YASM is installed, then check_language(ASM_NASM) +# below happily finds YASM, leading to weird errors at build time. Therefore, do an explicit check for NASM here. +find_program(NASM_PATH NAMES nasm) +if (NOT NASM_PATH) + message(FATAL_ERROR "Please install NASM from 'https://www.nasm.us/' because NASM compiler can not be found!") +endif () + +include(CheckLanguage) +check_language(ASM_NASM) +if(NOT CMAKE_ASM_NASM_COMPILER) + message(FATAL_ERROR "Please install NASM from 'https://www.nasm.us/' because NASM compiler can not be found!") +endif() + +enable_language(ASM_NASM) + +set(ISAL_C_SRC + ${ISAL_SOURCE_DIR}/crc/crc_base_aliases.c + ${ISAL_SOURCE_DIR}/crc/crc_base.c + ${ISAL_SOURCE_DIR}/crc/crc64_base.c + ${ISAL_SOURCE_DIR}/erasure_code/ec_base.c + ${ISAL_SOURCE_DIR}/erasure_code/ec_base_aliases.c + ${ISAL_SOURCE_DIR}/erasure_code/ec_highlevel_func.c + ${ISAL_SOURCE_DIR}/erasure_code/gen_rs_matrix_limits.c + ${ISAL_SOURCE_DIR}/erasure_code/gf_vect_dot_prod_1tbl.c + ${ISAL_SOURCE_DIR}/igzip/adler32_base.c + ${ISAL_SOURCE_DIR}/igzip/encode_df.c + ${ISAL_SOURCE_DIR}/igzip/flatten_ll.c + ${ISAL_SOURCE_DIR}/igzip/generate_custom_hufftables.c + ${ISAL_SOURCE_DIR}/igzip/generate_static_inflate.c + ${ISAL_SOURCE_DIR}/igzip/huff_codes.c + ${ISAL_SOURCE_DIR}/igzip/hufftables_c.c + ${ISAL_SOURCE_DIR}/igzip/igzip_base_aliases.c + ${ISAL_SOURCE_DIR}/igzip/igzip_base.c + ${ISAL_SOURCE_DIR}/igzip/igzip_icf_base.c + ${ISAL_SOURCE_DIR}/igzip/igzip_icf_body.c + ${ISAL_SOURCE_DIR}/igzip/igzip_inflate.c + ${ISAL_SOURCE_DIR}/igzip/igzip.c + ${ISAL_SOURCE_DIR}/mem/mem_zero_detect_base_aliases.c + ${ISAL_SOURCE_DIR}/mem/mem_zero_detect_base.c + ${ISAL_SOURCE_DIR}/programs/igzip_cli.c + ${ISAL_SOURCE_DIR}/raid/raid_base_aliases.c + ${ISAL_SOURCE_DIR}/raid/raid_base.c +) + +set(ISAL_ASM_SRC + ${ISAL_SOURCE_DIR}/crc/crc_multibinary.asm + ${ISAL_SOURCE_DIR}/crc/crc16_t10dif_01.asm + ${ISAL_SOURCE_DIR}/crc/crc16_t10dif_02.asm + ${ISAL_SOURCE_DIR}/crc/crc16_t10dif_by4.asm + ${ISAL_SOURCE_DIR}/crc/crc16_t10dif_by16_10.asm + ${ISAL_SOURCE_DIR}/crc/crc16_t10dif_copy_by4_02.asm + ${ISAL_SOURCE_DIR}/crc/crc16_t10dif_copy_by4.asm + ${ISAL_SOURCE_DIR}/crc/crc32_gzip_refl_by8_02.asm + ${ISAL_SOURCE_DIR}/crc/crc32_gzip_refl_by8.asm + ${ISAL_SOURCE_DIR}/crc/crc32_gzip_refl_by16_10.asm + ${ISAL_SOURCE_DIR}/crc/crc32_ieee_01.asm + ${ISAL_SOURCE_DIR}/crc/crc32_ieee_02.asm + ${ISAL_SOURCE_DIR}/crc/crc32_ieee_by4.asm + ${ISAL_SOURCE_DIR}/crc/crc32_ieee_by16_10.asm + ${ISAL_SOURCE_DIR}/crc/crc32_iscsi_00.asm + ${ISAL_SOURCE_DIR}/crc/crc32_iscsi_01.asm + ${ISAL_SOURCE_DIR}/crc/crc32_iscsi_by16_10.asm + ${ISAL_SOURCE_DIR}/crc/crc64_ecma_norm_by8.asm + ${ISAL_SOURCE_DIR}/crc/crc64_ecma_norm_by16_10.asm + ${ISAL_SOURCE_DIR}/crc/crc64_ecma_refl_by8.asm + ${ISAL_SOURCE_DIR}/crc/crc64_ecma_refl_by16_10.asm + ${ISAL_SOURCE_DIR}/crc/crc64_iso_norm_by8.asm + ${ISAL_SOURCE_DIR}/crc/crc64_iso_norm_by16_10.asm + ${ISAL_SOURCE_DIR}/crc/crc64_iso_refl_by8.asm + ${ISAL_SOURCE_DIR}/crc/crc64_iso_refl_by16_10.asm + ${ISAL_SOURCE_DIR}/crc/crc64_jones_norm_by8.asm + ${ISAL_SOURCE_DIR}/crc/crc64_jones_norm_by16_10.asm + ${ISAL_SOURCE_DIR}/crc/crc64_jones_refl_by8.asm + ${ISAL_SOURCE_DIR}/crc/crc64_jones_refl_by16_10.asm + ${ISAL_SOURCE_DIR}/crc/crc64_multibinary.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_2vect_dot_prod_avx.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_2vect_dot_prod_avx2.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_2vect_dot_prod_avx512.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_2vect_dot_prod_sse.asm + ${ISAL_SOURCE_DIR}/erasure_code/ec_multibinary.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_2vect_mad_avx.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_2vect_mad_avx2.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_2vect_mad_avx512.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_2vect_mad_sse.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_3vect_dot_prod_avx.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_3vect_dot_prod_avx2.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_3vect_dot_prod_avx512.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_3vect_dot_prod_sse.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_3vect_mad_avx.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_3vect_mad_avx2.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_3vect_mad_avx512.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_3vect_mad_sse.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_4vect_dot_prod_avx.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_4vect_dot_prod_avx2.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_4vect_dot_prod_avx512.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_4vect_dot_prod_sse.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_4vect_mad_avx.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_4vect_mad_avx2.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_4vect_mad_avx512.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_4vect_mad_sse.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_5vect_dot_prod_avx.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_5vect_dot_prod_avx2.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_5vect_dot_prod_avx512.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_5vect_dot_prod_sse.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_5vect_mad_avx.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_5vect_mad_avx2.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_5vect_mad_avx512.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_5vect_mad_sse.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_6vect_dot_prod_avx.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_6vect_dot_prod_avx2.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_6vect_dot_prod_avx512.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_6vect_dot_prod_sse.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_6vect_mad_avx.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_6vect_mad_avx2.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_6vect_mad_avx512.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_6vect_mad_sse.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_vect_dot_prod_avx.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_vect_dot_prod_avx2.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_vect_dot_prod_avx512.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_vect_dot_prod_sse.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_vect_mad_avx.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_vect_mad_avx2.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_vect_mad_avx512.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_vect_mad_sse.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_vect_mul_avx.asm + ${ISAL_SOURCE_DIR}/erasure_code/gf_vect_mul_sse.asm + ${ISAL_SOURCE_DIR}/igzip/adler32_avx2_4.asm + ${ISAL_SOURCE_DIR}/igzip/adler32_sse.asm + ${ISAL_SOURCE_DIR}/igzip/bitbuf2.asm + ${ISAL_SOURCE_DIR}/igzip/encode_df_04.asm + ${ISAL_SOURCE_DIR}/igzip/encode_df_06.asm + ${ISAL_SOURCE_DIR}/igzip/heap_macros.asm + ${ISAL_SOURCE_DIR}/igzip/huffman.asm + ${ISAL_SOURCE_DIR}/igzip/igzip_body.asm + ${ISAL_SOURCE_DIR}/igzip/igzip_compare_types.asm + ${ISAL_SOURCE_DIR}/igzip/igzip_decode_block_stateless_01.asm + ${ISAL_SOURCE_DIR}/igzip/igzip_decode_block_stateless_04.asm + ${ISAL_SOURCE_DIR}/igzip/igzip_deflate_hash.asm + ${ISAL_SOURCE_DIR}/igzip/igzip_finish.asm + ${ISAL_SOURCE_DIR}/igzip/igzip_gen_icf_map_lh1_04.asm + ${ISAL_SOURCE_DIR}/igzip/igzip_gen_icf_map_lh1_06.asm + ${ISAL_SOURCE_DIR}/igzip/igzip_icf_body_h1_gr_bt.asm + ${ISAL_SOURCE_DIR}/igzip/igzip_icf_finish.asm + ${ISAL_SOURCE_DIR}/igzip/igzip_inflate_multibinary.asm + ${ISAL_SOURCE_DIR}/igzip/igzip_multibinary.asm + ${ISAL_SOURCE_DIR}/igzip/igzip_set_long_icf_fg_04.asm + ${ISAL_SOURCE_DIR}/igzip/igzip_set_long_icf_fg_06.asm + ${ISAL_SOURCE_DIR}/igzip/igzip_update_histogram_01.asm + ${ISAL_SOURCE_DIR}/igzip/igzip_update_histogram_04.asm + ${ISAL_SOURCE_DIR}/igzip/lz0a_const.asm + ${ISAL_SOURCE_DIR}/igzip/options.asm + ${ISAL_SOURCE_DIR}/igzip/proc_heap.asm + ${ISAL_SOURCE_DIR}/igzip/rfc1951_lookup.asm + ${ISAL_SOURCE_DIR}/igzip/stdmac.asm + ${ISAL_SOURCE_DIR}/mem/mem_multibinary.asm + ${ISAL_SOURCE_DIR}/mem/mem_zero_detect_avx.asm + ${ISAL_SOURCE_DIR}/mem/mem_zero_detect_avx2.asm + ${ISAL_SOURCE_DIR}/mem/mem_zero_detect_avx512.asm + ${ISAL_SOURCE_DIR}/mem/mem_zero_detect_sse.asm + ${ISAL_SOURCE_DIR}/raid/pq_check_sse.asm + ${ISAL_SOURCE_DIR}/raid/pq_gen_avx.asm + ${ISAL_SOURCE_DIR}/raid/pq_gen_avx2.asm + ${ISAL_SOURCE_DIR}/raid/pq_gen_avx512.asm + ${ISAL_SOURCE_DIR}/raid/pq_gen_sse.asm + ${ISAL_SOURCE_DIR}/raid/raid_multibinary.asm + ${ISAL_SOURCE_DIR}/raid/xor_check_sse.asm + ${ISAL_SOURCE_DIR}/raid/xor_gen_avx.asm + ${ISAL_SOURCE_DIR}/raid/xor_gen_avx512.asm + ${ISAL_SOURCE_DIR}/raid/xor_gen_sse.asm +) + +# Adding ISA-L library target +add_library(_isal ${ISAL_C_SRC} ${ISAL_ASM_SRC}) + +# Setting external and internal interfaces for ISA-L library +target_include_directories(_isal + PUBLIC ${ISAL_SOURCE_DIR}/include + PUBLIC ${ISAL_SOURCE_DIR}/igzip + PUBLIC ${ISAL_SOURCE_DIR}/crc + PUBLIC ${ISAL_SOURCE_DIR}/erasure_code) + +# Here must remove "-fno-sanitize=undefined" from COMPILE_OPTIONS. +# Otherwise nasm compiler would fail to proceed due to unrecognition of "-fno-sanitize=undefined" +if (SANITIZE STREQUAL "undefined") + get_target_property(target_options _isal COMPILE_OPTIONS) + list(REMOVE_ITEM target_options "-fno-sanitize=undefined") + set_property(TARGET _isal PROPERTY COMPILE_OPTIONS ${target_options}) +endif() + +add_library(ch_contrib::isal ALIAS _isal) diff --git a/contrib/libbcrypt b/contrib/libbcrypt new file mode 160000 index 00000000000..8aa32ad94eb --- /dev/null +++ b/contrib/libbcrypt @@ -0,0 +1 @@ +Subproject commit 8aa32ad94ebe06b76853b0767c910c9fbf7ccef4 diff --git a/contrib/libbcrypt-cmake/CMakeLists.txt b/contrib/libbcrypt-cmake/CMakeLists.txt new file mode 100644 index 00000000000..d40d7f9195e --- /dev/null +++ b/contrib/libbcrypt-cmake/CMakeLists.txt @@ -0,0 +1,19 @@ +option(ENABLE_BCRYPT "Enable bcrypt" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_BCRYPT) + message(STATUS "Not using bcrypt") + return() +endif() + +set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/libbcrypt") + +set(SRCS + "${LIBRARY_DIR}/bcrypt.c" + "${LIBRARY_DIR}/crypt_blowfish/crypt_blowfish.c" + "${LIBRARY_DIR}/crypt_blowfish/crypt_gensalt.c" + "${LIBRARY_DIR}/crypt_blowfish/wrapper.c" +) + +add_library(_bcrypt ${SRCS}) +target_include_directories(_bcrypt SYSTEM PUBLIC "${LIBRARY_DIR}") +add_library(ch_contrib::bcrypt ALIAS _bcrypt) diff --git a/contrib/libhdfs3 b/contrib/libhdfs3 index 3c91d96ff29..164b89253fa 160000 --- a/contrib/libhdfs3 +++ b/contrib/libhdfs3 @@ -1 +1 @@ -Subproject commit 3c91d96ff29fe5928f055519c6d979c4b104db9e +Subproject commit 164b89253fad7991bce77882f01b51ab81d19f3d diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index c22cac731fe..fd9ed7dc182 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -70,6 +70,30 @@ set(SRCS "${HDFS3_SOURCE_DIR}/client/Token.cpp" "${HDFS3_SOURCE_DIR}/client/PacketPool.cpp" "${HDFS3_SOURCE_DIR}/client/OutputStream.cpp" + "${HDFS3_SOURCE_DIR}/client/AbstractNativeRawDecoder.cpp" + "${HDFS3_SOURCE_DIR}/client/AbstractNativeRawEncoder.cpp" + "${HDFS3_SOURCE_DIR}/client/ByteBufferDecodingState.cpp" + "${HDFS3_SOURCE_DIR}/client/ByteBufferEncodingState.cpp" + "${HDFS3_SOURCE_DIR}/client/CoderUtil.cpp" + "${HDFS3_SOURCE_DIR}/client/ECChunk.cpp" + "${HDFS3_SOURCE_DIR}/client/ErasureCoderOptions.cpp" + "${HDFS3_SOURCE_DIR}/client/GF256.cpp" + "${HDFS3_SOURCE_DIR}/client/GaloisField.cpp" + "${HDFS3_SOURCE_DIR}/client/NativeRSRawDecoder.cpp" + "${HDFS3_SOURCE_DIR}/client/NativeRSRawEncoder.cpp" + "${HDFS3_SOURCE_DIR}/client/Preconditions.cpp" + "${HDFS3_SOURCE_DIR}/client/RSUtil.cpp" + "${HDFS3_SOURCE_DIR}/client/RawErasureCoderFactory.cpp" + "${HDFS3_SOURCE_DIR}/client/RawErasureDecoder.cpp" + "${HDFS3_SOURCE_DIR}/client/RawErasureEncoder.cpp" + "${HDFS3_SOURCE_DIR}/client/StatefulStripeReader.cpp" + "${HDFS3_SOURCE_DIR}/client/StripeReader.cpp" + "${HDFS3_SOURCE_DIR}/client/StripedBlockUtil.cpp" + "${HDFS3_SOURCE_DIR}/client/StripedInputStreamImpl.cpp" + "${HDFS3_SOURCE_DIR}/client/StripedOutputStreamImpl.cpp" + "${HDFS3_SOURCE_DIR}/client/SystemECPolicies.cpp" + "${HDFS3_SOURCE_DIR}/client/dump.cpp" + "${HDFS3_SOURCE_DIR}/client/erasure_coder.cpp" "${HDFS3_SOURCE_DIR}/rpc/RpcChannelKey.cpp" "${HDFS3_SOURCE_DIR}/rpc/RpcProtocolInfo.cpp" "${HDFS3_SOURCE_DIR}/rpc/RpcClient.cpp" @@ -148,6 +172,11 @@ if (TARGET OpenSSL::SSL) target_link_libraries(_hdfs3 PRIVATE OpenSSL::Crypto OpenSSL::SSL) endif() +if (TARGET ch_contrib::isal) + target_link_libraries(_hdfs3 PRIVATE ch_contrib::isal) + add_definitions(-DHADOOP_ISAL_LIBRARY) +endif() + add_library(ch_contrib::hdfs ALIAS _hdfs3) if (ENABLE_CLICKHOUSE_BENCHMARK) diff --git a/contrib/qpl b/contrib/qpl index d75a29d95d8..0bce2b03423 160000 --- a/contrib/qpl +++ b/contrib/qpl @@ -1 +1 @@ -Subproject commit d75a29d95d8a548297fce3549d21020005364dc8 +Subproject commit 0bce2b03423f6fbeb8bce66cc8be0bf558058848 diff --git a/contrib/qpl-cmake/CMakeLists.txt b/contrib/qpl-cmake/CMakeLists.txt index fc5548b0652..334731d105f 100644 --- a/contrib/qpl-cmake/CMakeLists.txt +++ b/contrib/qpl-cmake/CMakeLists.txt @@ -40,9 +40,10 @@ set (LOG_HW_INIT OFF) set (SANITIZE_MEMORY OFF) set (SANITIZE_THREADS OFF) set (LIB_FUZZING_ENGINE OFF) +set (DYNAMIC_LOADING_LIBACCEL_CONFIG OFF) function(GetLibraryVersion _content _outputVar) - string(REGEX MATCHALL "Qpl VERSION (.+) LANGUAGES" VERSION_REGEX "${_content}") + string(REGEX MATCHALL "QPL VERSION (.+) LANGUAGES" VERSION_REGEX "${_content}") SET(${_outputVar} ${CMAKE_MATCH_1} PARENT_SCOPE) endfunction() @@ -240,7 +241,9 @@ add_library(core_iaa OBJECT ${HW_PATH_SRC}) target_include_directories(core_iaa PRIVATE ${UUID_DIR} PUBLIC $ - PRIVATE $ + PUBLIC $ + PRIVATE $ # status.h in own_checkers.h + PRIVATE $ # own_checkers.h PRIVATE $) target_compile_options(core_iaa @@ -339,4 +342,7 @@ target_link_libraries(_qpl PRIVATE ${CMAKE_DL_LIBS}) add_library (ch_contrib::qpl ALIAS _qpl) -target_include_directories(_qpl SYSTEM BEFORE PUBLIC "${QPL_PROJECT_DIR}/include") +target_include_directories(_qpl SYSTEM BEFORE + PUBLIC "${QPL_PROJECT_DIR}/include" + PUBLIC "${LIBACCEL_SOURCE_DIR}/accfg" + PUBLIC ${UUID_DIR}) diff --git a/contrib/sysroot b/contrib/sysroot index f0081b2649b..e0d1b64da66 160000 --- a/contrib/sysroot +++ b/contrib/sysroot @@ -1 +1 @@ -Subproject commit f0081b2649b94837855f3bc7d05ef326b100bad8 +Subproject commit e0d1b64da666afbfaa6f1ee0487c33f3fd2cd5cb diff --git a/docker/images.json b/docker/images.json index 9150abe1f1c..b4f3e755bd1 100644 --- a/docker/images.json +++ b/docker/images.json @@ -123,7 +123,8 @@ "docker/test/stateless", "docker/test/integration/base", "docker/test/fuzzer", - "docker/test/keeper-jepsen" + "docker/test/keeper-jepsen", + "docker/test/server-jepsen" ] }, "docker/test/integration/kerberized_hadoop": { @@ -139,6 +140,10 @@ "name": "clickhouse/keeper-jepsen-test", "dependent": [] }, + "docker/test/server-jepsen": { + "name": "clickhouse/server-jepsen-test", + "dependent": [] + }, "docker/test/install/deb": { "name": "clickhouse/install-deb-test", "dependent": [] diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index 35b94d2563e..73da4515ff4 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \ esac ARG REPOSITORY="https://s3.amazonaws.com/clickhouse-builds/22.4/31c367d3cd3aefd316778601ff6565119fe36682/package_release" -ARG VERSION="23.3.2.37" +ARG VERSION="23.4.2.11" ARG PACKAGES="clickhouse-keeper" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index a20feaf654f..1a5d2071f6b 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -33,7 +33,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="23.3.2.37" +ARG VERSION="23.4.2.11" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 4851e2b1fc7..8792d419a16 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -22,7 +22,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="23.3.2.37" +ARG VERSION="23.4.2.11" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docker/test/server-jepsen/Dockerfile b/docker/test/server-jepsen/Dockerfile index 958dbfa066a..a212427b2a1 100644 --- a/docker/test/server-jepsen/Dockerfile +++ b/docker/test/server-jepsen/Dockerfile @@ -16,6 +16,11 @@ ENV TESTS_TO_RUN="8" ENV TIME_LIMIT="30" ENV KEEPER_NODE="" +ENV NEMESIS="" +ENV WORKLOAD="" +ENV WITH_LOCAL_BINARY="" +ENV RATE="" +ENV CONCURRENCY="" # volumes diff --git a/docker/test/server-jepsen/run.sh b/docker/test/server-jepsen/run.sh index 4a966d50f74..c11a48f6d4c 100644 --- a/docker/test/server-jepsen/run.sh +++ b/docker/test/server-jepsen/run.sh @@ -15,8 +15,38 @@ if [ -z "$CLICKHOUSE_REPO_PATH" ]; then ls -lath ||: fi +clickhouse_source="--clickhouse-source $CLICKHOUSE_PACKAGE" +if [ -n "$WITH_LOCAL_BINARY" ]; then + clickhouse_source="--clickhouse-source /clickhouse" +fi + +tests_count="--test-count $TESTS_TO_RUN" +tests_to_run="test-all" +workload="" +if [ -n "$WORKLOAD" ]; then + tests_to_run="test" + workload="--workload $WORKLOAD" + tests_count="" +fi + +nemesis="" +if [ -n "$NEMESIS" ]; then + nemesis="--nemesis $NEMESIS" +fi + +rate="" +if [ -n "$RATE" ]; then + rate="--rate $RATE" +fi + +concurrency="" +if [ -n "$CONCURRENCY" ]; then + concurrency="--concurrency $CONCURRENCY" +fi + + cd "$CLICKHOUSE_REPO_PATH/tests/jepsen.clickhouse" -(lein run server test-all --keeper "$KEEPER_NODE" --nodes-file "$NODES_FILE_PATH" --username "$NODES_USERNAME" --logging-json --password "$NODES_PASSWORD" --time-limit "$TIME_LIMIT" --concurrency 50 -r 50 --clickhouse-source "$CLICKHOUSE_PACKAGE" --test-count "$TESTS_TO_RUN" || true) | tee "$TEST_OUTPUT/jepsen_run_all_tests.log" +(lein run server $tests_to_run $workload --keeper "$KEEPER_NODE" $concurrency $nemesis $rate --nodes-file "$NODES_FILE_PATH" --username "$NODES_USERNAME" --logging-json --password "$NODES_PASSWORD" --time-limit "$TIME_LIMIT" --concurrency 50 $clickhouse_source $tests_count --reuse-binary || true) | tee "$TEST_OUTPUT/jepsen_run_all_tests.log" mv store "$TEST_OUTPUT/" diff --git a/docker/test/upgrade/run.sh b/docker/test/upgrade/run.sh index fce90ca2537..10ba597a33a 100644 --- a/docker/test/upgrade/run.sh +++ b/docker/test/upgrade/run.sh @@ -59,6 +59,12 @@ install_packages previous_release_package_folder # available for dump via clickhouse-local configure +# local_blob_storage disk type does not exist in older versions +sudo cat /etc/clickhouse-server/config.d/storage_conf.xml \ + | sed "s|local_blob_storage|local|" \ + > /etc/clickhouse-server/config.d/storage_conf.xml.tmp +sudo mv /etc/clickhouse-server/config.d/storage_conf.xml.tmp /etc/clickhouse-server/config.d/storage_conf.xml + start stop mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.initial.log @@ -83,6 +89,11 @@ export USE_S3_STORAGE_FOR_MERGE_TREE=1 export ZOOKEEPER_FAULT_INJECTION=0 configure +sudo cat /etc/clickhouse-server/config.d/storage_conf.xml \ + | sed "s|local_blob_storage|local|" \ + > /etc/clickhouse-server/config.d/storage_conf.xml.tmp +sudo mv /etc/clickhouse-server/config.d/storage_conf.xml.tmp /etc/clickhouse-server/config.d/storage_conf.xml + start clickhouse-client --query="SELECT 'Server version: ', version()" diff --git a/docker/test/util/process_functional_tests_result.py b/docker/test/util/process_functional_tests_result.py index da58db8e45d..470eb61b3fa 100755 --- a/docker/test/util/process_functional_tests_result.py +++ b/docker/test/util/process_functional_tests_result.py @@ -18,7 +18,7 @@ SUCCESS_FINISH_SIGNS = ["All tests have finished", "No tests were run"] RETRIES_SIGN = "Some tests were restarted" -def process_test_log(log_path): +def process_test_log(log_path, broken_tests): total = 0 skipped = 0 unknown = 0 @@ -62,8 +62,12 @@ def process_test_log(log_path): failed += 1 test_results.append((test_name, "Timeout", test_time, [])) elif FAIL_SIGN in line: - failed += 1 - test_results.append((test_name, "FAIL", test_time, [])) + if test_name in broken_tests: + success += 1 + test_results.append((test_name, "OK", test_time, [])) + else: + failed += 1 + test_results.append((test_name, "FAIL", test_time, [])) elif UNKNOWN_SIGN in line: unknown += 1 test_results.append((test_name, "FAIL", test_time, [])) @@ -71,8 +75,19 @@ def process_test_log(log_path): skipped += 1 test_results.append((test_name, "SKIPPED", test_time, [])) else: - success += int(OK_SIGN in line) - test_results.append((test_name, "OK", test_time, [])) + if OK_SIGN in line and test_name in broken_tests: + failed += 1 + test_results.append( + ( + test_name, + "SKIPPED", + test_time, + ["This test passed. Update broken_tests.txt.\n"], + ) + ) + else: + success += int(OK_SIGN in line) + test_results.append((test_name, "OK", test_time, [])) test_end = False elif ( len(test_results) > 0 and test_results[-1][1] == "FAIL" and not test_end @@ -110,7 +125,7 @@ def process_test_log(log_path): ) -def process_result(result_path): +def process_result(result_path, broken_tests): test_results = [] state = "success" description = "" @@ -134,7 +149,7 @@ def process_result(result_path): success_finish, retries, test_results, - ) = process_test_log(result_path) + ) = process_test_log(result_path, broken_tests) is_flacky_check = 1 < int(os.environ.get("NUM_TRIES", 1)) logging.info("Is flaky check: %s", is_flacky_check) # If no tests were run (success == 0) it indicates an error (e.g. server did not start or crashed immediately) @@ -186,9 +201,17 @@ if __name__ == "__main__": parser.add_argument("--in-results-dir", default="/test_output/") parser.add_argument("--out-results-file", default="/test_output/test_results.tsv") parser.add_argument("--out-status-file", default="/test_output/check_status.tsv") + parser.add_argument("--broken-tests", default="/broken_tests.txt") args = parser.parse_args() - state, description, test_results = process_result(args.in_results_dir) + broken_tests = list() + if os.path.exists(args.broken_tests): + logging.info(f"File {args.broken_tests} with broken tests found") + with open(args.broken_tests) as f: + broken_tests = f.read().splitlines() + logging.info(f"Broken tests in the list: {len(broken_tests)}") + + state, description, test_results = process_result(args.in_results_dir, broken_tests) logging.info("Result parsed") status = (state, description) write_results(args.out_results_file, args.out_status_file, test_results, status) diff --git a/docs/changelogs/v23.4.1.1943-stable.md b/docs/changelogs/v23.4.1.1943-stable.md new file mode 100644 index 00000000000..ea16f5856be --- /dev/null +++ b/docs/changelogs/v23.4.1.1943-stable.md @@ -0,0 +1,375 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v23.4.1.1943-stable (3920eb987f7) FIXME as compared to v23.3.1.2823-lts (46e85357ce2) + +#### Backward Incompatible Change +* If `path` in cache configuration is not empty and is not absolute path, then it will be put in `/caches/`. [#48784](https://github.com/ClickHouse/ClickHouse/pull/48784) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Compatibility setting `parallelize_output_from_storages` to enable behavior before [#48727](https://github.com/ClickHouse/ClickHouse/issues/48727). [#49101](https://github.com/ClickHouse/ClickHouse/pull/49101) ([Igor Nikonov](https://github.com/devcrafter)). + +#### New Feature +* Add `extractKeyValuePairs` function to extract key value pairs from strings. Input strings might contain noise (i.e log files / do not need to be 100% formatted in key-value-pair format), the algorithm will look for key value pairs matching the arguments passed to the function. As of now, function accepts the following arguments: `data_column` (mandatory), `key_value_pair_delimiter` (defaults to `:`), `pair_delimiters` (defaults to `\space \, \;`) and `quoting_character` (defaults to double quotes). [#43606](https://github.com/ClickHouse/ClickHouse/pull/43606) ([Arthur Passos](https://github.com/arthurpassos)). +* Add MemoryTracker for the background tasks (merges and mutation). Introduces `merges_mutations_memory_usage_soft_limit` and `merges_mutations_memory_usage_to_ram_ratio` settings that represent the soft memory limit for merges and mutations. If this limit is reached ClickHouse won't schedule new merge or mutation tasks. Also `MergesMutationsMemoryTracking` metric is introduced to allow observing current memory usage of background tasks. Closes [#45710](https://github.com/ClickHouse/ClickHouse/issues/45710). [#46089](https://github.com/ClickHouse/ClickHouse/pull/46089) ([Dmitry Novik](https://github.com/novikd)). +* Support new aggregate function quantileGK/quantilesGK, like [approx_percentile](https://spark.apache.org/docs/latest/api/sql/index.html#approx_percentile) in spark. Greenwald-Khanna algorithm refer to http://infolab.stanford.edu/~datar/courses/cs361a/papers/quantiles.pdf. [#46428](https://github.com/ClickHouse/ClickHouse/pull/46428) ([李扬](https://github.com/taiyang-li)). +* Add statement `SHOW COLUMNS` which shows distilled information from system.columns. [#48017](https://github.com/ClickHouse/ClickHouse/pull/48017) ([Robert Schulze](https://github.com/rschu1ze)). +* Added `LIGHTWEIGHT` and `PULL` modifiers for `SYSTEM SYNC REPLICA` query. `LIGHTWEIGHT` version waits for fetches and drop-ranges only (merges and mutations are ignored). `PULL` version pulls new entries from ZooKeeper and does not wait for them. Fixes [#47794](https://github.com/ClickHouse/ClickHouse/issues/47794). [#48085](https://github.com/ClickHouse/ClickHouse/pull/48085) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add kafkaMurmurHash function for compatibility with Kafka DefaultPartitioner. Closes [#47834](https://github.com/ClickHouse/ClickHouse/issues/47834). [#48185](https://github.com/ClickHouse/ClickHouse/pull/48185) ([Nikolay Degterinsky](https://github.com/evillique)). +* Allow to easily create a user with the same grants as the current user by using `GRANT CURRENT GRANTS`. [#48262](https://github.com/ClickHouse/ClickHouse/pull/48262) ([pufit](https://github.com/pufit)). +* Add statistical aggregate function `kolmogorovSmirnovTest`. close [#48228](https://github.com/ClickHouse/ClickHouse/issues/48228). [#48325](https://github.com/ClickHouse/ClickHouse/pull/48325) ([FFFFFFFHHHHHHH](https://github.com/FFFFFFFHHHHHHH)). +* Added a `lost_part_count` column to the `system.replicas` table. The column value shows the total number of lost parts in the corresponding table. Value is stored in zookeeper and can be used instead of not persistent `ReplicatedDataLoss` profile event for monitoring. [#48526](https://github.com/ClickHouse/ClickHouse/pull/48526) ([Sergei Trifonov](https://github.com/serxa)). +* Add soundex function. Closes [#39880](https://github.com/ClickHouse/ClickHouse/issues/39880). [#48567](https://github.com/ClickHouse/ClickHouse/pull/48567) ([FriendLey](https://github.com/FriendLey)). +* Support map type for JSONExtract. [#48629](https://github.com/ClickHouse/ClickHouse/pull/48629) ([李扬](https://github.com/taiyang-li)). +* Add PrettyJSONEachRow format to output pretty JSON with new line delimieters and 4 space indents. [#48898](https://github.com/ClickHouse/ClickHouse/pull/48898) ([Kruglov Pavel](https://github.com/Avogar)). +* Add ParquetMetadata input format to read Parquet file metadata. [#48911](https://github.com/ClickHouse/ClickHouse/pull/48911) ([Kruglov Pavel](https://github.com/Avogar)). + +#### Performance Improvement +* Reading files in Parquet format is now much faster. IO and decoding are parallelized (controlled by `max_threads` setting), and only required data ranges are read. [#47964](https://github.com/ClickHouse/ClickHouse/pull/47964) ([Michael Kolupaev](https://github.com/al13n321)). +* Only check dependencies if necessary when applying `ALTER TABLE` queries. [#48062](https://github.com/ClickHouse/ClickHouse/pull/48062) ([Raúl Marín](https://github.com/Algunenano)). +* Optimize function `mapUpdate`. [#48118](https://github.com/ClickHouse/ClickHouse/pull/48118) ([Anton Popov](https://github.com/CurtizJ)). +* Now an internal query to local replica is sent explicitly and data from it received through loopback interface. Setting `prefer_localhost_replica` is not respected for parallel replicas. This is needed for better scheduling and makes the code cleaner: the initiator is only responsible for coordinating of the reading process and merging results, continiously answering for requests while all the secondary queries read the data. Note: Using loopback interface is not so performant, otherwise some replicas could starve for tasks which could lead to even slower query execution and not utilizing all possible resources. The initialization of the coordinator is now even more lazy. All incoming requests contain the information about the reading algorithm we initialize the coordinator with it when first request comes. If any replica will decide to read with different algorithm - an exception will be thrown and a query will be aborted. [#48246](https://github.com/ClickHouse/ClickHouse/pull/48246) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Do not build set for the right side of `IN` clause with subquery when it is used only for analysis of skip indexes and they are disabled by setting (`use_skip_indexes=0`). Previously it might affect the performance of queries. [#48299](https://github.com/ClickHouse/ClickHouse/pull/48299) ([Anton Popov](https://github.com/CurtizJ)). +* Query processing is parallelized right after reading `FROM file(...)`. Related to [#38755](https://github.com/ClickHouse/ClickHouse/issues/38755). [#48525](https://github.com/ClickHouse/ClickHouse/pull/48525) ([Igor Nikonov](https://github.com/devcrafter)). +* Query processing is parallelized right after reading from a data source. Affected data sources are mostly simple or external storages like table functions `url`, `file`. [#48727](https://github.com/ClickHouse/ClickHouse/pull/48727) ([Igor Nikonov](https://github.com/devcrafter)). +* Using correct memory order for counter in `numebers_mt()`. [#48729](https://github.com/ClickHouse/ClickHouse/pull/48729) ([Igor Nikonov](https://github.com/devcrafter)). +* Lowered contention of ThreadPool mutex (may increase performance for a huge amount of small jobs). [#48750](https://github.com/ClickHouse/ClickHouse/pull/48750) ([Sergei Trifonov](https://github.com/serxa)). +* Simplify accounting of approximate size of granule in prefetched read pool. [#49051](https://github.com/ClickHouse/ClickHouse/pull/49051) ([Nikita Taranov](https://github.com/nickitat)). + +#### Improvement +* Support config sections `keeper`/`keeper_server` as an alternative to `zookeeper`. Close [#34766](https://github.com/ClickHouse/ClickHouse/issues/34766) , [#34767](https://github.com/ClickHouse/ClickHouse/issues/34767). [#35113](https://github.com/ClickHouse/ClickHouse/pull/35113) ([李扬](https://github.com/taiyang-li)). +* Many issues in ClickHouse applications's help were fixed. Help is now written to stdout from all tools. Status code for `clickhouse help` invocation is now 0. Updated help for `clickhouse-local`, `clickhouse-benchmark`, `clickhouse-client`, `clickhouse hash`, `clickhouse su`, `clickhouse-install`. [#45819](https://github.com/ClickHouse/ClickHouse/pull/45819) ([Ilya Yatsishin](https://github.com/qoega)). +* Entries in the query cache are now squashed to max_block_size and compressed. [#45912](https://github.com/ClickHouse/ClickHouse/pull/45912) ([Robert Schulze](https://github.com/rschu1ze)). +* It is possible to set _secure_ flag in named_collections for a dictionary with a ClickHouse table source. Addresses [#38450](https://github.com/ClickHouse/ClickHouse/issues/38450) . [#46323](https://github.com/ClickHouse/ClickHouse/pull/46323) ([Ilya Golshtein](https://github.com/ilejn)). +* Functions replaceOne(), replaceAll(), replaceRegexpOne() and replaceRegexpAll() can now be called with non-const pattern and replacement arguments. [#46589](https://github.com/ClickHouse/ClickHouse/pull/46589) ([Robert Schulze](https://github.com/rschu1ze)). +* Bump internal ZSTD from 1.5.4 to 1.5.5. [#46797](https://github.com/ClickHouse/ClickHouse/pull/46797) ([Robert Schulze](https://github.com/rschu1ze)). +* If we run a mutation with IN (subquery) like this: `ALTER TABLE t UPDATE col='new value' WHERE id IN (SELECT id FROM huge_table)` and the table `t` has multiple parts than for each part a set for subquery `SELECT id FROM huge_table` is built in memory. And if there are many parts then this might consume a lot of memory (and lead to an OOM) and CPU. The solution is to introduce a short-lived cache of sets that are currently being built by mutation tasks. If another task of the same mutation is executed concurrently it can lookup the set in the cache, wait for it be be built and reuse it. [#46835](https://github.com/ClickHouse/ClickHouse/pull/46835) ([Alexander Gololobov](https://github.com/davenger)). +* Added configurable retries for all operations with [Zoo]Keeper for Backup queries. [#47224](https://github.com/ClickHouse/ClickHouse/pull/47224) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add async connection to socket and async writing to socket. Make creating connections and sending query/external tables async across shards. Refactor code with fibers. Closes [#46931](https://github.com/ClickHouse/ClickHouse/issues/46931). We will be able to increase `connect_timeout_with_failover_ms` by default after this PR (https://github.com/ClickHouse/ClickHouse/issues/5188). [#47229](https://github.com/ClickHouse/ClickHouse/pull/47229) ([Kruglov Pavel](https://github.com/Avogar)). +* Formatter '%M' in function formatDateTime() now prints the month name instead of the minutes. This makes the behavior consistent with MySQL. The previous behavior can be restored using setting "formatdatetime_parsedatetime_m_is_month_name = 0". [#47246](https://github.com/ClickHouse/ClickHouse/pull/47246) ([Robert Schulze](https://github.com/rschu1ze)). +* Several improvements around data lakes: - Make StorageIceberg work with non-partitioned data. - Support Iceberg format version V2 (previously only V1 was supported) - Support reading partitioned data for DeltaLake/Hudi - Faster reading of DeltaLake metadata by using Delta's checkpoint files - Fixed incorrect Hudi reads: previously it incorrectly chose which data to read and therefore was able to read correctly only small size tables - Made these engines to pickup updates of changed data (previously the state was set on table creation) - Make proper testing for Iceberg/DeltaLake/Hudi using spark. [#47307](https://github.com/ClickHouse/ClickHouse/pull/47307) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Enable `use_environment_credentials` for S3 by default, so the entire provider chain is constructed by default. [#47397](https://github.com/ClickHouse/ClickHouse/pull/47397) ([Antonio Andelic](https://github.com/antonio2368)). +* Currently, the JSON_VALUE function is similar as spark's get_json_object function, which support to get value from json string by a path like '$.key'. But still has something different - 1. in spark's get_json_object will return null while the path is not exist, but in JSON_VALUE will return empty string; - 2. in spark's get_json_object will return a complext type value, such as a json object/array value, but in JSON_VALUE will return empty string. [#47494](https://github.com/ClickHouse/ClickHouse/pull/47494) ([KevinyhZou](https://github.com/KevinyhZou)). +* Add CNF/constraint optimizer in new analyzer. [#47617](https://github.com/ClickHouse/ClickHouse/pull/47617) ([Antonio Andelic](https://github.com/antonio2368)). +* For use_structure_from_insertion_table_in_table_functions more flexible insert table structure propagation to table function. Fixed bug with name mapping and using virtual columns. No more need for 'auto' setting. [#47962](https://github.com/ClickHouse/ClickHouse/pull/47962) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Do not continue retrying to connect to ZK if the query is killed or over limits. [#47985](https://github.com/ClickHouse/ClickHouse/pull/47985) ([Raúl Marín](https://github.com/Algunenano)). +* Added functions to work with columns of type `Map`: `mapConcat`, `mapSort`, `mapExists`. [#48071](https://github.com/ClickHouse/ClickHouse/pull/48071) ([Anton Popov](https://github.com/CurtizJ)). +* Support Enum output/input in BSONEachRow, allow all map key types and avoid extra calculations on output. [#48122](https://github.com/ClickHouse/ClickHouse/pull/48122) ([Kruglov Pavel](https://github.com/Avogar)). +* Support more ClickHouse types in ORC/Arrow/Parquet formats: Enum(8|16), (U)Int(128|256), Decimal256 (for ORC), allow reading IPv4 from Int32 values (ORC outputs IPv4 as Int32 and we couldn't read it back), fix reading Nullable(IPv6) from binary data for ORC. [#48126](https://github.com/ClickHouse/ClickHouse/pull/48126) ([Kruglov Pavel](https://github.com/Avogar)). +* Add columns `perform_ttl_move_on_insert`, `load_balancing` for table `system.storage_policies`, modify column `volume_type` type to `enum8`. [#48167](https://github.com/ClickHouse/ClickHouse/pull/48167) ([lizhuoyu5](https://github.com/lzydmxy)). +* Added support for `BACKUP ALL` command which backups all tables and databases, including temporary and system ones. [#48189](https://github.com/ClickHouse/ClickHouse/pull/48189) ([Vitaly Baranov](https://github.com/vitlibar)). +* Function mapFromArrays support map type as input. [#48207](https://github.com/ClickHouse/ClickHouse/pull/48207) ([李扬](https://github.com/taiyang-li)). +* The output of some SHOW PROCESSLIST is now sorted. [#48241](https://github.com/ClickHouse/ClickHouse/pull/48241) ([Robert Schulze](https://github.com/rschu1ze)). +* Per-query/per-server throttling for remote IO/local IO/BACKUPs (server settings: `max_remote_read_network_bandwidth_for_server`, `max_remote_write_network_bandwidth_for_server`, `max_local_read_bandwidth_for_server`, `max_local_write_bandwidth_for_server`, `max_backup_bandwidth_for_server`, settings: `max_remote_read_network_bandwidth`, `max_remote_write_network_bandwidth`, `max_local_read_bandwidth`, `max_local_write_bandwidth`, `max_backup_bandwidth`). [#48242](https://github.com/ClickHouse/ClickHouse/pull/48242) ([Azat Khuzhin](https://github.com/azat)). +* Support more types in CapnProto format: Map, (U)Int(128|256), Decimal(128|256). Allow integer conversions during input/output. [#48257](https://github.com/ClickHouse/ClickHouse/pull/48257) ([Kruglov Pavel](https://github.com/Avogar)). +* It is now possible to define per-user quotas in the query cache. [#48284](https://github.com/ClickHouse/ClickHouse/pull/48284) ([Robert Schulze](https://github.com/rschu1ze)). +* Don't throw CURRENT_WRITE_BUFFER_IS_EXHAUSTED for normal behaviour. [#48288](https://github.com/ClickHouse/ClickHouse/pull/48288) ([Raúl Marín](https://github.com/Algunenano)). +* Add new setting `keeper_map_strict_mode` which enforces extra guarantees on operations made on top of `KeeperMap` tables. [#48293](https://github.com/ClickHouse/ClickHouse/pull/48293) ([Antonio Andelic](https://github.com/antonio2368)). +* Check primary key type for simple dictionary is native unsigned integer type Add setting `check_dictionary_primary_key ` for compatibility(set `check_dictionary_primary_key =false` to disable checking). [#48335](https://github.com/ClickHouse/ClickHouse/pull/48335) ([lizhuoyu5](https://github.com/lzydmxy)). +* Don't replicate mutations for `KeeperMap` because it's unnecessary. [#48354](https://github.com/ClickHouse/ClickHouse/pull/48354) ([Antonio Andelic](https://github.com/antonio2368)). +* Allow write/read unnamed tuple as nested Message in Protobuf format. Tuple elements and Message fields are mathced by position. [#48390](https://github.com/ClickHouse/ClickHouse/pull/48390) ([Kruglov Pavel](https://github.com/Avogar)). +* Support `additional_table_filters` and `additional_result_filter` settings in the new planner. Also, add a documentation entry for `additional_result_filter`. [#48405](https://github.com/ClickHouse/ClickHouse/pull/48405) ([Dmitry Novik](https://github.com/novikd)). +* Parsedatetime now understands format string '%f' (fractional seconds). [#48420](https://github.com/ClickHouse/ClickHouse/pull/48420) ([Robert Schulze](https://github.com/rschu1ze)). +* Format string "%f" in formatDateTime() now prints "000000" if the formatted value has no fractional seconds, the previous behavior (single zero) can be restored using setting "formatdatetime_f_prints_single_zero = 1". [#48422](https://github.com/ClickHouse/ClickHouse/pull/48422) ([Robert Schulze](https://github.com/rschu1ze)). +* Don't replicate DELETE and TRUNCATE for KeeperMap. [#48434](https://github.com/ClickHouse/ClickHouse/pull/48434) ([Antonio Andelic](https://github.com/antonio2368)). +* Generate valid Decimals and Bools in generateRandom function. [#48436](https://github.com/ClickHouse/ClickHouse/pull/48436) ([Kruglov Pavel](https://github.com/Avogar)). +* Allow trailing commas in expression list of SELECT query, for example `SELECT a, b, c, FROM table`. Closes [#37802](https://github.com/ClickHouse/ClickHouse/issues/37802). [#48438](https://github.com/ClickHouse/ClickHouse/pull/48438) ([Nikolay Degterinsky](https://github.com/evillique)). +* Override `CLICKHOUSE_USER` and `CLICKHOUSE_PASSWORD` environment variables with `--user` and `--password` client parameters. Closes [#38909](https://github.com/ClickHouse/ClickHouse/issues/38909). [#48440](https://github.com/ClickHouse/ClickHouse/pull/48440) ([Nikolay Degterinsky](https://github.com/evillique)). +* Added retries to loading of data parts in `MergeTree` tables in case of retryable errors. [#48442](https://github.com/ClickHouse/ClickHouse/pull/48442) ([Anton Popov](https://github.com/CurtizJ)). +* Add support for `Date`, `Date32`, `DateTime`, `DateTime64` data types to `arrayMin`, `arrayMax`, `arrayDifference` functions. Closes [#21645](https://github.com/ClickHouse/ClickHouse/issues/21645). [#48445](https://github.com/ClickHouse/ClickHouse/pull/48445) ([Nikolay Degterinsky](https://github.com/evillique)). +* Reduce memory usage for multiple `ALTER DELETE` mutations. [#48522](https://github.com/ClickHouse/ClickHouse/pull/48522) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Primary/secondary indices and sorting keys with identical expressions are now rejected. This behavior can be disabled using setting `allow_suspicious_indices`. [#48536](https://github.com/ClickHouse/ClickHouse/pull/48536) ([凌涛](https://github.com/lingtaolf)). +* Just fix small typo in comment around `lockForAlter` method in `IStorage.h`. [#48559](https://github.com/ClickHouse/ClickHouse/pull/48559) ([artem-pershin](https://github.com/artem-pershin)). +* Add support for `{server_uuid}` macro. It is useful for identifying replicas in autoscaled clusters when new replicas are constantly added and removed in runtime. This closes [#48554](https://github.com/ClickHouse/ClickHouse/issues/48554). [#48563](https://github.com/ClickHouse/ClickHouse/pull/48563) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The installation script will create a hard link instead of copying if it is possible. [#48578](https://github.com/ClickHouse/ClickHouse/pull/48578) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support `SHOW TABLE` syntax meaning the same as `SHOW CREATE TABLE`. Closes [#48580](https://github.com/ClickHouse/ClickHouse/issues/48580). [#48591](https://github.com/ClickHouse/ClickHouse/pull/48591) ([flynn](https://github.com/ucasfl)). +* HTTP temporary buffer support working with fs cache. [#48664](https://github.com/ClickHouse/ClickHouse/pull/48664) ([Vladimir C](https://github.com/vdimir)). +* Make Schema inference works for `CREATE AS SELECT`. Closes [#47599](https://github.com/ClickHouse/ClickHouse/issues/47599). [#48679](https://github.com/ClickHouse/ClickHouse/pull/48679) ([flynn](https://github.com/ucasfl)). +* Added a `replicated_max_mutations_in_one_entry` setting for `ReplicatedMergeTree` that allows limiting the number of mutation commands per one `MUTATE_PART` entry (default is 10000). [#48731](https://github.com/ClickHouse/ClickHouse/pull/48731) ([Alexander Tokmakov](https://github.com/tavplubix)). +* In AggregateFunction types, don't count unused arena bytes as `read_bytes`. [#48745](https://github.com/ClickHouse/ClickHouse/pull/48745) ([Raúl Marín](https://github.com/Algunenano)). +* Fix some mysql related settings not being handled with mysql dictionary source + named collection. Closes [#48402](https://github.com/ClickHouse/ClickHouse/issues/48402). [#48759](https://github.com/ClickHouse/ClickHouse/pull/48759) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix squashing in query cache. [#48763](https://github.com/ClickHouse/ClickHouse/pull/48763) ([Robert Schulze](https://github.com/rschu1ze)). +* Support following new jsonpath format - '$.1key', path element begins with number - '$[key]', '$[“key”]', '$[\\\'key\\\']', '$["key 123"]', path element embraced in []. [#48768](https://github.com/ClickHouse/ClickHouse/pull/48768) ([lgbo](https://github.com/lgbo-ustc)). +* If a user set `max_single_part_upload_size` to a very large value, it can lead to a crash due to a bug in the AWS S3 SDK. This fixes [#47679](https://github.com/ClickHouse/ClickHouse/issues/47679). [#48816](https://github.com/ClickHouse/ClickHouse/pull/48816) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Not for changelog. [#48824](https://github.com/ClickHouse/ClickHouse/pull/48824) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Fix data race in `StorageRabbitMQ` ([report](https://pastila.nl/?004f7100/de1505289ab5bb355e67ebe6c7cc8707)), refactor the code. [#48845](https://github.com/ClickHouse/ClickHouse/pull/48845) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add aliases `name` and `part_name` form `system.parts` and `system.part_log`. Closes [#48718](https://github.com/ClickHouse/ClickHouse/issues/48718). [#48850](https://github.com/ClickHouse/ClickHouse/pull/48850) ([sichenzhao](https://github.com/sichenzhao)). +* Functions "arrayDifferenceSupport()", "arrayCumSum()" and "arrayCumSumNonNegative()" now support input arrays of wide integer types (U)Int128/256. [#48866](https://github.com/ClickHouse/ClickHouse/pull/48866) ([cluster](https://github.com/infdahai)). +* Multi-line history in clickhouse-client is now no longer padded. This makes pasting more natural. [#48870](https://github.com/ClickHouse/ClickHouse/pull/48870) ([Joanna Hulboj](https://github.com/jh0x)). +* Not for changelog. [#48873](https://github.com/ClickHouse/ClickHouse/pull/48873) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Implement a slight improvement for the rare case when ClickHouse is run inside LXC and LXCFS is used. The LXCFS has an issue: sometimes it returns an error "Transport endpoint is not connected" on reading from the file inside `/proc`. This error was correctly logged into ClickHouse's server log. We have additionally workaround this issue by reopening a file. This is a minuscule change. [#48922](https://github.com/ClickHouse/ClickHouse/pull/48922) ([Real](https://github.com/RunningXie)). +* Improve memory accounting for prefetches. Randomise prefetch settings In CI. [#48973](https://github.com/ClickHouse/ClickHouse/pull/48973) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Correctly set headers for native copy operations on GCS. [#48981](https://github.com/ClickHouse/ClickHouse/pull/48981) ([Antonio Andelic](https://github.com/antonio2368)). +* Add support for specifying setting names in the command line with dashes instead of underscores, for example, `--max-threads` instead of `--max_threads`. Additionally, support Unicode dash characters like `—` instead of `--` - this is useful when you communicate with a team in another company, and a manager from that team copy-pasted code from MS Word. [#48985](https://github.com/ClickHouse/ClickHouse/pull/48985) ([alekseygolub](https://github.com/alekseygolub)). +* Add fallback to password authentication when authentication with SSL user certificate has failed. Closes [#48974](https://github.com/ClickHouse/ClickHouse/issues/48974). [#48989](https://github.com/ClickHouse/ClickHouse/pull/48989) ([Nikolay Degterinsky](https://github.com/evillique)). +* Increase default value for `connect_timeout_with_failover_ms` to 1000 ms (because of adding async connections in https://github.com/ClickHouse/ClickHouse/pull/47229) . Closes [#5188](https://github.com/ClickHouse/ClickHouse/issues/5188). [#49009](https://github.com/ClickHouse/ClickHouse/pull/49009) ([Kruglov Pavel](https://github.com/Avogar)). +* Improve the embedded dashboard. Close [#46671](https://github.com/ClickHouse/ClickHouse/issues/46671). [#49036](https://github.com/ClickHouse/ClickHouse/pull/49036) ([Kevin Zhang](https://github.com/Kinzeng)). +* Add profile events for log messages, so you can easily see the count of log messages by severity. [#49042](https://github.com/ClickHouse/ClickHouse/pull/49042) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* `bitCount` function support `FixedString` and `String` data type. [#49044](https://github.com/ClickHouse/ClickHouse/pull/49044) ([flynn](https://github.com/ucasfl)). +* In previous versions, the `LineAsString` format worked inconsistently when the parallel parsing was enabled or not, in presence of DOS or MacOS Classic line breaks. This closes [#49039](https://github.com/ClickHouse/ClickHouse/issues/49039). [#49052](https://github.com/ClickHouse/ClickHouse/pull/49052) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* The exception message about the unparsed query parameter will also tell about the name of the parameter. Reimplement [#48878](https://github.com/ClickHouse/ClickHouse/issues/48878). Close [#48772](https://github.com/ClickHouse/ClickHouse/issues/48772). [#49061](https://github.com/ClickHouse/ClickHouse/pull/49061) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Added field `rows` with number of rows parsed from asynchronous insert to `system.asynchronous_insert_log`. [#49120](https://github.com/ClickHouse/ClickHouse/pull/49120) ([Anton Popov](https://github.com/CurtizJ)). +* 1. Bump Intel QPL from v1.0.0 to v1.1.0 (fixes build issue [#47877](https://github.com/ClickHouse/ClickHouse/issues/47877)) 2. the DEFLATE_QPL codec now respects the maximum hardware jobs returned by libaccel_config. [#49126](https://github.com/ClickHouse/ClickHouse/pull/49126) ([jasperzhu](https://github.com/jinjunzh)). + +#### Build/Testing/Packaging Improvement +* Reduce the number of dependencies in the header files to speed up the build. [#47984](https://github.com/ClickHouse/ClickHouse/pull/47984) ([Dmitry Novik](https://github.com/novikd)). +* Randomize compression of marks and indices in tests. [#48286](https://github.com/ClickHouse/ClickHouse/pull/48286) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Randomize vertical merges from compact to wide parts in tests. [#48287](https://github.com/ClickHouse/ClickHouse/pull/48287) ([Raúl Marín](https://github.com/Algunenano)). +* With the current approach, all ports are calculated at the beginning and could overlap or even be highjacked, see [the report](https://s3.amazonaws.com/clickhouse-test-reports/46793/02928ae50c52f31ce8e5bfa99eb1b5db046f4a4f/integration_tests__release__[1/2]/integration_run_parallel8_0.log) for `port is already allocated`. It's possibly the reason for [#45368](https://github.com/ClickHouse/ClickHouse/issues/45368). [#48393](https://github.com/ClickHouse/ClickHouse/pull/48393) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update time zones. The following were updated: Africa/Cairo, Africa/Casablanca, Africa/El_Aaiun, America/Bogota, America/Cambridge_Bay, America/Ciudad_Juarez, America/Godthab, America/Inuvik, America/Iqaluit, America/Nuuk, America/Ojinaga, America/Pangnirtung, America/Rankin_Inlet, America/Resolute, America/Whitehorse, America/Yellowknife, Asia/Gaza, Asia/Hebron, Asia/Kuala_Lumpur, Asia/Singapore, Canada/Yukon, Egypt, Europe/Kirov, Europe/Volgograd, Singapore. [#48572](https://github.com/ClickHouse/ClickHouse/pull/48572) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Support for CRC32 checksum in HDFS. Fix performance issues. [#48614](https://github.com/ClickHouse/ClickHouse/pull/48614) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove remainders of GCC support. [#48671](https://github.com/ClickHouse/ClickHouse/pull/48671) ([Robert Schulze](https://github.com/rschu1ze)). +* Add CI run with new analyzer infrastructure enabled. [#48719](https://github.com/ClickHouse/ClickHouse/pull/48719) ([Dmitry Novik](https://github.com/novikd)). +* Not for changelog. [#48879](https://github.com/ClickHouse/ClickHouse/pull/48879) ([larryluogit](https://github.com/larryluogit)). +* After the recent update, the `dockerd` requires `--tlsverify=false` together with the http port explicitly. [#48924](https://github.com/ClickHouse/ClickHouse/pull/48924) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Run more functional tests concurrently. [#48970](https://github.com/ClickHouse/ClickHouse/pull/48970) ([alesapin](https://github.com/alesapin)). +* Fix glibc compatibility check: replace `preadv` from musl. [#49144](https://github.com/ClickHouse/ClickHouse/pull/49144) ([alesapin](https://github.com/alesapin)). +* Use position independent encoding/code for sanitizers (at least msan :D) build to avoid issues with maximum relocation size. [#49145](https://github.com/ClickHouse/ClickHouse/pull/49145) ([alesapin](https://github.com/alesapin)). + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Fix system.query_views_log for MVs that are pushed from background threads [#46668](https://github.com/ClickHouse/ClickHouse/pull/46668) ([Azat Khuzhin](https://github.com/azat)). +* Fix several `RENAME COLUMN` bugs [#46946](https://github.com/ClickHouse/ClickHouse/pull/46946) ([alesapin](https://github.com/alesapin)). +* Fix minor hiliting issues in clickhouse-format [#47610](https://github.com/ClickHouse/ClickHouse/pull/47610) ([Natasha Murashkina](https://github.com/murfel)). +* Fix crash for uploading parts which size is greater then INT_MAX to S3 [#47693](https://github.com/ClickHouse/ClickHouse/pull/47693) ([Azat Khuzhin](https://github.com/azat)). +* Fix overflow in sparkbar function [#48121](https://github.com/ClickHouse/ClickHouse/pull/48121) ([Vladimir C](https://github.com/vdimir)). +* Fix race in StorageS3 [#48190](https://github.com/ClickHouse/ClickHouse/pull/48190) ([Anton Popov](https://github.com/CurtizJ)). +* Remove a feature [#48195](https://github.com/ClickHouse/ClickHouse/pull/48195) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix alter formatting (minor) [#48289](https://github.com/ClickHouse/ClickHouse/pull/48289) ([Natasha Murashkina](https://github.com/murfel)). +* Fix cpu usage in rabbitmq (was worsened in 23.2 after [#44404](https://github.com/ClickHouse/ClickHouse/issues/44404)) [#48311](https://github.com/ClickHouse/ClickHouse/pull/48311) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix ThreadPool for DistributedSink and use StrongTypedef for CurrentMetrics/ProfileEvents/StatusInfo to avoid further errors [#48314](https://github.com/ClickHouse/ClickHouse/pull/48314) ([Azat Khuzhin](https://github.com/azat)). +* Fix crash in EXPLAIN PIPELINE for Merge over Distributed [#48320](https://github.com/ClickHouse/ClickHouse/pull/48320) ([Azat Khuzhin](https://github.com/azat)). +* Check node for Backup Restore concurrency [#48342](https://github.com/ClickHouse/ClickHouse/pull/48342) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* close client [#48347](https://github.com/ClickHouse/ClickHouse/pull/48347) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Fix serializing LowCardinality as Arrow dictionary [#48361](https://github.com/ClickHouse/ClickHouse/pull/48361) ([Kruglov Pavel](https://github.com/Avogar)). +* Reset downloader for cache file segment in TemporaryFileStream [#48386](https://github.com/ClickHouse/ClickHouse/pull/48386) ([Vladimir C](https://github.com/vdimir)). +* Fix possible SYSTEM SYNC REPLICA stuck in case of DROP/REPLACE PARTITION [#48391](https://github.com/ClickHouse/ClickHouse/pull/48391) ([Azat Khuzhin](https://github.com/azat)). +* ClickHouse startup error when loading a distributed table that depends on a dictionary [#48419](https://github.com/ClickHouse/ClickHouse/pull/48419) ([MikhailBurdukov](https://github.com/MikhailBurdukov)). +* Don't check dependencies when renaming system tables automatically [#48431](https://github.com/ClickHouse/ClickHouse/pull/48431) ([Raúl Marín](https://github.com/Algunenano)). +* Some fixes for parallel replicas [#48433](https://github.com/ClickHouse/ClickHouse/pull/48433) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Update only affected rows in KV storage [#48435](https://github.com/ClickHouse/ClickHouse/pull/48435) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix possible segfault in cache [#48469](https://github.com/ClickHouse/ClickHouse/pull/48469) ([Kseniia Sumarokova](https://github.com/kssenii)). +* toTimeZone function throw an error when no constant string is provided [#48471](https://github.com/ClickHouse/ClickHouse/pull/48471) ([Jordi Villar](https://github.com/jrdi)). +* Fix logical error with IPv4 in Protobuf, add support for Date32 [#48486](https://github.com/ClickHouse/ClickHouse/pull/48486) ([Kruglov Pavel](https://github.com/Avogar)). +* "changed" flag in system.settings is calculated incorrectly for settings with multiple values [#48516](https://github.com/ClickHouse/ClickHouse/pull/48516) ([MikhailBurdukov](https://github.com/MikhailBurdukov)). +* Fix storage `Memory` with enabled compression [#48517](https://github.com/ClickHouse/ClickHouse/pull/48517) ([Anton Popov](https://github.com/CurtizJ)). +* Fix bracketed-paste mode messing up password input in client reconnect [#48528](https://github.com/ClickHouse/ClickHouse/pull/48528) ([Michael Kolupaev](https://github.com/al13n321)). +* Avoid sending `nullptr` to `memcpy` in `copyStringInArena` [#48532](https://github.com/ClickHouse/ClickHouse/pull/48532) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix nested map for keys of IP and UUID types [#48556](https://github.com/ClickHouse/ClickHouse/pull/48556) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Fix uncaught exception in case of parallel loader for hashed dictionaries [#48571](https://github.com/ClickHouse/ClickHouse/pull/48571) ([Azat Khuzhin](https://github.com/azat)). +* `groupArray` returns cannot be nullable [#48593](https://github.com/ClickHouse/ClickHouse/pull/48593) ([lgbo](https://github.com/lgbo-ustc)). +* Fix bug in Keeper when a node is not created with scheme `auth` in ACL sometimes. [#48595](https://github.com/ClickHouse/ClickHouse/pull/48595) ([Aleksei Filatov](https://github.com/aalexfvk)). +* Fix IPv4 comparable with UInt [#48611](https://github.com/ClickHouse/ClickHouse/pull/48611) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Fix possible error from cache [#48636](https://github.com/ClickHouse/ClickHouse/pull/48636) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix async inserts with empty data [#48663](https://github.com/ClickHouse/ClickHouse/pull/48663) ([Anton Popov](https://github.com/CurtizJ)). +* Fix table dependencies in case of failed RENAME TABLE [#48683](https://github.com/ClickHouse/ClickHouse/pull/48683) ([Azat Khuzhin](https://github.com/azat)). +* Fix zero-copy-replication on encrypted disks. [#48741](https://github.com/ClickHouse/ClickHouse/pull/48741) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix skip_unavailable_shards in case of unavailable hosts [#48771](https://github.com/ClickHouse/ClickHouse/pull/48771) ([Azat Khuzhin](https://github.com/azat)). +* Fix key condition on duplicate primary keys [#48838](https://github.com/ClickHouse/ClickHouse/pull/48838) ([Amos Bird](https://github.com/amosbird)). +* Fix for race in ZooKeeper when joining send_thread/receive_thread [#48849](https://github.com/ClickHouse/ClickHouse/pull/48849) ([Alexander Gololobov](https://github.com/davenger)). +* Fix unexpected part name error when trying to drop a ignored detached part with zero copy replication [#48862](https://github.com/ClickHouse/ClickHouse/pull/48862) ([Michael Lex](https://github.com/mlex)). +* Fix reading Date32 Parquet/Arrow column into not Date32 column [#48864](https://github.com/ClickHouse/ClickHouse/pull/48864) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix UNKNOWN_IDENTIFIER error while select from table with row policy and column with dots [#48976](https://github.com/ClickHouse/ClickHouse/pull/48976) ([Kruglov Pavel](https://github.com/Avogar)). +* Fix aggregate empty string error [#48999](https://github.com/ClickHouse/ClickHouse/pull/48999) ([LiuNeng](https://github.com/liuneng1994)). +* Fix postgres database setting [#49100](https://github.com/ClickHouse/ClickHouse/pull/49100) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix flaky test_cache_with_full_disk_space [#49110](https://github.com/ClickHouse/ClickHouse/pull/49110) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix "prepared statement insert already exists" [#49154](https://github.com/ClickHouse/ClickHouse/pull/49154) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix replace[Regexp]{One,All}() with const haystacks [#49220](https://github.com/ClickHouse/ClickHouse/pull/49220) ([Robert Schulze](https://github.com/rschu1ze)). + +#### Build Improvement + +* Fixed hashing issue in creating partition IDs for s390x. [#48134](https://github.com/ClickHouse/ClickHouse/pull/48134) ([Harry Lee](https://github.com/HarryLeeIBM)). + +#### NO CL ENTRY + +* NO CL ENTRY: 'Revert "Randomize JIT settings in tests"'. [#48277](https://github.com/ClickHouse/ClickHouse/pull/48277) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Fix test "02494_query_cache_drop.sql"'. [#48358](https://github.com/ClickHouse/ClickHouse/pull/48358) ([Anton Popov](https://github.com/CurtizJ)). +* NO CL ENTRY: 'Revert "Check simple dictionary key is native unsigned integer"'. [#48732](https://github.com/ClickHouse/ClickHouse/pull/48732) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Revert "Make Schema inference works for CREATE AS SELECT"'. [#48758](https://github.com/ClickHouse/ClickHouse/pull/48758) ([pufit](https://github.com/pufit)). +* NO CL ENTRY: 'Revert "Add MemoryTracker for the background tasks"'. [#48760](https://github.com/ClickHouse/ClickHouse/pull/48760) ([Alexander Tokmakov](https://github.com/tavplubix)). +* NO CL ENTRY: 'Revert "Added tests for ClickHouse apps help and fixed help issues"'. [#48991](https://github.com/ClickHouse/ClickHouse/pull/48991) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* NO CL ENTRY: 'Revert "Adapt marks count for prefetch read pool"'. [#49068](https://github.com/ClickHouse/ClickHouse/pull/49068) ([Nikita Taranov](https://github.com/nickitat)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* merge and mutation make thread group for setting memory trackers right [#47104](https://github.com/ClickHouse/ClickHouse/pull/47104) ([Sema Checherinda](https://github.com/CheSema)). +* Query plan: update sort description [#47319](https://github.com/ClickHouse/ClickHouse/pull/47319) ([Igor Nikonov](https://github.com/devcrafter)). +* Sqllogic [#47784](https://github.com/ClickHouse/ClickHouse/pull/47784) ([Sema Checherinda](https://github.com/CheSema)). +* Fix race between DROP MatView and RESTART REPLICAS [#47863](https://github.com/ClickHouse/ClickHouse/pull/47863) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Merge [#35113](https://github.com/ClickHouse/ClickHouse/issues/35113) [#47934](https://github.com/ClickHouse/ClickHouse/pull/47934) ([Antonio Andelic](https://github.com/antonio2368)). +* Add a test for ClientInfo initial_query_start_time in inter-server mode [#48036](https://github.com/ClickHouse/ClickHouse/pull/48036) ([Azat Khuzhin](https://github.com/azat)). +* Make custom key for parallel replicas work in new analyzer [#48054](https://github.com/ClickHouse/ClickHouse/pull/48054) ([Antonio Andelic](https://github.com/antonio2368)). +* throw exception while non-parametric functions having parameters [#48115](https://github.com/ClickHouse/ClickHouse/pull/48115) ([save-my-heart](https://github.com/save-my-heart)). +* Move FunctionsJSON implementation to header file [#48142](https://github.com/ClickHouse/ClickHouse/pull/48142) ([DimasKovas](https://github.com/DimasKovas)). +* Use ThreadPool in PipelineExecutor [#48146](https://github.com/ClickHouse/ClickHouse/pull/48146) ([Azat Khuzhin](https://github.com/azat)). +* Add sanity checks for writing number in variable length format (resubmit) [#48154](https://github.com/ClickHouse/ClickHouse/pull/48154) ([Azat Khuzhin](https://github.com/azat)). +* Try fix 02151_hash_table_sizes_stats.sh test [#48178](https://github.com/ClickHouse/ClickHouse/pull/48178) ([Nikita Taranov](https://github.com/nickitat)). +* Add scripts for sparse checkout of some contribs [#48183](https://github.com/ClickHouse/ClickHouse/pull/48183) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Do not take lock for shared context in setTempDataOnDisk [#48219](https://github.com/ClickHouse/ClickHouse/pull/48219) ([Vladimir C](https://github.com/vdimir)). +* parseDateTime[InJodaSyntax](): Require format argument [#48222](https://github.com/ClickHouse/ClickHouse/pull/48222) ([Robert Schulze](https://github.com/rschu1ze)). +* Do not partially cancel processors added from expand pipeline. [#48231](https://github.com/ClickHouse/ClickHouse/pull/48231) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix some tests [#48267](https://github.com/ClickHouse/ClickHouse/pull/48267) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix compiling examples without Hive [#48269](https://github.com/ClickHouse/ClickHouse/pull/48269) ([Azat Khuzhin](https://github.com/azat)). +* In messages, put values into quotes [#48271](https://github.com/ClickHouse/ClickHouse/pull/48271) ([Vadim Chekan](https://github.com/vchekan)). +* Fix 01710_projection_optimize_materialize flakiness [#48276](https://github.com/ClickHouse/ClickHouse/pull/48276) ([Azat Khuzhin](https://github.com/azat)). +* Fix UB (signed integer overflow) in StorageMergeTree::backupData() [#48278](https://github.com/ClickHouse/ClickHouse/pull/48278) ([Azat Khuzhin](https://github.com/azat)). +* Update version after release [#48279](https://github.com/ClickHouse/ClickHouse/pull/48279) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update version_date.tsv and changelogs after v23.3.1.2823-lts [#48281](https://github.com/ClickHouse/ClickHouse/pull/48281) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Small follow-up to [#48017](https://github.com/ClickHouse/ClickHouse/issues/48017) [#48292](https://github.com/ClickHouse/ClickHouse/pull/48292) ([Robert Schulze](https://github.com/rschu1ze)). +* Try to update arrow library to release 11.0.0 [#48294](https://github.com/ClickHouse/ClickHouse/pull/48294) ([Kruglov Pavel](https://github.com/Avogar)). +* fix test numbers again 2 [#48295](https://github.com/ClickHouse/ClickHouse/pull/48295) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix: copy forgotten show_secrets in FormatSettings semi-copy-ctor [#48297](https://github.com/ClickHouse/ClickHouse/pull/48297) ([Natasha Murashkina](https://github.com/murfel)). +* Do not remove inputs from maybe compiled DAG. [#48303](https://github.com/ClickHouse/ClickHouse/pull/48303) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Update version_date.tsv and changelogs after v22.3.20.29-lts [#48304](https://github.com/ClickHouse/ClickHouse/pull/48304) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v22.12.6.22-stable, v22.3.20.29-lts [#48305](https://github.com/ClickHouse/ClickHouse/pull/48305) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Merging [#46323](https://github.com/ClickHouse/ClickHouse/issues/46323) [#48312](https://github.com/ClickHouse/ClickHouse/pull/48312) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Follow-up to [#47863](https://github.com/ClickHouse/ClickHouse/issues/47863) [#48315](https://github.com/ClickHouse/ClickHouse/pull/48315) ([Alexander Tokmakov](https://github.com/tavplubix)). +* test / some complex query (it fails with analyzer enabled) [#48324](https://github.com/ClickHouse/ClickHouse/pull/48324) ([Denny Crane](https://github.com/den-crane)). +* Fix constraints after merge [#48328](https://github.com/ClickHouse/ClickHouse/pull/48328) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add logging for concurrency checks for backups [#48337](https://github.com/ClickHouse/ClickHouse/pull/48337) ([Vitaly Baranov](https://github.com/vitlibar)). +* Update version_date.tsv and changelogs after v23.1.6.42-stable [#48345](https://github.com/ClickHouse/ClickHouse/pull/48345) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v23.2.5.46-stable [#48346](https://github.com/ClickHouse/ClickHouse/pull/48346) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Fix lambda type resolution [#48355](https://github.com/ClickHouse/ClickHouse/pull/48355) ([Dmitry Novik](https://github.com/novikd)). +* Avoid abort in protobuf library in debug build [#48356](https://github.com/ClickHouse/ClickHouse/pull/48356) ([Kruglov Pavel](https://github.com/Avogar)). +* Batch fix for projections analysis with analyzer. [#48357](https://github.com/ClickHouse/ClickHouse/pull/48357) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix tests with explain and analyzer where names changed. [#48360](https://github.com/ClickHouse/ClickHouse/pull/48360) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Small follow-up to [#45912](https://github.com/ClickHouse/ClickHouse/issues/45912) [#48373](https://github.com/ClickHouse/ClickHouse/pull/48373) ([Robert Schulze](https://github.com/rschu1ze)). +* Update version_date.tsv and changelogs after v22.8.16.32-lts [#48376](https://github.com/ClickHouse/ClickHouse/pull/48376) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Add script for a slack bot that reports broken tests [#48382](https://github.com/ClickHouse/ClickHouse/pull/48382) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix flaky `test_keeper_mntr_data_size` [#48384](https://github.com/ClickHouse/ClickHouse/pull/48384) ([Antonio Andelic](https://github.com/antonio2368)). +* WITH FILL clarification and cleanup [#48395](https://github.com/ClickHouse/ClickHouse/pull/48395) ([Igor Nikonov](https://github.com/devcrafter)). +* Cleanup mess in .clang-tidy [#48396](https://github.com/ClickHouse/ClickHouse/pull/48396) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix test_backup_all [#48400](https://github.com/ClickHouse/ClickHouse/pull/48400) ([Vitaly Baranov](https://github.com/vitlibar)). +* Find big allocations without memory limits checks [#48401](https://github.com/ClickHouse/ClickHouse/pull/48401) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix issue with krb5 and building w/ OpenSSL [#48407](https://github.com/ClickHouse/ClickHouse/pull/48407) ([Boris Kuschel](https://github.com/bkuschel)). +* Make CI slack bot less noisy [#48409](https://github.com/ClickHouse/ClickHouse/pull/48409) ([Alexander Tokmakov](https://github.com/tavplubix)). +* AST fuzzer: Fix assertion in TopK serialization [#48412](https://github.com/ClickHouse/ClickHouse/pull/48412) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix possible flakiness of lightweight delete tests (due to index granularity randomization) [#48413](https://github.com/ClickHouse/ClickHouse/pull/48413) ([Azat Khuzhin](https://github.com/azat)). +* Fix flaky `test_keeper_snapshots` [#48417](https://github.com/ClickHouse/ClickHouse/pull/48417) ([Antonio Andelic](https://github.com/antonio2368)). +* Update sort desc: more efficient original node search in ActionsDAG [#48427](https://github.com/ClickHouse/ClickHouse/pull/48427) ([Igor Nikonov](https://github.com/devcrafter)). +* test for [#16399](https://github.com/ClickHouse/ClickHouse/issues/16399) [#48439](https://github.com/ClickHouse/ClickHouse/pull/48439) ([Denny Crane](https://github.com/den-crane)). +* Better exception messages from Keeper client [#48444](https://github.com/ClickHouse/ClickHouse/pull/48444) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Small documentation follow-up to [#47246](https://github.com/ClickHouse/ClickHouse/issues/47246) [#48463](https://github.com/ClickHouse/ClickHouse/pull/48463) ([Robert Schulze](https://github.com/rschu1ze)). +* Update 00002_log_and_exception_messages_formatting.sql [#48467](https://github.com/ClickHouse/ClickHouse/pull/48467) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Avoid operation on uninitialised data in readDateTimeTextImpl [#48472](https://github.com/ClickHouse/ClickHouse/pull/48472) ([Kruglov Pavel](https://github.com/Avogar)). +* Add reading step for system zookeeper. Analyze path from filter DAG. [#48485](https://github.com/ClickHouse/ClickHouse/pull/48485) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix deadlock due to debug tracking of memory allocations [#48487](https://github.com/ClickHouse/ClickHouse/pull/48487) ([Azat Khuzhin](https://github.com/azat)). +* Register datediff and trim aliases in system.functions [#48489](https://github.com/ClickHouse/ClickHouse/pull/48489) ([Robert Schulze](https://github.com/rschu1ze)). +* Change error code [#48490](https://github.com/ClickHouse/ClickHouse/pull/48490) ([Anton Popov](https://github.com/CurtizJ)). +* Update 00002_log_and_exception_messages_formatting.sql [#48499](https://github.com/ClickHouse/ClickHouse/pull/48499) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix query cache with sparse columns [#48500](https://github.com/ClickHouse/ClickHouse/pull/48500) ([Anton Popov](https://github.com/CurtizJ)). +* Use std::string_view to get rid of strlen [#48509](https://github.com/ClickHouse/ClickHouse/pull/48509) ([ltrk2](https://github.com/ltrk2)). +* Fix bytesSize() of zk SetRequest [#48512](https://github.com/ClickHouse/ClickHouse/pull/48512) ([Sergei Trifonov](https://github.com/serxa)). +* Remove dead code and unused dependencies [#48518](https://github.com/ClickHouse/ClickHouse/pull/48518) ([ltrk2](https://github.com/ltrk2)). +* Use forward declaration of ThreadPool [#48519](https://github.com/ClickHouse/ClickHouse/pull/48519) ([Azat Khuzhin](https://github.com/azat)). +* Use std::string_view instead of strlen [#48520](https://github.com/ClickHouse/ClickHouse/pull/48520) ([ltrk2](https://github.com/ltrk2)). +* Use std::string::starts_with instead of a roll your own variant [#48521](https://github.com/ClickHouse/ClickHouse/pull/48521) ([ltrk2](https://github.com/ltrk2)). +* Fix flaky `test_alternative_keeper_config` [#48533](https://github.com/ClickHouse/ClickHouse/pull/48533) ([Antonio Andelic](https://github.com/antonio2368)). +* Use one ThreadGroup while pushing to materialized views (and some refactoring for ThreadGroup) [#48543](https://github.com/ClickHouse/ClickHouse/pull/48543) ([Azat Khuzhin](https://github.com/azat)). +* Fix some tests [#48550](https://github.com/ClickHouse/ClickHouse/pull/48550) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix 02477_projection_materialize_and_zero_copy flakiness (due to index granularity randomization) [#48551](https://github.com/ClickHouse/ClickHouse/pull/48551) ([Azat Khuzhin](https://github.com/azat)). +* Better exception message for ZSTD [#48552](https://github.com/ClickHouse/ClickHouse/pull/48552) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove misleading comment and block [#48562](https://github.com/ClickHouse/ClickHouse/pull/48562) ([Sergei Trifonov](https://github.com/serxa)). +* Update 02207_allow_plaintext_and_no_password.sh [#48566](https://github.com/ClickHouse/ClickHouse/pull/48566) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* bugfix: compare Bits and sizeof(Arithmetic) * 8 [#48569](https://github.com/ClickHouse/ClickHouse/pull/48569) ([caipengxiang](https://github.com/awfeequdng)). +* Remove superfluous includes of logger_userful.h from headers [#48570](https://github.com/ClickHouse/ClickHouse/pull/48570) ([Azat Khuzhin](https://github.com/azat)). +* Remove slow test from debug builds [#48574](https://github.com/ClickHouse/ClickHouse/pull/48574) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Don't use type conversion with String query parameters [#48577](https://github.com/ClickHouse/ClickHouse/pull/48577) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix TSan report in Kerberos [#48579](https://github.com/ClickHouse/ClickHouse/pull/48579) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add second_deadlock_stack=1 for TSan on CI and fix some lock-order-inversion problems [#48596](https://github.com/ClickHouse/ClickHouse/pull/48596) ([Azat Khuzhin](https://github.com/azat)). +* Fix LOGICAL_ERROR in executable table function [#48605](https://github.com/ClickHouse/ClickHouse/pull/48605) ([Nikolay Degterinsky](https://github.com/evillique)). +* Fix flakiness of test_store_cleanup in case of image rebuild [#48610](https://github.com/ClickHouse/ClickHouse/pull/48610) ([Azat Khuzhin](https://github.com/azat)). +* Remove strange code [#48612](https://github.com/ClickHouse/ClickHouse/pull/48612) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Minor refactoring of formatDateTime() [#48627](https://github.com/ClickHouse/ClickHouse/pull/48627) ([Robert Schulze](https://github.com/rschu1ze)). +* Better handling of values too large for VarInt encoding [#48628](https://github.com/ClickHouse/ClickHouse/pull/48628) ([Robert Schulze](https://github.com/rschu1ze)). +* refine some messages of exception in regexp tree [#48632](https://github.com/ClickHouse/ClickHouse/pull/48632) ([Han Fei](https://github.com/hanfei1991)). +* Partially revert e0252db8d and fix pr-bugfix labeling [#48637](https://github.com/ClickHouse/ClickHouse/pull/48637) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix build src/Interpreters/InterpreterInsertQuery.h [#48638](https://github.com/ClickHouse/ClickHouse/pull/48638) ([Vladimir C](https://github.com/vdimir)). +* Fix build ThreadGroupPtr [#48641](https://github.com/ClickHouse/ClickHouse/pull/48641) ([Vladimir C](https://github.com/vdimir)). +* Fix flaky test test_drop_replica_and_achieve_quorum [#48642](https://github.com/ClickHouse/ClickHouse/pull/48642) ([Kruglov Pavel](https://github.com/Avogar)). +* fix 02504_regexp_dictionary_table_source [#48662](https://github.com/ClickHouse/ClickHouse/pull/48662) ([Han Fei](https://github.com/hanfei1991)). +* Remove strange code from MutateTask [#48666](https://github.com/ClickHouse/ClickHouse/pull/48666) ([alesapin](https://github.com/alesapin)). +* SonarCloud: C++ Reporting Standards [#48668](https://github.com/ClickHouse/ClickHouse/pull/48668) ([Julio Jimenez](https://github.com/juliojimenez)). +* Remove lock for duplicated parts UUIDs (allow_experimental_query_deduplication=1) [#48670](https://github.com/ClickHouse/ClickHouse/pull/48670) ([Azat Khuzhin](https://github.com/azat)). +* show result of minio listings for test test_attach_detach_partition [#48674](https://github.com/ClickHouse/ClickHouse/pull/48674) ([Sema Checherinda](https://github.com/CheSema)). +* Fix tests for analyzer [#48675](https://github.com/ClickHouse/ClickHouse/pull/48675) ([Igor Nikonov](https://github.com/devcrafter)). +* Call IProcessor::onCancel() once [#48687](https://github.com/ClickHouse/ClickHouse/pull/48687) ([Igor Nikonov](https://github.com/devcrafter)). +* Update MergeTree syntax for optional index granularity argument [#48692](https://github.com/ClickHouse/ClickHouse/pull/48692) ([Robert Schulze](https://github.com/rschu1ze)). +* Add test for old bug [#7826](https://github.com/ClickHouse/ClickHouse/issues/7826) [#48697](https://github.com/ClickHouse/ClickHouse/pull/48697) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix flaky `test_keeper_session` [#48699](https://github.com/ClickHouse/ClickHouse/pull/48699) ([Antonio Andelic](https://github.com/antonio2368)). +* Better messages formatting in the CI Slack bot [#48712](https://github.com/ClickHouse/ClickHouse/pull/48712) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Add trusted contributors [#48715](https://github.com/ClickHouse/ClickHouse/pull/48715) ([Aleksei Filatov](https://github.com/aalexfvk)). +* Do not remove broken detached parts on startup [#48730](https://github.com/ClickHouse/ClickHouse/pull/48730) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Remove `-Wshadow` suppression which leaked into global namespace [#48737](https://github.com/ClickHouse/ClickHouse/pull/48737) ([Robert Schulze](https://github.com/rschu1ze)). +* VarInt coding: Always perform sanity check [#48740](https://github.com/ClickHouse/ClickHouse/pull/48740) ([Robert Schulze](https://github.com/rschu1ze)). +* Try to fix flaky 02455_one_row_from_csv_memory_usage [#48756](https://github.com/ClickHouse/ClickHouse/pull/48756) ([Dmitry Novik](https://github.com/novikd)). +* insert UInt32 Hashvalue in reverse order on big endian machine [#48764](https://github.com/ClickHouse/ClickHouse/pull/48764) ([Suzy Wang](https://github.com/SuzyWangIBMer)). +* Limit size of messages from the CI slack bot [#48766](https://github.com/ClickHouse/ClickHouse/pull/48766) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Update README.md [#48776](https://github.com/ClickHouse/ClickHouse/pull/48776) ([Tyler Hannan](https://github.com/tylerhannan)). +* Remove duplicate definition of SingleEndpointHTTPSessionPool [#48779](https://github.com/ClickHouse/ClickHouse/pull/48779) ([JaySon](https://github.com/JaySon-Huang)). +* Fix flaky test_version_update_after_mutation/test.py::test_upgrade_while_mutation [#48783](https://github.com/ClickHouse/ClickHouse/pull/48783) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix flaky test test_backup_all [#48789](https://github.com/ClickHouse/ClickHouse/pull/48789) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix a confusing warning about interserver mode [#48793](https://github.com/ClickHouse/ClickHouse/pull/48793) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Store clusters from ClusterDiscovery in separate map [#48795](https://github.com/ClickHouse/ClickHouse/pull/48795) ([Vladimir C](https://github.com/vdimir)). +* Reimplement [#48790](https://github.com/ClickHouse/ClickHouse/issues/48790) [#48797](https://github.com/ClickHouse/ClickHouse/pull/48797) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Allow running integration tests without spark [#48803](https://github.com/ClickHouse/ClickHouse/pull/48803) ([Vitaly Baranov](https://github.com/vitlibar)). +* forbid gwpsan in debug mode to rescue stress tests [#48804](https://github.com/ClickHouse/ClickHouse/pull/48804) ([Han Fei](https://github.com/hanfei1991)). +* Simplify FileCacheFactory [#48805](https://github.com/ClickHouse/ClickHouse/pull/48805) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix double whitespace in exception message [#48815](https://github.com/ClickHouse/ClickHouse/pull/48815) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Add a test for [#38128](https://github.com/ClickHouse/ClickHouse/issues/38128) [#48817](https://github.com/ClickHouse/ClickHouse/pull/48817) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove excessive logging [#48826](https://github.com/ClickHouse/ClickHouse/pull/48826) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* remove duplicate indentwith in clang-format [#48834](https://github.com/ClickHouse/ClickHouse/pull/48834) ([cluster](https://github.com/infdahai)). +* Try fix flacky test_concurrent_alter_move_and_drop [#48843](https://github.com/ClickHouse/ClickHouse/pull/48843) ([Sergei Trifonov](https://github.com/serxa)). +* fix the race wait loading parts [#48844](https://github.com/ClickHouse/ClickHouse/pull/48844) ([Sema Checherinda](https://github.com/CheSema)). +* suppress assert of progress for test_system_replicated_fetches [#48856](https://github.com/ClickHouse/ClickHouse/pull/48856) ([Han Fei](https://github.com/hanfei1991)). +* Fix: do not run test_store_cleanup_disk_s3 in parallel [#48863](https://github.com/ClickHouse/ClickHouse/pull/48863) ([Igor Nikonov](https://github.com/devcrafter)). +* Update README.md [#48883](https://github.com/ClickHouse/ClickHouse/pull/48883) ([Tyler Hannan](https://github.com/tylerhannan)). +* Fix test reference files for join using nullable column [#48893](https://github.com/ClickHouse/ClickHouse/pull/48893) ([Vladimir C](https://github.com/vdimir)). +* bitNot marked as NO_SANITIZE_UNDEFINED [#48899](https://github.com/ClickHouse/ClickHouse/pull/48899) ([Vladimir C](https://github.com/vdimir)). +* Fix order by in test_storage_delta [#48903](https://github.com/ClickHouse/ClickHouse/pull/48903) ([Vladimir C](https://github.com/vdimir)). +* Fix segfault when set is not built yet [#48904](https://github.com/ClickHouse/ClickHouse/pull/48904) ([Alexander Gololobov](https://github.com/davenger)). +* A non significant change (does not affect anything): add support for signed integers in the maskBits function [#48920](https://github.com/ClickHouse/ClickHouse/pull/48920) ([caipengxiang](https://github.com/awfeequdng)). +* Follow-up to [#48866](https://github.com/ClickHouse/ClickHouse/issues/48866) [#48929](https://github.com/ClickHouse/ClickHouse/pull/48929) ([Robert Schulze](https://github.com/rschu1ze)). +* Un-flake 01079_new_range_reader_segfault [#48934](https://github.com/ClickHouse/ClickHouse/pull/48934) ([Robert Schulze](https://github.com/rschu1ze)). +* Add building stage to the fasttests report, respect existing status on rerun [#48935](https://github.com/ClickHouse/ClickHouse/pull/48935) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update Settings.h [#48948](https://github.com/ClickHouse/ClickHouse/pull/48948) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Update cluster.py [#48949](https://github.com/ClickHouse/ClickHouse/pull/48949) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Docs: Replace annoying three spaces in enumerations by a single space [#48951](https://github.com/ClickHouse/ClickHouse/pull/48951) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix flaky 02706_arrow_different_dictionaries [#48952](https://github.com/ClickHouse/ClickHouse/pull/48952) ([Kruglov Pavel](https://github.com/Avogar)). +* Use default `{replica}`, `{shard}` arguments in Replicated engine [#48961](https://github.com/ClickHouse/ClickHouse/pull/48961) ([Nikolay Degterinsky](https://github.com/evillique)). +* Rename quantileApprox -> quantileGK [#48969](https://github.com/ClickHouse/ClickHouse/pull/48969) ([Vladimir C](https://github.com/vdimir)). +* Don't throw logical error when column is not found in Parquet/Arrow schema [#48987](https://github.com/ClickHouse/ClickHouse/pull/48987) ([Kruglov Pavel](https://github.com/Avogar)). +* Reimplement [#48986](https://github.com/ClickHouse/ClickHouse/issues/48986) [#49005](https://github.com/ClickHouse/ClickHouse/pull/49005) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Dont allow bad changelogs [#49006](https://github.com/ClickHouse/ClickHouse/pull/49006) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update README.md [#49007](https://github.com/ClickHouse/ClickHouse/pull/49007) ([Nick-71](https://github.com/Nick-71)). +* Remove outdated test [#49014](https://github.com/ClickHouse/ClickHouse/pull/49014) ([alesapin](https://github.com/alesapin)). +* Fix typo [#49027](https://github.com/ClickHouse/ClickHouse/pull/49027) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix assertion after [#48636](https://github.com/ClickHouse/ClickHouse/issues/48636) [#49029](https://github.com/ClickHouse/ClickHouse/pull/49029) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix build error for big-endian platforms [#49037](https://github.com/ClickHouse/ClickHouse/pull/49037) ([ltrk2](https://github.com/ltrk2)). +* Update version_date.tsv and changelogs after v22.8.17.17-lts [#49046](https://github.com/ClickHouse/ClickHouse/pull/49046) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v23.1.7.30-stable [#49047](https://github.com/ClickHouse/ClickHouse/pull/49047) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Update version_date.tsv and changelogs after v23.3.2.37-lts [#49048](https://github.com/ClickHouse/ClickHouse/pull/49048) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Remove some code [#49054](https://github.com/ClickHouse/ClickHouse/pull/49054) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Remove some dead code in poco [#49075](https://github.com/ClickHouse/ClickHouse/pull/49075) ([Robert Schulze](https://github.com/rschu1ze)). +* Prevent false positive report by static analyzer [#49078](https://github.com/ClickHouse/ClickHouse/pull/49078) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Update version_date.tsv and changelogs after v23.2.6.34-stable [#49080](https://github.com/ClickHouse/ClickHouse/pull/49080) ([robot-clickhouse](https://github.com/robot-clickhouse)). +* Enforce documentation change for a new-feature PR [#49090](https://github.com/ClickHouse/ClickHouse/pull/49090) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update clickhouse-test [#49094](https://github.com/ClickHouse/ClickHouse/pull/49094) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Disable long 02581 in debug, enable with sanitizers [#49105](https://github.com/ClickHouse/ClickHouse/pull/49105) ([Alexander Gololobov](https://github.com/davenger)). +* Fix flaky integration test test_async_query_sending [#49107](https://github.com/ClickHouse/ClickHouse/pull/49107) ([Kruglov Pavel](https://github.com/Avogar)). +* Correct functional test to reflect interoperability [#49108](https://github.com/ClickHouse/ClickHouse/pull/49108) ([ltrk2](https://github.com/ltrk2)). +* Cleanup build guide [#49119](https://github.com/ClickHouse/ClickHouse/pull/49119) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix building iceberg without avro [#49125](https://github.com/ClickHouse/ClickHouse/pull/49125) ([Azat Khuzhin](https://github.com/azat)). +* Add slash for close tag of user_defined_zookeeper_path [#49131](https://github.com/ClickHouse/ClickHouse/pull/49131) ([Hollin](https://github.com/Hooollin)). +* Improve some lambdas [#49133](https://github.com/ClickHouse/ClickHouse/pull/49133) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Do not randomize prefetch settings for debug build [#49134](https://github.com/ClickHouse/ClickHouse/pull/49134) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Don't throw LOGICAL_ERROR when reading from remote if there is no local replica [#49136](https://github.com/ClickHouse/ClickHouse/pull/49136) ([Raúl Marín](https://github.com/Algunenano)). +* Docs: Make caption of processors_profile_log page consistent with other pages [#49138](https://github.com/ClickHouse/ClickHouse/pull/49138) ([Robert Schulze](https://github.com/rschu1ze)). +* Improve test reports [#49151](https://github.com/ClickHouse/ClickHouse/pull/49151) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Add a note regarding private/public repo to logs [#49152](https://github.com/ClickHouse/ClickHouse/pull/49152) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* suppress two timeout tests [#49175](https://github.com/ClickHouse/ClickHouse/pull/49175) ([Han Fei](https://github.com/hanfei1991)). +* Document makeDateTime() and its variants [#49183](https://github.com/ClickHouse/ClickHouse/pull/49183) ([Robert Schulze](https://github.com/rschu1ze)). +* Fix after [#49110](https://github.com/ClickHouse/ClickHouse/issues/49110) [#49206](https://github.com/ClickHouse/ClickHouse/pull/49206) ([Kseniia Sumarokova](https://github.com/kssenii)). + diff --git a/docs/changelogs/v23.4.2.11-stable.md b/docs/changelogs/v23.4.2.11-stable.md new file mode 100644 index 00000000000..3c572b9c1cb --- /dev/null +++ b/docs/changelogs/v23.4.2.11-stable.md @@ -0,0 +1,20 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v23.4.2.11-stable (b6442320f9d) FIXME as compared to v23.4.1.1943-stable (3920eb987f7) + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Revert "Fix GCS native copy ([#48981](https://github.com/ClickHouse/ClickHouse/issues/48981))" [#49194](https://github.com/ClickHouse/ClickHouse/pull/49194) ([Raúl Marín](https://github.com/Algunenano)). +* Fix race on Outdated parts loading [#49223](https://github.com/ClickHouse/ClickHouse/pull/49223) ([Alexander Tokmakov](https://github.com/tavplubix)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Implement status comment [#48468](https://github.com/ClickHouse/ClickHouse/pull/48468) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Update curl to 8.0.1 (for CVEs) [#48765](https://github.com/ClickHouse/ClickHouse/pull/48765) ([Boris Kuschel](https://github.com/bkuschel)). +* Fallback auth gh api [#49314](https://github.com/ClickHouse/ClickHouse/pull/49314) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + diff --git a/docs/en/development/build.md b/docs/en/development/build.md index 6df46d9dec7..a55d44bdf93 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -13,23 +13,23 @@ Supported platforms: - AArch64 - Power9 (experimental) -## Normal Build for Development on Ubuntu +## Building on Ubuntu -The following tutorial is based on the Ubuntu Linux system. With appropriate changes, it should also work on any other Linux distribution. +The following tutorial is based on Ubuntu Linux. +With appropriate changes, it should also work on any other Linux distribution. +The minimum recommended Ubuntu version for development is 22.04 LTS. ### Install Prerequisites {#install-prerequisites} ``` bash -sudo apt-get install git cmake ccache python3 ninja-build yasm gawk +sudo apt-get install git cmake ccache python3 ninja-build nasm yasm gawk ``` -Or cmake3 instead of cmake on older systems. +### Install and Use the Clang compiler -### Install the latest clang (recommended) +On Ubuntu/Debian you can use LLVM's automatic installation script, see [here](https://apt.llvm.org/). -On Ubuntu/Debian you can use the automatic installation script (check [official webpage](https://apt.llvm.org/)) - -```bash +``` bash sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ``` @@ -40,19 +40,17 @@ sudo apt-get install software-properties-common sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test ``` -For other Linux distribution - check the availability of the [prebuild packages](https://releases.llvm.org/download.html) or build clang [from sources](https://clang.llvm.org/get_started.html). +For other Linux distribution - check the availability of LLVM's [prebuild packages](https://releases.llvm.org/download.html). -#### Use the latest clang for Builds +As of April 2023, any version of Clang >= 15 will work. +GCC as a compiler is not supported +To build with a specific Clang version: ``` bash export CC=clang-15 export CXX=clang++-15 ``` -In this example we use version 15 that is the latest as of Sept 2022. - -Gcc cannot be used. - ### Checkout ClickHouse Sources {#checkout-clickhouse-sources} ``` bash @@ -70,79 +68,46 @@ git clone --recursive --shallow-submodules https://github.com/ClickHouse/ClickHo ``` bash cd ClickHouse mkdir build -cd build -cmake .. -ninja +cmake -S . -B build +cmake --build build # or: `cd build; ninja` ``` -To create an executable, run `ninja clickhouse`. -This will create the `programs/clickhouse` executable, which can be used with `client` or `server` arguments. +To create an executable, run `cmake --build build --target clickhouse` (or: `cd build; ninja clickhouse`). +This will create executable `build/programs/clickhouse` which can be used with `client` or `server` arguments. -## How to Build ClickHouse on Any Linux {#how-to-build-clickhouse-on-any-linux} +## Building on Any Linux {#how-to-build-clickhouse-on-any-linux} The build requires the following components: -- Git (is used only to checkout the sources, it’s not needed for the build) -- CMake 3.15 or newer +- Git (used to checkout the sources, not needed for the build) +- CMake 3.20 or newer +- Compiler: Clang 15 or newer +- Linker: lld 15 or newer - Ninja -- C++ compiler: clang-15 or newer -- Linker: lld - Yasm - Gawk If all the components are installed, you may build in the same way as the steps above. -Example for Ubuntu Eoan: -``` bash -sudo apt update -sudo apt install git cmake ninja-build clang++ python yasm gawk -git clone --recursive https://github.com/ClickHouse/ClickHouse.git -mkdir build && cd build -cmake ../ClickHouse -ninja -``` - Example for OpenSUSE Tumbleweed: + ``` bash -sudo zypper install git cmake ninja clang-c++ python lld yasm gawk +sudo zypper install git cmake ninja clang-c++ python lld nasm yasm gawk git clone --recursive https://github.com/ClickHouse/ClickHouse.git -mkdir build && cd build -cmake ../ClickHouse -ninja +mkdir build +cmake -S . -B build +cmake --build build ``` Example for Fedora Rawhide: + ``` bash sudo yum update -sudo yum --nogpg install git cmake make clang python3 ccache yasm gawk +sudo yum --nogpg install git cmake make clang python3 ccache nasm yasm gawk git clone --recursive https://github.com/ClickHouse/ClickHouse.git -mkdir build && cd build -cmake ../ClickHouse -make -j $(nproc) -``` - -Here is an example of how to build `clang` and all the llvm infrastructure from sources: - -``` -git clone git@github.com:llvm/llvm-project.git -mkdir llvm-build && cd llvm-build -cmake -DCMAKE_BUILD_TYPE:STRING=Release -DLLVM_ENABLE_PROJECTS=all ../llvm-project/llvm/ -make -j16 -sudo make install -hash clang -clang --version -``` - -You can install the older clang like clang-11 from packages and then use it to build the new clang from sources. - -Here is an example of how to install the new `cmake` from the official website: - -``` -wget https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-linux-x86_64.sh -chmod +x cmake-3.22.2-linux-x86_64.sh -./cmake-3.22.2-linux-x86_64.sh -export PATH=/home/milovidov/work/cmake-3.22.2-linux-x86_64/bin/:${PATH} -hash cmake +mkdir build +cmake -S . -B build +cmake --build build ``` ## You Don’t Have to Build ClickHouse {#you-dont-have-to-build-clickhouse} diff --git a/docs/en/engines/table-engines/integrations/s3.md b/docs/en/engines/table-engines/integrations/s3.md index cde09d79cd8..cf887a498ea 100644 --- a/docs/en/engines/table-engines/integrations/s3.md +++ b/docs/en/engines/table-engines/integrations/s3.md @@ -155,6 +155,9 @@ The following settings can be specified in configuration file for given endpoint - `no_sign_request` - Ignore all the credentials so requests are not signed. Useful for accessing public buckets. - `header` — Adds specified HTTP header to a request to given endpoint. Optional, can be specified multiple times. - `server_side_encryption_customer_key_base64` — If specified, required headers for accessing S3 objects with SSE-C encryption will be set. Optional. +- `server_side_encryption_kms_key_id` - If specified, required headers for accessing S3 objects with [SSE-KMS encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html) will be set. If an empty string is specified, the AWS managed S3 key will be used. Optional. +- `server_side_encryption_kms_encryption_context` - If specified alongside `server_side_encryption_kms_key_id`, the given encryption context header for SSE-KMS will be set. Optional. +- `server_side_encryption_kms_bucket_key_enabled` - If specified alongside `server_side_encryption_kms_key_id`, the header to enable S3 bucket keys for SSE-KMS will be set. Optional, can be `true` or `false`, defaults to nothing (matches the bucket-level setting). - `max_single_read_retries` — The maximum number of attempts during single read. Default value is `4`. Optional. - `max_put_rps`, `max_put_burst`, `max_get_rps` and `max_get_burst` - Throttling settings (see description above) to use for specific endpoint instead of per query. Optional. @@ -173,6 +176,9 @@ The following settings can be specified in configuration file for given endpoint + + + diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index d5189d4b9d9..4044087256c 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -439,6 +439,50 @@ Syntax: `ngrambf_v1(n, size_of_bloom_filter_in_bytes, number_of_hash_functions, - `number_of_hash_functions` — The number of hash functions used in the Bloom filter. - `random_seed` — The seed for Bloom filter hash functions. +Users can create [UDF](/docs/en/sql-reference/statements/create/function.md) to estimate the parameters set of `ngrambf_v1`. Query statements are as follows: + +```sql +CREATE FUNCTION bfEstimateFunctions [ON CLUSTER cluster] +AS +(total_nubmer_of_all_grams, size_of_bloom_filter_in_bits) -> round((size_of_bloom_filter_in_bits / total_nubmer_of_all_grams) * log(2)); + +CREATE FUNCTION bfEstimateBmSize [ON CLUSTER cluster] +AS +(total_nubmer_of_all_grams, probability_of_false_positives) -> ceil((total_nubmer_of_all_grams * log(probability_of_false_positives)) / log(1 / pow(2, log(2)))); + +CREATE FUNCTION bfEstimateFalsePositive [ON CLUSTER cluster] +AS +(total_nubmer_of_all_grams, number_of_hash_functions, size_of_bloom_filter_in_bytes) -> pow(1 - exp(-number_of_hash_functions/ (size_of_bloom_filter_in_bytes / total_nubmer_of_all_grams)), number_of_hash_functions); + +CREATE FUNCTION bfEstimateGramNumber [ON CLUSTER cluster] +AS +(number_of_hash_functions, probability_of_false_positives, size_of_bloom_filter_in_bytes) -> ceil(size_of_bloom_filter_in_bytes / (-number_of_hash_functions / log(1 - exp(log(probability_of_false_positives) / number_of_hash_functions)))) + +``` +To use those functions,we need to specify two parameter at least. +For example, if there 4300 ngrams in the granule and we expect false positives to be less than 0.0001. The other parameters can be estimated by executing following queries: + + +```sql +--- estimate number of bits in the filter +SELECT bfEstimateBmSize(4300, 0.0001) / 8 as size_of_bloom_filter_in_bytes; + +┌─size_of_bloom_filter_in_bytes─┐ +│ 10304 │ +└───────────────────────────────┘ + +--- estimate number of hash functions +SELECT bfEstimateFunctions(4300, bfEstimateBmSize(4300, 0.0001)) as number_of_hash_functions + +┌─number_of_hash_functions─┐ +│ 13 │ +└──────────────────────────┘ + +``` +Of course, you can also use those functions to estimate parameters by other conditions. +The functions refer to the content [here](https://hur.st/bloomfilter). + + #### Token Bloom Filter The same as `ngrambf_v1`, but stores tokens instead of ngrams. Tokens are sequences separated by non-alphanumeric characters. @@ -731,7 +775,13 @@ The names given to the described entities can be found in the system tables, [sy ### Configuration {#table_engine-mergetree-multiple-volumes_configure} -Disks, volumes and storage policies should be declared inside the `` tag either in the main file `config.xml` or in a distinct file in the `config.d` directory. +Disks, volumes and storage policies should be declared inside the `` tag either in a file in the `config.d` directory. + +:::tip +Disks can also be declared in the `SETTINGS` section of a query. This is useful +for adhoc analysis to temporarily attach a disk that is, for example, hosted at a URL. +See [dynamic storage](#dynamic-storage) for more details. +::: Configuration structure: @@ -876,6 +926,87 @@ You could change storage policy after table creation with [ALTER TABLE ... MODIF The number of threads performing background moves of data parts can be changed by [background_move_pool_size](/docs/en/operations/server-configuration-parameters/settings.md/#background_move_pool_size) setting. +### Dynamic Storage + +This example query shows how to attach a table stored at a URL and configure the +remote storage within the query. The web storage is not configured in the ClickHouse +configuration files; all the settings are in the CREATE/ATTACH query. + +:::note +The example uses `type=web`, but any disk type can be configured as dynamic, even Local disk. Local disks require a path argument to be inside the server config parameter `custom_local_disks_base_directory`, which has no default, so set that also when using local disk. +::: + +```sql +ATTACH TABLE uk_price_paid UUID 'cf712b4f-2ca8-435c-ac23-c4393efe52f7' +( + price UInt32, + date Date, + postcode1 LowCardinality(String), + postcode2 LowCardinality(String), + type Enum8('other' = 0, 'terraced' = 1, 'semi-detached' = 2, 'detached' = 3, 'flat' = 4), + is_new UInt8, + duration Enum8('unknown' = 0, 'freehold' = 1, 'leasehold' = 2), + addr1 String, + addr2 String, + street LowCardinality(String), + locality LowCardinality(String), + town LowCardinality(String), + district LowCardinality(String), + county LowCardinality(String) +) +ENGINE = MergeTree +ORDER BY (postcode1, postcode2, addr1, addr2) + # highlight-start + SETTINGS disk = disk( + type=web, + endpoint='https://raw.githubusercontent.com/ClickHouse/web-tables-demo/main/web/' + ); + # highlight-end +``` + +### Nested Dynamic Storage + +This example query builds on the above dynamic disk configuration and shows how to +use a local disk to cache data from a table stored at a URL. Neither the cache disk +nor the web storage is configured in the ClickHouse configuration files; both are +configured in the CREATE/ATTACH query settings. + +In the settings highlighted below notice that the disk of `type=web` is nested within +the disk of `type=cache`. + +```sql +ATTACH TABLE uk_price_paid UUID 'cf712b4f-2ca8-435c-ac23-c4393efe52f7' +( + price UInt32, + date Date, + postcode1 LowCardinality(String), + postcode2 LowCardinality(String), + type Enum8('other' = 0, 'terraced' = 1, 'semi-detached' = 2, 'detached' = 3, 'flat' = 4), + is_new UInt8, + duration Enum8('unknown' = 0, 'freehold' = 1, 'leasehold' = 2), + addr1 String, + addr2 String, + street LowCardinality(String), + locality LowCardinality(String), + town LowCardinality(String), + district LowCardinality(String), + county LowCardinality(String) +) +ENGINE = MergeTree +ORDER BY (postcode1, postcode2, addr1, addr2) + # highlight-start + SETTINGS disk = disk( + type=cache, + max_size='1Gi', + path='/var/lib/clickhouse/custom_disk_cache/', + disk=disk( + type=web, + endpoint='https://raw.githubusercontent.com/ClickHouse/web-tables-demo/main/web/' + ) + ); + # highlight-end +``` + ### Details {#details} In the case of `MergeTree` tables, data is getting to disk in different ways: @@ -924,7 +1055,11 @@ Configuration markup: your_access_key_id your_secret_access_key +
Authorization: Bearer SOME-TOKEN
your_base64_encoded_customer_key + your_kms_key_id + your_kms_encryption_context + true http://proxy1 http://proxy2 @@ -975,7 +1110,11 @@ Optional parameters: - `min_bytes_for_seek` — Minimal number of bytes to use seek operation instead of sequential read. Default value is `1 Mb`. - `metadata_path` — Path on local FS to store metadata files for S3. Default value is `/var/lib/clickhouse/disks//`. - `skip_access_check` — If true, disk access checks will not be performed on disk start-up. Default value is `false`. +- `header` — Adds specified HTTP header to a request to given endpoint. Optional, can be specified multiple times. - `server_side_encryption_customer_key_base64` — If specified, required headers for accessing S3 objects with SSE-C encryption will be set. +- `server_side_encryption_kms_key_id` - If specified, required headers for accessing S3 objects with [SSE-KMS encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html) will be set. If an empty string is specified, the AWS managed S3 key will be used. Optional. +- `server_side_encryption_kms_encryption_context` - If specified alongside `server_side_encryption_kms_key_id`, the given encryption context header for SSE-KMS will be set. Optional. +- `server_side_encryption_kms_bucket_key_enabled` - If specified alongside `server_side_encryption_kms_key_id`, the header to enable S3 bucket keys for SSE-KMS will be set. Optional, can be `true` or `false`, defaults to nothing (matches the bucket-level setting). - `s3_max_put_rps` — Maximum PUT requests per second rate before throttling. Default value is `0` (unlimited). - `s3_max_put_burst` — Max number of requests that can be issued simultaneously before hitting request per second limit. By default (`0` value) equals to `s3_max_put_rps`. - `s3_max_get_rps` — Maximum GET requests per second rate before throttling. Default value is `0` (unlimited). diff --git a/docs/en/interfaces/cli.md b/docs/en/interfaces/cli.md index 4ebaaf8a82d..9bf4a465962 100644 --- a/docs/en/interfaces/cli.md +++ b/docs/en/interfaces/cli.md @@ -119,7 +119,7 @@ When processing a query, the client shows: 1. Progress, which is updated no more than 10 times per second (by default). For quick queries, the progress might not have time to be displayed. 2. The formatted query after parsing, for debugging. 3. The result in the specified format. -4. The number of lines in the result, the time passed, and the average speed of query processing. +4. The number of lines in the result, the time passed, and the average speed of query processing. All data amounts refer to uncompressed data. You can cancel a long query by pressing Ctrl+C. However, you will still need to wait for a little for the server to abort the request. It is not possible to cancel a query at certain stages. If you do not wait and press Ctrl+C a second time, the client will exit. diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 02145a2fb6c..267f37fd075 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -1324,7 +1324,7 @@ The trailing slash is mandatory. /var/lib/clickhouse/ ``` -## prometheus {#server_configuration_parameters-prometheus} +## Prometheus {#server_configuration_parameters-prometheus} Exposing metrics data for scraping from [Prometheus](https://prometheus.io). @@ -1339,13 +1339,25 @@ Settings: **Example** ``` xml - - /metrics - 9363 - true - true - true - + + 0.0.0.0 + 8123 + 9000 + + + /metrics + 9363 + true + true + true + + + +``` + +Check (replace `127.0.0.1` with the IP addr or hostname of your ClickHouse server): +```bash +curl 127.0.0.1:9363/metrics ``` ## query_log {#server_configuration_parameters-query-log} @@ -2056,3 +2068,20 @@ Possible values: - Positive integer. Default value: `10000`. + +## display_secrets_in_show_and_select {#display_secrets_in_show_and_select} + +Enables or disables showing secrets in `SHOW` and `SELECT` queries for tables, databases, +table functions, and dictionaries. + +User wishing to see secrets must also have +[`format_display_secrets_in_show_and_select` format setting](../settings/formats#format_display_secrets_in_show_and_select) +turned on and a +[`displaySecretsInShowAndSelect`](../../sql-reference/statements/grant#grant-display-secrets) privilege. + +Possible values: + +- 0 — Disabled. +- 1 — Enabled. + +Default value: 0. diff --git a/docs/en/operations/settings/constraints-on-settings.md b/docs/en/operations/settings/constraints-on-settings.md index 83ef46053a4..1895a79cd3e 100644 --- a/docs/en/operations/settings/constraints-on-settings.md +++ b/docs/en/operations/settings/constraints-on-settings.md @@ -40,7 +40,7 @@ If the user tries to violate the constraints an exception is thrown and the sett There are supported few types of constraints: `min`, `max`, `readonly` (with alias `const`) and `changeable_in_readonly`. The `min` and `max` constraints specify upper and lower boundaries for a numeric setting and can be used in combination. The `readonly` or `const` constraint specifies that the user cannot change the corresponding setting at all. The `changeable_in_readonly` constraint type allows user to change the setting within `min`/`max` range even if `readonly` setting is set to 1, otherwise settings are not allow to be changed in `readonly=1` mode. Note that `changeable_in_readonly` is supported only if `settings_constraints_replace_previous` is enabled: ``` xml - true + true ``` diff --git a/docs/en/operations/settings/settings-formats.md b/docs/en/operations/settings/settings-formats.md index ef4bbeeba89..3b87b829c92 100644 --- a/docs/en/operations/settings/settings-formats.md +++ b/docs/en/operations/settings/settings-formats.md @@ -7,6 +7,23 @@ toc_max_heading_level: 2 # Format settings {#format-settings} +## format_display_secrets_in_show_and_select {#format_display_secrets_in_show_and_select} + +Enables or disables showing secrets in `SHOW` and `SELECT` queries for tables, databases, +table functions, and dictionaries. + +User wishing to see secrets must also have +[`display_secrets_in_show_and_select` server setting](../server-configuration-parameters/settings#display_secrets_in_show_and_select) +turned on and a +[`displaySecretsInShowAndSelect`](../../sql-reference/statements/grant#grant-display-secrets) privilege. + +Possible values: + +- 0 — Disabled. +- 1 — Enabled. + +Default value: 0. + ## input_format_skip_unknown_fields {#input_format_skip_unknown_fields} Enables or disables skipping insertion of extra data. diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 66a24ac3fca..c6fdcf317c3 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -890,7 +890,7 @@ Write time that processor spent during execution/waiting for data to `system.pro See also: -- [`system.processors_profile_log`](../../operations/system-tables/processors_profile_log.md#system-processors_profile_log) +- [`system.processors_profile_log`](../../operations/system-tables/processors_profile_log.md) - [`EXPLAIN PIPELINE`](../../sql-reference/statements/explain.md#explain-pipeline) ## max_insert_block_size {#settings-max_insert_block_size} diff --git a/docs/en/operations/system-tables/clusters.md b/docs/en/operations/system-tables/clusters.md index 4b1e75c25a1..deb9a0aaeb3 100644 --- a/docs/en/operations/system-tables/clusters.md +++ b/docs/en/operations/system-tables/clusters.md @@ -20,6 +20,9 @@ Columns: - `errors_count` ([UInt32](../../sql-reference/data-types/int-uint.md)) — The number of times this host failed to reach replica. - `slowdowns_count` ([UInt32](../../sql-reference/data-types/int-uint.md)) — The number of slowdowns that led to changing replica when establishing a connection with hedged requests. - `estimated_recovery_time` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Seconds remaining until the replica error count is zeroed and it is considered to be back to normal. +- `database_shard_name` ([String](../../sql-reference/data-types/string.md)) — The name of the `Replicated` database shard (for clusters that belong to a `Replicated` database). +- `database_replica_name` ([String](../../sql-reference/data-types/string.md)) — The name of the `Replicated` database replica (for clusters that belong to a `Replicated` database). +- `is_active` ([Nullable(UInt8)](../../sql-reference/data-types/int-uint.md)) — The status of the `Replicated` database replica (for clusters that belong to a `Replicated` database): 1 means "replica is online", 0 means "replica is offline", `NULL` means "unknown". **Example** @@ -47,6 +50,9 @@ default_database: errors_count: 0 slowdowns_count: 0 estimated_recovery_time: 0 +database_shard_name: +database_replica_name: +is_active: NULL Row 2: ────── @@ -63,6 +69,9 @@ default_database: errors_count: 0 slowdowns_count: 0 estimated_recovery_time: 0 +database_shard_name: +database_replica_name: +is_active: NULL ``` **See Also** diff --git a/docs/en/operations/system-tables/processors_profile_log.md b/docs/en/operations/system-tables/processors_profile_log.md index e849525e495..a6ff15642a1 100644 --- a/docs/en/operations/system-tables/processors_profile_log.md +++ b/docs/en/operations/system-tables/processors_profile_log.md @@ -1,4 +1,4 @@ -# system.processors_profile_log {#system-processors_profile_log} +# processors_profile_log This table contains profiling on processors level (that you can find in [`EXPLAIN PIPELINE`](../../sql-reference/statements/explain.md#explain-pipeline)). @@ -73,4 +73,4 @@ Here you can see: **See Also** -- [`EXPLAIN PIPELINE`](../../sql-reference/statements/explain.md#explain-pipeline) \ No newline at end of file +- [`EXPLAIN PIPELINE`](../../sql-reference/statements/explain.md#explain-pipeline) diff --git a/docs/en/operations/system-tables/users.md b/docs/en/operations/system-tables/users.md index a90fa01a45d..58cdb82d31f 100644 --- a/docs/en/operations/system-tables/users.md +++ b/docs/en/operations/system-tables/users.md @@ -12,7 +12,7 @@ Columns: - `storage` ([String](../../sql-reference/data-types/string.md)) — Path to the storage of users. Configured in the `access_control_path` parameter. -- `auth_type` ([Enum8](../../sql-reference/data-types/enum.md)('no_password' = 0,'plaintext_password' = 1, 'sha256_password' = 2, 'double_sha1_password' = 3, 'ldap' = 4, 'kerberos' = 5, 'ssl_certificate' = 6)) — Shows the authentication type. There are multiple ways of user identification: with no password, with plain text password, with [SHA256](https://ru.wikipedia.org/wiki/SHA-2)-encoded password or with [double SHA-1](https://ru.wikipedia.org/wiki/SHA-1)-encoded password. +- `auth_type` ([Enum8](../../sql-reference/data-types/enum.md)('no_password' = 0, 'plaintext_password' = 1, 'sha256_password' = 2, 'double_sha1_password' = 3, 'ldap' = 4, 'kerberos' = 5, 'ssl_certificate' = 6, 'bcrypt_password' = 7)) — Shows the authentication type. There are multiple ways of user identification: with no password, with plain text password, with [SHA256](https://en.wikipedia.org/wiki/SHA-2)-encoded password, with [double SHA-1](https://en.wikipedia.org/wiki/SHA-1)-encoded password or with [bcrypt](https://en.wikipedia.org/wiki/Bcrypt)-encoded password. - `auth_params` ([String](../../sql-reference/data-types/string.md)) — Authentication parameters in the JSON format depending on the `auth_type`. diff --git a/docs/en/operations/utilities/clickhouse-local.md b/docs/en/operations/utilities/clickhouse-local.md index f64d8337387..a7ecadf19fa 100644 --- a/docs/en/operations/utilities/clickhouse-local.md +++ b/docs/en/operations/utilities/clickhouse-local.md @@ -41,9 +41,9 @@ If the file is sitting on the same machine as `clickhouse-local`, use the `file` ``` ClickHouse knows the file uses a tab-separated format from filename extension. If you need to explicitly specify the format, simply add one of the [many ClickHouse input formats](../../interfaces/formats.md): - ```bash - ./clickhouse local -q "SELECT * FROM file('reviews.tsv', 'TabSeparated')" - ``` +```bash +./clickhouse local -q "SELECT * FROM file('reviews.tsv', 'TabSeparated')" +``` The `file` table function creates a table, and you can use `DESCRIBE` to see the inferred schema: diff --git a/docs/en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md b/docs/en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md new file mode 100644 index 00000000000..3da9645181e --- /dev/null +++ b/docs/en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md @@ -0,0 +1,118 @@ +--- +slug: /en/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest +sidebar_position: 300 +sidebar_label: kolmogorovSmirnovTest +--- + +# kolmogorovSmirnovTest + +Applies Kolmogorov-Smirnov's test to samples from two populations. + +**Syntax** + +``` sql +kolmogorovSmirnovTest([alternative, computation_method])(sample_data, sample_index) +``` + +Values of both samples are in the `sample_data` column. If `sample_index` equals to 0 then the value in that row belongs to the sample from the first population. Otherwise it belongs to the sample from the second population. +Samples must belong to continuous, one-dimensional probability distributions. + +**Arguments** + +- `sample_data` — Sample data. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md). +- `sample_index` — Sample index. [Integer](../../../sql-reference/data-types/int-uint.md). + +**Parameters** + +- `alternative` — alternative hypothesis. (Optional, default: `'two-sided'`.) [String](../../../sql-reference/data-types/string.md). + Let F(x) and G(x) be the CDFs of the first and second distributions respectively. + - `'two-sided'` + The null hypothesis is that samples come from the same distribution, e.g. F(x) = G(x) for all x. + And the alternative is that the distributions are not identical. + - `'greater'` + The null hypothesis is that values in the first sample are *stohastically smaller* than those in the second one, + e.g. the CDF of first distribution lies above and hence to the left of that for the second one. + Which in fact means that F(x) >= G(x) for all x. And the alternative in this case is that F(x) < G(x) for at least one x. + - `'less'`. + The null hypothesis is that values in the first sample are *stohastically greater* than those in the second one, + e.g. the CDF of first distribution lies below and hence to the right of that for the second one. + Which in fact means that F(x) <= G(x) for all x. And the alternative in this case is that F(x) > G(x) for at least one x. +- `computation_method` — the method used to compute p-value. (Optional, default: `'auto'`.) [String](../../../sql-reference/data-types/string.md). + - `'exact'` - calculation is performed using precise probability distribution of the test statistics. Compute intensive and wasteful except for small samples. + - `'asymp'` (`'asymptotic'`) - calculation is performed using an approximation. For large sample sizes, the exact and asymptotic p-values are very similar. + - `'auto'` - the `'exact'` method is used when a maximum number of samples is less than 10'000. + + +**Returned values** + +[Tuple](../../../sql-reference/data-types/tuple.md) with two elements: + +- calculated statistic. [Float64](../../../sql-reference/data-types/float.md). +- calculated p-value. [Float64](../../../sql-reference/data-types/float.md). + + +**Example** + +Query: + +``` sql +SELECT kolmogorovSmirnovTest('less', 'exact')(value, num) +FROM +( + SELECT + randNormal(0, 10) AS value, + 0 AS num + FROM numbers(10000) + UNION ALL + SELECT + randNormal(0, 10) AS value, + 1 AS num + FROM numbers(10000) +) +``` + +Result: + +``` text +┌─kolmogorovSmirnovTest('less', 'exact')(value, num)─┐ +│ (0.009899999999999996,0.37528595205132287) │ +└────────────────────────────────────────────────────┘ +``` + +Note: +P-value is bigger than 0.05 (for confidence level of 95%), so null hypothesis is not rejected. + + +Query: + +``` sql +SELECT kolmogorovSmirnovTest('two-sided', 'exact')(value, num) +FROM +( + SELECT + randStudentT(10) AS value, + 0 AS num + FROM numbers(100) + UNION ALL + SELECT + randNormal(0, 10) AS value, + 1 AS num + FROM numbers(100) +) +``` + +Result: + +``` text +┌─kolmogorovSmirnovTest('two-sided', 'exact')(value, num)─┐ +│ (0.4100000000000002,6.61735760482795e-8) │ +└─────────────────────────────────────────────────────────┘ +``` + +Note: +P-value is less than 0.05 (for confidence level of 95%), so null hypothesis is rejected. + + +**See Also** + +- [Kolmogorov-Smirnov'test](https://en.wikipedia.org/wiki/Kolmogorov%E2%80%93Smirnov_test) diff --git a/docs/en/sql-reference/data-types/index.md b/docs/en/sql-reference/data-types/index.md index c61a3069db6..2ad8ac4bb23 100644 --- a/docs/en/sql-reference/data-types/index.md +++ b/docs/en/sql-reference/data-types/index.md @@ -27,7 +27,7 @@ ClickHouse data types include: - **Aggregation function types**: use [`SimpleAggregateFunction`](./simpleaggregatefunction.md) and [`AggregateFunction`](./aggregatefunction.md) for storing the intermediate status of aggregate function results - **Nested data structures**: A [`Nested` data structure](./nested-data-structures/index.md) is like a table inside a cell - **Tuples**: A [`Tuple` of elements](./tuple.md), each having an individual type. -- **Nullable**: [`Nullbale`](./nullable.md) allows you to store a value as `NULL` when a value is "missing" (instead of the column gettings its default value for the data type) +- **Nullable**: [`Nullable`](./nullable.md) allows you to store a value as `NULL` when a value is "missing" (instead of the column gettings its default value for the data type) - **IP addresses**: use [`IPv4`](./domains/ipv4.md) and [`IPv6`](./domains/ipv6.md) to efficiently store IP addresses - **Geo types**: for[ geographical data](./geo.md), including `Point`, `Ring`, `Polygon` and `MultiPolygon` - **Special data types**: including [`Expression`](./special-data-types/expression.md), [`Set`](./special-data-types/set.md), [`Nothing`](./special-data-types/nothing.md) and [`Interval`](./special-data-types/interval.md) \ No newline at end of file diff --git a/docs/en/sql-reference/data-types/nullable.md b/docs/en/sql-reference/data-types/nullable.md index 230b4af7960..28180f7f991 100644 --- a/docs/en/sql-reference/data-types/nullable.md +++ b/docs/en/sql-reference/data-types/nullable.md @@ -8,7 +8,7 @@ sidebar_label: Nullable Allows to store special marker ([NULL](../../sql-reference/syntax.md)) that denotes “missing value” alongside normal values allowed by `TypeName`. For example, a `Nullable(Int8)` type column can store `Int8` type values, and the rows that do not have a value will store `NULL`. -For a `TypeName`, you can’t use composite data types [Array](../../sql-reference/data-types/array.md) and [Tuple](../../sql-reference/data-types/tuple.md). Composite data types can contain `Nullable` type values, such as `Array(Nullable(Int8))`. +For a `TypeName`, you can’t use composite data types [Array](../../sql-reference/data-types/array.md), [Map](../../sql-reference/data-types/map.md) and [Tuple](../../sql-reference/data-types/tuple.md). Composite data types can contain `Nullable` type values, such as `Array(Nullable(Int8))`. A `Nullable` type field can’t be included in table indexes. diff --git a/docs/en/sql-reference/dictionaries/index.md b/docs/en/sql-reference/dictionaries/index.md index 48a8ce45d33..189673cdae7 100644 --- a/docs/en/sql-reference/dictionaries/index.md +++ b/docs/en/sql-reference/dictionaries/index.md @@ -1658,6 +1658,7 @@ Example of settings: test dictionary_source + ssl=true ``` @@ -1672,6 +1673,7 @@ SOURCE(MONGODB( password '' db 'test' collection 'dictionary_source' + options 'ssl=true' )) ``` @@ -1683,6 +1685,8 @@ Setting fields: - `password` – Password of the MongoDB user. - `db` – Name of the database. - `collection` – Name of the collection. +- `options` - MongoDB connection string options (optional parameter). + ### Redis diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index c22b46a7eea..15644b54c2b 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -24,6 +24,90 @@ SELECT └─────────────────────┴────────────┴────────────┴─────────────────────┘ ``` +## makeDate + +Creates a [Date](../../sql-reference/data-types/date.md) from a year, month and day argument. + +**Syntax** + +``` sql +makeDate(year, month, day) +``` + +**Arguments** + +- `year` — Year. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). +- `month` — Month. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). +- `day` — Day. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). + +**Returned value** + +- A date created from the arguments. + +Type: [Date](../../sql-reference/data-types/date.md). + +**Example** + +``` sql +SELECT makeDate(2023, 2, 28) AS Date; +``` + +Result: + +``` text +┌───────date─┐ +│ 2023-02-28 │ +└────────────┘ +``` + +## makeDate32 + +Like [makeDate](#makeDate) but produces a [Date32](../../sql-reference/data-types/date32.md). + +## makeDateTime + +Creates a [DateTime](../../sql-reference/data-types/datetime.md) from a year, month, day, hour, minute and second argument. + +**Syntax** + +``` sql +makeDateTime(year, month, day, hour, minute, second[, timezone]) +``` + +**Arguments** + +- `year` — Year. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). +- `month` — Month. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). +- `day` — Day. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). +- `hour` — Hour. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). +- `minute` — Minute. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). +- `second` — Second. [Integer](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md) or [Decimal](../../sql-reference/data-types/decimal.md). +- `timezone` — [Timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone) for the returned value (optional). + +**Returned value** + +- A date with time created from the arguments. + +Type: [DateTime](../../sql-reference/data-types/datetime.md). + +**Example** + +``` sql +SELECT makeDateTime(2023, 2, 28, 17, 12, 33) AS DateTime; +``` + +Result: + +``` text +┌────────────DateTime─┐ +│ 2023-02-28 17:12:33 │ +└─────────────────────┘ +``` + +## makeDateTime64 + +Like [makeDateTime](#makedatetime) but produces a [DateTime64](../../sql-reference/data-types/datetime64.md). + ## timeZone Returns the timezone of the server. diff --git a/docs/en/sql-reference/statements/alter/partition.md b/docs/en/sql-reference/statements/alter/partition.md index 2fb20e4e462..52e99d93109 100644 --- a/docs/en/sql-reference/statements/alter/partition.md +++ b/docs/en/sql-reference/statements/alter/partition.md @@ -109,7 +109,7 @@ For the query to run successfully, the following conditions must be met: - Both tables must have the same structure. - Both tables must have the same partition key, the same order by key and the same primary key. -- Both tables must have the same storage policy (a disk where the partition is stored should be available for both tables). +- Both tables must have the same storage policy. ## REPLACE PARTITION @@ -123,7 +123,7 @@ For the query to run successfully, the following conditions must be met: - Both tables must have the same structure. - Both tables must have the same partition key, the same order by key and the same primary key. -- Both tables must have the same storage policy (a disk where the partition is stored should be available for both tables). +- Both tables must have the same storage policy. ## MOVE PARTITION TO TABLE @@ -137,7 +137,7 @@ For the query to run successfully, the following conditions must be met: - Both tables must have the same structure. - Both tables must have the same partition key, the same order by key and the same primary key. -- Both tables must have the same storage policy (a disk where the partition is stored should be available for both tables). +- Both tables must have the same storage policy. - Both tables must be the same engine family (replicated or non-replicated). ## CLEAR COLUMN IN PARTITION diff --git a/docs/en/sql-reference/statements/create/user.md b/docs/en/sql-reference/statements/create/user.md index 3548ef7cc07..d168be63c36 100644 --- a/docs/en/sql-reference/statements/create/user.md +++ b/docs/en/sql-reference/statements/create/user.md @@ -32,9 +32,12 @@ There are multiple ways of user identification: - `IDENTIFIED WITH sha256_hash BY 'hash'` or `IDENTIFIED WITH sha256_hash BY 'hash' SALT 'salt'` - `IDENTIFIED WITH double_sha1_password BY 'qwerty'` - `IDENTIFIED WITH double_sha1_hash BY 'hash'` +- `IDENTIFIED WITH bcrypt_password BY 'qwerty'` +- `IDENTIFIED WITH bcrypt_hash BY 'hash'` - `IDENTIFIED WITH ldap SERVER 'server_name'` - `IDENTIFIED WITH kerberos` or `IDENTIFIED WITH kerberos REALM 'realm'` - `IDENTIFIED WITH ssl_certificate CN 'mysite.com:user'` +- `IDENTIFIED BY 'qwerty'` ## Examples @@ -54,21 +57,12 @@ There are multiple ways of user identification: The password is stored in a SQL text file in `/var/lib/clickhouse/access`, so it's not a good idea to use `plaintext_password`. Try `sha256_password` instead, as demonstrated next... ::: -3. The best option is to use a password that is hashed using SHA-256. ClickHouse will hash the password for you when you specify `IDENTIFIED WITH sha256_password`. For example: +3. The most common option is to use a password that is hashed using SHA-256. ClickHouse will hash the password for you when you specify `IDENTIFIED WITH sha256_password`. For example: ```sql CREATE USER name3 IDENTIFIED WITH sha256_password BY 'my_password' ``` - Notice ClickHouse generates and runs the following command for you: - - ```response - CREATE USER name3 - IDENTIFIED WITH sha256_hash - BY '8B3404953FCAA509540617F082DB13B3E0734F90FF6365C19300CC6A6EA818D6' - SALT 'D6489D8B5692D82FF944EA6415785A8A8A1AF33825456AFC554487725A74A609' - ``` - The `name3` user can now login using `my_password`, but the password is stored as the hashed value above. THe following SQL file was created in `/var/lib/clickhouse/access` and gets executed at server startup: ```bash @@ -92,6 +86,34 @@ There are multiple ways of user identification: CREATE USER name4 IDENTIFIED WITH double_sha1_hash BY 'CCD3A959D6A004B9C3807B728BC2E55B67E10518' ``` +5. The `bcrypt_password` is the most secure option for storing passwords. It uses the [bcrypt](https://en.wikipedia.org/wiki/Bcrypt) algorithm, which is resilient against brute force attacks even if the password hash is compromised. + + ```sql + CREATE USER name5 IDENTIFIED WITH bcrypt_password BY 'my_password' + ``` + + The length of the password is limited to 72 characters with this method. The bcrypt work factor parameter, which defines the amount of computations and time needed to compute the hash and verify the password, can be modified in the server configuration: + + ```xml + 12 + ``` + + The work factor must be between 4 and 31, with a default value of 12. + +6. The type of the password can also be omitted: + + ```sql + CREATE USER name6 IDENTIFIED BY 'my_password' + ``` + + In this case, ClickHouse will use the default password type specified in the server configuration: + + ```xml + sha256_password + ``` + + The available password types are: `plaintext_password`, `sha256_password`, `double_sha1_password`. + ## User Host User host is a host from which a connection to ClickHouse server could be established. The host can be specified in the `HOST` query section in the following ways: diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md index 8fa3b5de1b8..b6d6f285f3d 100644 --- a/docs/en/sql-reference/statements/grant.md +++ b/docs/en/sql-reference/statements/grant.md @@ -36,6 +36,18 @@ GRANT [ON CLUSTER cluster_name] role [,...] TO {user | another_role | CURRENT_US The `WITH ADMIN OPTION` clause grants [ADMIN OPTION](#admin-option-privilege) privilege to `user` or `role`. The `WITH REPLACE OPTION` clause replace old roles by new role for the `user` or `role`, if is not specified it appends roles. +## Grant Current Grants Syntax +``` sql +GRANT CURRENT GRANTS{(privilege[(column_name [,...])] [,...] ON {db.table|db.*|*.*|table|*}) | ON {db.table|db.*|*.*|table|*}} TO {user | role | CURRENT_USER} [,...] [WITH GRANT OPTION] [WITH REPLACE OPTION] +``` + +- `privilege` — Type of privilege. +- `role` — ClickHouse user role. +- `user` — ClickHouse user account. + +Using the `CURRENT GRANTS` statement allows you to give all specified privileges to the given user or role. +If none of the privileges were specified, then the given user or role will receive all available privileges for `CURRENT_USER`. + ## Usage To use `GRANT`, your account must have the `GRANT OPTION` privilege. You can grant privileges only inside the scope of your account privileges. @@ -188,6 +200,7 @@ Hierarchy of privileges: - `HDFS` - `S3` - [dictGet](#grant-dictget) +- [displaySecretsInShowAndSelect](#grant-display-secrets) Examples of how this hierarchy is treated: @@ -473,6 +486,15 @@ Privilege level: `DICTIONARY`. - `GRANT dictGet ON mydb.mydictionary TO john` - `GRANT dictGet ON mydictionary TO john` + +### displaySecretsInShowAndSelect {#grant-display-secrets} + +Allows a user to view secrets in `SHOW` and `SELECT` queries if both +[`display_secrets_in_show_and_select` server setting](../../operations/server-configuration-parameters/settings#display_secrets_in_show_and_select) +and +[`format_display_secrets_in_show_and_select` format setting](../../operations/settings/formats#format_display_secrets_in_show_and_select) +are turned on. + ### ALL Grants all the privileges on regulated entity to a user account or a role. diff --git a/docs/en/sql-reference/statements/show.md b/docs/en/sql-reference/statements/show.md index 428a04ae030..ed3f8a074c8 100644 --- a/docs/en/sql-reference/statements/show.md +++ b/docs/en/sql-reference/statements/show.md @@ -6,6 +6,13 @@ sidebar_label: SHOW # SHOW Statements +N.B. `SHOW CREATE (TABLE|DATABASE|USER)` hides secrets unless +[`display_secrets_in_show_and_select` server setting](../../operations/server-configuration-parameters/settings#display_secrets_in_show_and_select) +is turned on, +[`format_display_secrets_in_show_and_select` format setting](../../operations/settings/formats#format_display_secrets_in_show_and_select) +is turned on and user has +[`displaySecretsInShowAndSelect`](grant.md#grant-display-secrets) privilege. + ## SHOW CREATE TABLE | DICTIONARY | VIEW | DATABASE ``` sql @@ -293,8 +300,6 @@ If user is not specified, the query returns privileges for the current user. Shows parameters that were used at a [user creation](../../sql-reference/statements/create/user.md). -`SHOW CREATE USER` does not output user passwords. - **Syntax** ``` sql diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index 5a5a771f239..c5596b7ba5f 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -76,7 +76,7 @@ Resets the mark cache. ## DROP REPLICA -Dead replicas can be dropped using following syntax: +Dead replicas of `ReplicatedMergeTree` tables can be dropped using following syntax: ``` sql SYSTEM DROP REPLICA 'replica_name' FROM TABLE database.table; @@ -85,13 +85,25 @@ SYSTEM DROP REPLICA 'replica_name'; SYSTEM DROP REPLICA 'replica_name' FROM ZKPATH '/path/to/table/in/zk'; ``` -Queries will remove the replica path in ZooKeeper. It is useful when the replica is dead and its metadata cannot be removed from ZooKeeper by `DROP TABLE` because there is no such table anymore. It will only drop the inactive/stale replica, and it cannot drop local replica, please use `DROP TABLE` for that. `DROP REPLICA` does not drop any tables and does not remove any data or metadata from disk. +Queries will remove the `ReplicatedMergeTree` replica path in ZooKeeper. It is useful when the replica is dead and its metadata cannot be removed from ZooKeeper by `DROP TABLE` because there is no such table anymore. It will only drop the inactive/stale replica, and it cannot drop local replica, please use `DROP TABLE` for that. `DROP REPLICA` does not drop any tables and does not remove any data or metadata from disk. The first one removes metadata of `'replica_name'` replica of `database.table` table. The second one does the same for all replicated tables in the database. The third one does the same for all replicated tables on the local server. The fourth one is useful to remove metadata of dead replica when all other replicas of a table were dropped. It requires the table path to be specified explicitly. It must be the same path as was passed to the first argument of `ReplicatedMergeTree` engine on table creation. +## DROP DATABASE REPLICA + +Dead replicas of `Replicated` databases can be dropped using following syntax: + +``` sql +SYSTEM DROP DATABASE REPLICA 'replica_name' [FROM SHARD 'shard_name'] FROM DATABASE database; +SYSTEM DROP DATABASE REPLICA 'replica_name' [FROM SHARD 'shard_name']; +SYSTEM DROP DATABASE REPLICA 'replica_name' [FROM SHARD 'shard_name'] FROM ZKPATH '/path/to/table/in/zk'; +``` + +Similar to `SYSTEM DROP REPLICA`, but removes the `Replicated` database replica path from ZooKeeper when there's no database to run `DROP DATABASE`. Please note that it does not remove `ReplicatedMergeTree` replicas (so you may need `SYSTEM DROP REPLICA` as well). Shard and replica names are the names that were specified in `Replicated` engine arguments when creating the database. Also, these names can be obtained from `database_shard_name` and `database_replica_name` columns in `system.clusters`. If the `FROM SHARD` clause is missing, then `replica_name` must be a full replica name in `shard_name|replica_name` format. + ## DROP UNCOMPRESSED CACHE Reset the uncompressed data cache. diff --git a/docs/ru/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md b/docs/ru/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md new file mode 100644 index 00000000000..2f8c6bb6760 --- /dev/null +++ b/docs/ru/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest.md @@ -0,0 +1,117 @@ +--- +slug: /ru/sql-reference/aggregate-functions/reference/kolmogorovsmirnovtest +sidebar_position: 300 +sidebar_label: kolmogorovSmirnovTest +--- + +# kolmogorovSmirnovTest {#kolmogorovSmirnovTest} + +Проводит статистический тест Колмогорова-Смирнова для двух независимых выборок. + +**Синтаксис** + +``` sql +kolmogorovSmirnovTest([alternative, computation_method])(sample_data, sample_index) +``` + +Значения выборок берутся из столбца `sample_data`. Если `sample_index` равно 0, то значение из этой строки принадлежит первой выборке. Во всех остальных случаях значение принадлежит второй выборке. +Выборки должны принадлежать непрерывным одномерным распределениям. + +**Аргументы** + +- `sample_data` — данные выборок. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md). +- `sample_index` — индексы выборок. [Integer](../../../sql-reference/data-types/int-uint.md). + +**Параметры** + +- `alternative` — альтернативная гипотеза (Необязательный параметр, по умолчанию: `'two-sided'`.) [String](../../../sql-reference/data-types/string.md). + Пусть F(x) и G(x) - функции распределения первой и второй выборки соотвественно. + - `'two-sided'` + Нулевая гипотеза состоит в том, что выборки происходит из одного и того же распределение, то есть F(x) = G(x) для любого x. + Альтернатива - выборки принадлежат разным распределениям. + - `'greater'` + Нулевая гипотеза состоит в том, что элементы первой выборки в асимптотически почти наверное меньше элементов из второй выборки, + то есть функция распределения первой выборки лежит выше и соотвественно левее, чем функция распределения второй выборки. + Таким образом это означает, что F(x) >= G(x) for любого x, а альтернатива в этом случае состоит в том, что F(x) < G(x) хотя бы для одного x. + - `'less'`. + Нулевая гипотеза состоит в том, что элементы первой выборки в асимптотически почти наверное больше элементов из второй выборки, + то есть функция распределения первой выборки лежит ниже и соотвественно правее, чем функция распределения второй выборки. + Таким образом это означает, что F(x) <= G(x) for любого x, а альтернатива в этом случае состоит в том, что F(x) > G(x) хотя бы для одного x. +- `computation_method` — метод, используемый для вычисления p-value. (Необязательный параметр, по умолчанию: `'auto'`.) [String](../../../sql-reference/data-types/string.md). + - `'exact'` - вычисление производится с помощью вычисления точного распределения статистики. Требует большого количества вычислительных ресурсов и расточительно для больших выборок. + - `'asymp'`(`'asymptotic'`) - используется приближенное вычисление. Для больших выборок приближенный результат и точный почти идентичны. + - `'auto'` - значение вычисляется точно (с помощью метода `'exact'`), если максимальный размер двух выборок не превышает 10'000. + +**Возвращаемые значения** + +[Кортеж](../../../sql-reference/data-types/tuple.md) с двумя элементами: + +- вычисленное статистики. [Float64](../../../sql-reference/data-types/float.md). +- вычисленное p-value. [Float64](../../../sql-reference/data-types/float.md). + + +**Пример** + +Запрос: + +``` sql +SELECT kolmogorovSmirnovTest('less', 'exact')(value, num) +FROM +( + SELECT + randNormal(0, 10) AS value, + 0 AS num + FROM numbers(10000) + UNION ALL + SELECT + randNormal(0, 10) AS value, + 1 AS num + FROM numbers(10000) +) +``` + +Результат: + +``` text +┌─kolmogorovSmirnovTest('less', 'exact')(value, num)─┐ +│ (0.009899999999999996,0.37528595205132287) │ +└────────────────────────────────────────────────────┘ +``` + +Заметки: +P-value больше чем 0.05 (для уровня значимости 95%), то есть нулевая гипотеза не отвергается. + + +Запрос: + +``` sql +SELECT kolmogorovSmirnovTest('two-sided', 'exact')(value, num) +FROM +( + SELECT + randStudentT(10) AS value, + 0 AS num + FROM numbers(100) + UNION ALL + SELECT + randNormal(0, 10) AS value, + 1 AS num + FROM numbers(100) +) +``` + +Результат: + +``` text +┌─kolmogorovSmirnovTest('two-sided', 'exact')(value, num)─┐ +│ (0.4100000000000002,6.61735760482795e-8) │ +└─────────────────────────────────────────────────────────┘ +``` + +Заметки: +P-value меньше чем 0.05 (для уровня значимости 95%), то есть нулевая гипотеза отвергается. + + +**Смотрите также** + +- [Критерий согласия Колмогорова-Смирнова](https://ru.wikipedia.org/wiki/%D0%9A%D1%80%D0%B8%D1%82%D0%B5%D1%80%D0%B8%D0%B9_%D1%81%D0%BE%D0%B3%D0%BB%D0%B0%D1%81%D0%B8%D1%8F_%D0%9A%D0%BE%D0%BB%D0%BC%D0%BE%D0%B3%D0%BE%D1%80%D0%BE%D0%B2%D0%B0) diff --git a/docs/ru/sql-reference/statements/grant.md b/docs/ru/sql-reference/statements/grant.md index 73c63850750..9b8fafabfcc 100644 --- a/docs/ru/sql-reference/statements/grant.md +++ b/docs/ru/sql-reference/statements/grant.md @@ -37,6 +37,19 @@ GRANT [ON CLUSTER cluster_name] role [,...] TO {user | another_role | CURRENT_US `WITH ADMIN OPTION` присваивает привилегию [ADMIN OPTION](#admin-option-privilege) пользователю или роли. `WITH REPLACE OPTION` заменяет все старые роли новыми ролями для пользователя `user` или `role`, если не указано, добавляет новые новые роли. +## Синтаксис присвоения текущих привилегий {#grant-current-grants-syntax} + +```sql +GRANT CURRENT GRANTS{(privilege[(column_name [,...])] [,...] ON {db.table|db.*|*.*|table|*}) | ON {db.table|db.*|*.*|table|*}} TO {user | role | CURRENT_USER} [,...] [WITH GRANT OPTION] [WITH REPLACE OPTION] +``` + +- `privilege` — Тип привилегии +- `role` — Роль пользователя ClickHouse. +- `user` — Пользователь ClickHouse. + +Использование выражения `CURRENT GRANTS` позволяет присвоить все указанные и доступные для присвоения привилегии. +Если список привелегий не задан, то указанный пользователь или роль получат все доступные привилегии для `CURRENT_USER`. + ## Использование {#grant-usage} Для использования `GRANT` пользователь должен иметь привилегию `GRANT OPTION`. Пользователь может выдавать привилегии только внутри области действий назначенных ему самому привилегий. diff --git a/programs/diagnostics/internal/platform/data/file_test.go b/programs/diagnostics/internal/platform/data/file_test.go index b93c4fc3350..938c34281f1 100644 --- a/programs/diagnostics/internal/platform/data/file_test.go +++ b/programs/diagnostics/internal/platform/data/file_test.go @@ -135,7 +135,7 @@ func TestConfigFileFrameCopy(t *testing.T) { sizes := map[string]int64{ "users.xml": int64(2017), "default-password.xml": int64(188), - "config.xml": int64(61260), + "config.xml": int64(61662), "server-include.xml": int64(168), "user-include.xml": int64(559), } diff --git a/programs/diagnostics/testdata/configs/xml/config.xml b/programs/diagnostics/testdata/configs/xml/config.xml index 18997855955..21a0821f89d 100644 --- a/programs/diagnostics/testdata/configs/xml/config.xml +++ b/programs/diagnostics/testdata/configs/xml/config.xml @@ -1260,8 +1260,12 @@ REPLACE_ME REPLACE_ME +
Authorization: Bearer SOME-TOKEN
your_base64_encoded_customer_key + REPLACE_ME + REPLACE_ME + true http://proxy1 http://proxy2 diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 5768e744f94..8925f50fe97 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -26,12 +26,13 @@ #include #include #include +#include #include #include #include #include #include -#include +#include #include #include #include @@ -133,6 +134,11 @@ void LocalServer::initialize(Poco::Util::Application & self) config().getUInt("max_io_thread_pool_size", 100), config().getUInt("max_io_thread_pool_free_size", 0), config().getUInt("io_thread_pool_queue_size", 10000)); + + OutdatedPartsLoadingThreadPool::initialize( + config().getUInt("max_outdated_parts_loading_thread_pool_size", 16), + 0, // We don't need any threads one all the parts will be loaded + config().getUInt("outdated_part_loading_thread_pool_queue_size", 10000)); } diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 8c0d50bae55..bbd536d9300 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -41,10 +41,9 @@ #include #include #include -#include #include #include -#include +#include #include #include #include @@ -778,6 +777,11 @@ try server_settings.max_backups_io_thread_pool_free_size, server_settings.backups_io_thread_pool_queue_size); + OutdatedPartsLoadingThreadPool::initialize( + server_settings.max_outdated_parts_loading_thread_pool_size, + 0, // We don't need any threads one all the parts will be loaded + server_settings.outdated_part_loading_thread_pool_queue_size); + /// Initialize global local cache for remote filesystem. if (config().has("local_cache_for_remote_fs")) { @@ -1852,7 +1856,7 @@ try LOG_INFO(log, "Closed all listening sockets."); /// Killing remaining queries. - if (server_settings.shutdown_wait_unfinished_queries) + if (!server_settings.shutdown_wait_unfinished_queries) global_context->getProcessList().killAllQueries(); if (current_connections) diff --git a/programs/server/config.xml b/programs/server/config.xml index 7a75d7251a9..51aa04ba0e5 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -476,6 +476,14 @@ 1 1 + + sha256_password + + + 12 + - + \n" + f"This is an automated comment for commit {pr_info.sha} with " + f"description of existing statuses. It's updated for the latest CI running\n" + f"The full report is available [here]({report_url})\n" + f"{worst_state}\n\n" + "\n" + "" + ) + # group checks by the name to get the worst one per each + grouped_statuses = {} # type: Dict[CheckDescription, CommitStatuses] + for status in statuses: + cd = None + for c in CHECK_DESCRIPTIONS: + if c.match_func(status.context): + cd = c + break + + if cd is None or cd == CHECK_DESCRIPTIONS[-1]: + # This is the case for either non-found description or a fallback + cd = CheckDescription( + status.context, + CHECK_DESCRIPTIONS[-1].description, + CHECK_DESCRIPTIONS[-1].match_func, + ) + + if cd in grouped_statuses: + grouped_statuses[cd].append(status) + else: + grouped_statuses[cd] = [status] + + table_rows = [] # type: List[str] + for desc, gs in grouped_statuses.items(): + table_rows.append( + f"" + f"\n" + ) + + table_rows.sort() + + comment_footer = "
Check nameDescriptionStatus
{desc.name}{desc.description}{beauty_state(get_worst_state(gs))}
" + return "".join([comment_body, *table_rows, comment_footer]) + + +def get_worst_state(statuses: CommitStatuses) -> str: + worst_status = None + states = {"error": 0, "failure": 1, "pending": 2, "success": 3} + for status in statuses: + if worst_status is None: + worst_status = status + continue + if states[status.state] < states[worst_status.state]: + worst_status = status + if worst_status.state == "error": + break + + if worst_status is None: + return "" + return worst_status.state + + +def create_ci_report(pr_info: PRInfo, statuses: CommitStatuses) -> str: + """The function converst the statuses to TestResults and uploads the report + to S3 tests bucket. Then it returns the URL""" + test_results = [] # type: TestResults + for status in statuses: + log_urls = None + if status.target_url is not None: + log_urls = [status.target_url] + test_results.append(TestResult(status.context, status.state, log_urls=log_urls)) + return upload_results( + S3Helper(), pr_info.number, pr_info.sha, test_results, [], CI_STATUS_NAME + ) def post_commit_status_to_file( @@ -90,8 +290,16 @@ def get_commit_filtered_statuses(commit: Commit) -> CommitStatuses: return list(filtered.values()) +def get_repo(gh: Github) -> Repository: + global GH_REPO + if GH_REPO is not None: + return GH_REPO + GH_REPO = gh.get_repo(GITHUB_REPOSITORY) + return GH_REPO + + def remove_labels(gh: Github, pr_info: PRInfo, labels_names: List[str]) -> None: - repo = gh.get_repo(GITHUB_REPOSITORY) + repo = get_repo(gh) pull_request = repo.get_pull(pr_info.number) for label in labels_names: pull_request.remove_from_labels(label) @@ -99,7 +307,7 @@ def remove_labels(gh: Github, pr_info: PRInfo, labels_names: List[str]) -> None: def post_labels(gh: Github, pr_info: PRInfo, labels_names: List[str]) -> None: - repo = gh.get_repo(GITHUB_REPOSITORY) + repo = get_repo(gh) pull_request = repo.get_pull(pr_info.number) for label in labels_names: pull_request.add_to_labels(label) diff --git a/tests/ci/compatibility_check.py b/tests/ci/compatibility_check.py index 432e9ec7c01..04203617dca 100644 --- a/tests/ci/compatibility_check.py +++ b/tests/ci/compatibility_check.py @@ -16,13 +16,12 @@ from clickhouse_helper import ( mark_flaky_tests, prepare_tests_results_for_clickhouse, ) -from commit_status_helper import post_commit_status +from commit_status_helper import RerunHelper, get_commit, post_commit_status from docker_pull_helper import get_images_with_versions from env_helper import TEMP_PATH, REPORTS_PATH from get_robot_token import get_best_robot_token from pr_info import PRInfo from report import TestResults, TestResult -from rerun_helper import RerunHelper from s3_helper import S3Helper from stopwatch import Stopwatch from upload_result_helper import upload_results @@ -150,8 +149,9 @@ def main(): pr_info = PRInfo() gh = Github(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) - rerun_helper = RerunHelper(gh, pr_info, args.check_name) + rerun_helper = RerunHelper(commit, args.check_name) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) @@ -242,7 +242,7 @@ def main(): args.check_name, ) print(f"::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, args.check_name, description, state, report_url) + post_commit_status(commit, state, report_url, description, args.check_name, pr_info) prepared_events = prepare_tests_results_for_clickhouse( pr_info, diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index f2b1105b3b0..16a58a90dcf 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -14,7 +14,7 @@ from typing import Any, Dict, List, Optional, Set, Tuple, Union from github import Github from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse -from commit_status_helper import format_description, post_commit_status +from commit_status_helper import format_description, get_commit, post_commit_status from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP, GITHUB_RUN_URL from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo @@ -474,7 +474,8 @@ def main(): return gh = Github(get_best_robot_token(), per_page=100) - post_commit_status(gh, pr_info.sha, NAME, description, status, url) + commit = get_commit(gh, pr_info.sha) + post_commit_status(commit, status, url, description, NAME, pr_info) prepared_events = prepare_tests_results_for_clickhouse( pr_info, diff --git a/tests/ci/docker_manifests_merge.py b/tests/ci/docker_manifests_merge.py index 0484ea8f641..d89708b9277 100644 --- a/tests/ci/docker_manifests_merge.py +++ b/tests/ci/docker_manifests_merge.py @@ -10,7 +10,7 @@ from typing import List, Dict, Tuple from github import Github from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse -from commit_status_helper import format_description, post_commit_status +from commit_status_helper import format_description, get_commit, post_commit_status from env_helper import RUNNER_TEMP from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo @@ -221,7 +221,8 @@ def main(): description = format_description(description) gh = Github(get_best_robot_token(), per_page=100) - post_commit_status(gh, pr_info.sha, NAME, description, status, url) + commit = get_commit(gh, pr_info.sha) + post_commit_status(commit, status, url, description, NAME, pr_info) prepared_events = prepare_tests_results_for_clickhouse( pr_info, diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index c6854c5aa78..a434d3cc841 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -15,7 +15,7 @@ from github import Github from build_check import get_release_or_pr from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse -from commit_status_helper import format_description, post_commit_status +from commit_status_helper import format_description, get_commit, post_commit_status from docker_images_check import DockerImage from env_helper import CI, GITHUB_RUN_URL, RUNNER_TEMP, S3_BUILDS_BUCKET, S3_DOWNLOAD from get_robot_token import get_best_robot_token, get_parameter_from_ssm @@ -372,7 +372,8 @@ def main(): description = format_description(description) gh = Github(get_best_robot_token(), per_page=100) - post_commit_status(gh, pr_info.sha, NAME, description, status, url) + commit = get_commit(gh, pr_info.sha) + post_commit_status(commit, status, url, description, NAME, pr_info) prepared_events = prepare_tests_results_for_clickhouse( pr_info, diff --git a/tests/ci/docs_check.py b/tests/ci/docs_check.py index ed2743ca965..e3930a20bd9 100644 --- a/tests/ci/docs_check.py +++ b/tests/ci/docs_check.py @@ -9,13 +9,18 @@ import sys from github import Github from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse -from commit_status_helper import post_commit_status, get_commit, update_mergeable_check +from commit_status_helper import ( + NotSet, + RerunHelper, + get_commit, + post_commit_status, + update_mergeable_check, +) from docker_pull_helper import get_image_with_version from env_helper import TEMP_PATH, REPO_COPY from get_robot_token import get_best_robot_token from pr_info import PRInfo from report import TestResults, TestResult -from rerun_helper import RerunHelper from s3_helper import S3Helper from stopwatch import Stopwatch from tee_popen import TeePopen @@ -52,8 +57,9 @@ def main(): pr_info = PRInfo(need_changed_files=True) gh = Github(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) - rerun_helper = RerunHelper(gh, pr_info, NAME) + rerun_helper = RerunHelper(commit, NAME) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) @@ -61,9 +67,8 @@ def main(): if not pr_info.has_changes_in_documentation() and not args.force: logging.info("No changes in documentation") - commit = get_commit(gh, pr_info.sha) - commit.create_status( - context=NAME, description="No changes in docs", state="success" + post_commit_status( + commit, "success", NotSet, "No changes in docs", NAME, pr_info ) sys.exit(0) @@ -132,7 +137,7 @@ def main(): s3_helper, pr_info.number, pr_info.sha, test_results, additional_files, NAME ) print("::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, NAME, description, status, report_url) + post_commit_status(commit, status, report_url, description, NAME, pr_info) prepared_events = prepare_tests_results_for_clickhouse( pr_info, diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index a5a4913be0b..5c2139ae0bc 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -1,7 +1,7 @@ import os from os import path as p -from build_download_helper import get_with_retries +from build_download_helper import get_gh_api module_dir = p.abspath(p.dirname(__file__)) git_root = p.abspath(p.join(module_dir, "..", "..")) @@ -46,7 +46,7 @@ def GITHUB_JOB_ID() -> str: jobs = [] page = 1 while not _GITHUB_JOB_ID: - response = get_with_retries( + response = get_gh_api( f"https://api.github.com/repos/{GITHUB_REPOSITORY}/" f"actions/runs/{GITHUB_RUN_ID}/jobs?per_page=100&page={page}" ) diff --git a/tests/ci/fast_test_check.py b/tests/ci/fast_test_check.py index f13b4099657..89066ade2cb 100644 --- a/tests/ci/fast_test_check.py +++ b/tests/ci/fast_test_check.py @@ -17,6 +17,8 @@ from clickhouse_helper import ( prepare_tests_results_for_clickhouse, ) from commit_status_helper import ( + RerunHelper, + get_commit, post_commit_status, update_mergeable_check, ) @@ -25,7 +27,6 @@ from env_helper import S3_BUILDS_BUCKET, TEMP_PATH from get_robot_token import get_best_robot_token from pr_info import FORCE_TESTS_LABEL, PRInfo from report import TestResults, read_test_results -from rerun_helper import RerunHelper from s3_helper import S3Helper from stopwatch import Stopwatch from tee_popen import TeePopen @@ -106,10 +107,11 @@ def main(): pr_info = PRInfo() gh = Github(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) atexit.register(update_mergeable_check, gh, pr_info, NAME) - rerun_helper = RerunHelper(gh, pr_info, NAME) + rerun_helper = RerunHelper(commit, NAME) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") status = rerun_helper.get_finished_status() @@ -197,7 +199,7 @@ def main(): NAME, ) print(f"::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, NAME, description, state, report_url) + post_commit_status(commit, state, report_url, description, NAME, pr_info) prepared_events = prepare_tests_results_for_clickhouse( pr_info, diff --git a/tests/ci/finish_check.py b/tests/ci/finish_check.py index ea2f5eb3136..aa8a0cf9553 100644 --- a/tests/ci/finish_check.py +++ b/tests/ci/finish_check.py @@ -2,32 +2,42 @@ import logging from github import Github -from env_helper import GITHUB_RUN_URL -from pr_info import PRInfo +from commit_status_helper import ( + CI_STATUS_NAME, + NotSet, + get_commit, + get_commit_filtered_statuses, + post_commit_status, +) from get_robot_token import get_best_robot_token -from commit_status_helper import get_commit, get_commit_filtered_statuses - -NAME = "Run Check" +from pr_info import PRInfo -if __name__ == "__main__": +def main(): logging.basicConfig(level=logging.INFO) pr_info = PRInfo(need_orgs=True) gh = Github(get_best_robot_token(), per_page=100) commit = get_commit(gh, pr_info.sha) - url = GITHUB_RUN_URL - statuses = get_commit_filtered_statuses(commit) - pending_status = any( # find NAME status in pending state - True - for status in statuses - if status.context == NAME and status.state == "pending" - ) - if pending_status: - commit.create_status( - context=NAME, - description="All checks finished", - state="success", - target_url=url, + statuses = [ + status + for status in get_commit_filtered_statuses(commit) + if status.context == CI_STATUS_NAME + ] + if not statuses: + return + status = statuses[0] + if status.state == "pending": + post_commit_status( + commit, + "success", + status.target_url or NotSet, + "All checks finished", + CI_STATUS_NAME, + pr_info, ) + + +if __name__ == "__main__": + main() diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 8e55c084f21..037bb13f1f8 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -20,9 +20,11 @@ from clickhouse_helper import ( prepare_tests_results_for_clickhouse, ) from commit_status_helper import ( - post_commit_status, + NotSet, + RerunHelper, get_commit, override_status, + post_commit_status, post_commit_status_to_file, update_mergeable_check, ) @@ -32,7 +34,6 @@ from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH from get_robot_token import get_best_robot_token from pr_info import FORCE_TESTS_LABEL, PRInfo from report import TestResults, read_test_results -from rerun_helper import RerunHelper from s3_helper import S3Helper from stopwatch import Stopwatch from tee_popen import TeePopen @@ -53,6 +54,8 @@ def get_additional_envs(check_name, run_by_hash_num, run_by_hash_total): result.append("USE_PARALLEL_REPLICAS=1") if "s3 storage" in check_name: result.append("USE_S3_STORAGE_FOR_MERGE_TREE=1") + if "analyzer" in check_name: + result.append("USE_NEW_ANALYZER=1") if run_by_hash_total != 0: result.append(f"RUN_BY_HASH_NUM={run_by_hash_num}") @@ -71,6 +74,7 @@ def get_image_name(check_name): def get_run_command( + check_name, builds_path, repo_tests_path, result_path, @@ -103,10 +107,16 @@ def get_run_command( envs += [f"-e {e}" for e in additional_envs] env_str = " ".join(envs) + volume_with_broken_test = ( + f"--volume={repo_tests_path}/broken_tests.txt:/broken_tests.txt" + if "analyzer" in check_name + else "" + ) return ( f"docker run --volume={builds_path}:/package_folder " f"--volume={repo_tests_path}:/usr/share/clickhouse-test " + f"{volume_with_broken_test} " f"--volume={result_path}:/test_output --volume={server_log_path}:/var/log/clickhouse-server " f"--cap-add=SYS_PTRACE {env_str} {additional_options_str} {image}" ) @@ -238,6 +248,7 @@ def main(): need_changed_files=run_changed_tests, pr_event_from_api=validate_bugfix_check ) + commit = get_commit(gh, pr_info.sha) atexit.register(update_mergeable_check, gh, pr_info, check_name) if not os.path.exists(temp_path): @@ -265,7 +276,7 @@ def main(): run_by_hash_total = 0 check_name_with_group = check_name - rerun_helper = RerunHelper(gh, pr_info, check_name_with_group) + rerun_helper = RerunHelper(commit, check_name_with_group) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) @@ -274,13 +285,15 @@ def main(): if run_changed_tests: tests_to_run = get_tests_to_run(pr_info) if not tests_to_run: - commit = get_commit(gh, pr_info.sha) state = override_status("success", check_name, validate_bugfix_check) if args.post_commit_status == "commit_status": - commit.create_status( - context=check_name_with_group, - description=NO_CHANGES_MSG, - state=state, + post_commit_status( + commit, + state, + NotSet, + NO_CHANGES_MSG, + check_name_with_group, + pr_info, ) elif args.post_commit_status == "file": post_commit_status_to_file( @@ -322,6 +335,7 @@ def main(): additional_envs.append("GLOBAL_TAGS=no-random-settings") run_command = get_run_command( + check_name, packages_path, repo_tests_path, result_path, @@ -366,16 +380,16 @@ def main(): if args.post_commit_status == "commit_status": if "parallelreplicas" in check_name.lower(): post_commit_status( - gh, - pr_info.sha, - check_name_with_group, - description, + commit, "success", report_url, + description, + check_name_with_group, + pr_info, ) else: post_commit_status( - gh, pr_info.sha, check_name_with_group, description, state, report_url + commit, state, report_url, description, check_name_with_group, pr_info ) elif args.post_commit_status == "file": if "parallelreplicas" in check_name.lower(): diff --git a/tests/ci/get_robot_token.py b/tests/ci/get_robot_token.py index 6ecaf468ed1..b41eba49cc3 100644 --- a/tests/ci/get_robot_token.py +++ b/tests/ci/get_robot_token.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import logging from dataclasses import dataclass +from typing import Optional import boto3 # type: ignore from github import Github @@ -20,7 +21,13 @@ def get_parameter_from_ssm(name, decrypt=True, client=None): return client.get_parameter(Name=name, WithDecryption=decrypt)["Parameter"]["Value"] +ROBOT_TOKEN = None # type: Optional[Token] + + def get_best_robot_token(token_prefix_env_name="github_robot_token_"): + global ROBOT_TOKEN + if ROBOT_TOKEN is not None: + return ROBOT_TOKEN.value client = boto3.client("ssm", region_name="us-east-1") parameters = client.describe_parameters( ParameterFilters=[ @@ -28,7 +35,6 @@ def get_best_robot_token(token_prefix_env_name="github_robot_token_"): ] )["Parameters"] assert parameters - token = None for token_name in [p["Name"] for p in parameters]: value = get_parameter_from_ssm(token_name, True, client) @@ -38,15 +44,17 @@ def get_best_robot_token(token_prefix_env_name="github_robot_token_"): user = gh.get_user() rest, _ = gh.rate_limiting logging.info("Get token with %s remaining requests", rest) - if token is None: - token = Token(user, value, rest) + if ROBOT_TOKEN is None: + ROBOT_TOKEN = Token(user, value, rest) continue - if token.rest < rest: - token.user, token.value, token.rest = user, value, rest + if ROBOT_TOKEN.rest < rest: + ROBOT_TOKEN.user, ROBOT_TOKEN.value, ROBOT_TOKEN.rest = user, value, rest - assert token + assert ROBOT_TOKEN logging.info( - "User %s with %s remaining requests is used", token.user.login, token.rest + "User %s with %s remaining requests is used", + ROBOT_TOKEN.user.login, + ROBOT_TOKEN.rest, ) - return token.value + return ROBOT_TOKEN.value diff --git a/tests/ci/install_check.py b/tests/ci/install_check.py index 54245670b26..d619ce96cee 100644 --- a/tests/ci/install_check.py +++ b/tests/ci/install_check.py @@ -19,7 +19,9 @@ from clickhouse_helper import ( prepare_tests_results_for_clickhouse, ) from commit_status_helper import ( + RerunHelper, format_description, + get_commit, post_commit_status, update_mergeable_check, ) @@ -29,7 +31,6 @@ from env_helper import CI, TEMP_PATH as TEMP, REPORTS_PATH from get_robot_token import get_best_robot_token from pr_info import PRInfo from report import TestResults, TestResult -from rerun_helper import RerunHelper from s3_helper import S3Helper from stopwatch import Stopwatch from tee_popen import TeePopen @@ -268,9 +269,10 @@ def main(): if CI: gh = Github(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) atexit.register(update_mergeable_check, gh, pr_info, args.check_name) - rerun_helper = RerunHelper(gh, pr_info, args.check_name) + rerun_helper = RerunHelper(commit, args.check_name) if rerun_helper.is_already_finished_by_status(): logging.info( "Check is already finished according to github status, exiting" @@ -347,7 +349,7 @@ def main(): description = format_description(description) - post_commit_status(gh, pr_info.sha, args.check_name, description, state, report_url) + post_commit_status(commit, state, report_url, description, args.check_name, pr_info) prepared_events = prepare_tests_results_for_clickhouse( pr_info, diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index f864751e830..8ef6244a1c5 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -19,8 +19,10 @@ from clickhouse_helper import ( prepare_tests_results_for_clickhouse, ) from commit_status_helper import ( - post_commit_status, + RerunHelper, + get_commit, override_status, + post_commit_status, post_commit_status_to_file, ) from docker_pull_helper import get_images_with_versions @@ -29,7 +31,6 @@ from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH from get_robot_token import get_best_robot_token from pr_info import PRInfo from report import TestResults, read_test_results -from rerun_helper import RerunHelper from s3_helper import S3Helper from stopwatch import Stopwatch from tee_popen import TeePopen @@ -198,8 +199,9 @@ def main(): sys.exit(0) gh = Github(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) - rerun_helper = RerunHelper(gh, pr_info, check_name_with_group) + rerun_helper = RerunHelper(commit, check_name_with_group) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) @@ -284,15 +286,10 @@ def main(): print(f"::notice:: {check_name} Report url: {report_url}") if args.post_commit_status == "commit_status": post_commit_status( - gh, pr_info.sha, check_name_with_group, description, state, report_url + commit, state, report_url, description, check_name_with_group, pr_info ) elif args.post_commit_status == "file": - post_commit_status_to_file( - post_commit_path, - description, - state, - report_url, - ) + post_commit_status_to_file(post_commit_path, description, state, report_url) else: raise Exception( f'Unknown post_commit_status option "{args.post_commit_status}"' diff --git a/tests/ci/jepsen_check.py b/tests/ci/jepsen_check.py index ffa9e45373f..9d35d2d6e35 100644 --- a/tests/ci/jepsen_check.py +++ b/tests/ci/jepsen_check.py @@ -13,13 +13,12 @@ from github import Github from build_download_helper import get_build_name_for_check from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse -from commit_status_helper import post_commit_status +from commit_status_helper import RerunHelper, get_commit, post_commit_status from compress_files import compress_fast from env_helper import REPO_COPY, TEMP_PATH, S3_BUILDS_BUCKET, S3_DOWNLOAD from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo from report import TestResults, TestResult -from rerun_helper import RerunHelper from s3_helper import S3Helper from ssh import SSHKey from stopwatch import Stopwatch @@ -181,10 +180,11 @@ if __name__ == "__main__": sys.exit(0) gh = Github(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) check_name = KEEPER_CHECK_NAME if args.program == "keeper" else SERVER_CHECK_NAME - rerun_helper = RerunHelper(gh, pr_info, check_name) + rerun_helper = RerunHelper(commit, check_name) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) @@ -293,7 +293,7 @@ if __name__ == "__main__": ) print(f"::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, check_name, description, status, report_url) + post_commit_status(commit, status, report_url, description, check_name, pr_info) ch_helper = ClickHouseHelper() prepared_events = prepare_tests_results_for_clickhouse( diff --git a/tests/ci/mark_release_ready.py b/tests/ci/mark_release_ready.py index b103dd053bb..4501d40e4d3 100755 --- a/tests/ci/mark_release_ready.py +++ b/tests/ci/mark_release_ready.py @@ -4,7 +4,7 @@ import argparse import logging import os -from commit_status_helper import get_commit +from commit_status_helper import NotSet, get_commit, post_commit_status from env_helper import GITHUB_JOB_URL from get_robot_token import get_best_robot_token from github_helper import GitHub @@ -34,6 +34,7 @@ def main(): args = parser.parse_args() url = "" description = "the release can be created from the commit, manually set" + pr_info = None if not args.commit: pr_info = PRInfo() if pr_info.event == pr_info.default_event: @@ -45,14 +46,10 @@ def main(): gh = GitHub(args.token, create_cache_dir=False) # Get the rate limits for a quick fail - gh.get_rate_limit() commit = get_commit(gh, args.commit) - - commit.create_status( - context=RELEASE_READY_STATUS, - description=description, - state="success", - target_url=url, + gh.get_rate_limit() + post_commit_status( + commit, "success", url or NotSet, description, RELEASE_READY_STATUS, pr_info ) diff --git a/tests/ci/performance_comparison_check.py b/tests/ci/performance_comparison_check.py index d0c84d56496..bf5704f31bd 100644 --- a/tests/ci/performance_comparison_check.py +++ b/tests/ci/performance_comparison_check.py @@ -12,13 +12,12 @@ from typing import Dict from github import Github -from commit_status_helper import get_commit, post_commit_status +from commit_status_helper import RerunHelper, get_commit, post_commit_status from ci_config import CI_CONFIG from docker_pull_helper import get_image_with_version from env_helper import GITHUB_EVENT_PATH, GITHUB_RUN_URL, S3_BUILDS_BUCKET, S3_DOWNLOAD from get_robot_token import get_best_robot_token, get_parameter_from_ssm from pr_info import PRInfo -from rerun_helper import RerunHelper from s3_helper import S3Helper from tee_popen import TeePopen @@ -118,7 +117,7 @@ if __name__ == "__main__": message = "Skipped, not labeled with 'pr-performance'" report_url = GITHUB_RUN_URL post_commit_status( - gh, pr_info.sha, check_name_with_group, message, status, report_url + commit, status, report_url, message, check_name_with_group, pr_info ) sys.exit(0) @@ -131,7 +130,7 @@ if __name__ == "__main__": "Fill fliter our performance tests by grep -v %s", test_grep_exclude_filter ) - rerun_helper = RerunHelper(gh, pr_info, check_name_with_group) + rerun_helper = RerunHelper(commit, check_name_with_group) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) @@ -142,6 +141,7 @@ if __name__ == "__main__": .replace("(", "_") .replace(")", "_") .replace(",", "_") + .replace("/", "_") ) docker_image = get_image_with_version(reports_path, IMAGE_NAME) @@ -266,7 +266,7 @@ if __name__ == "__main__": report_url = uploaded["report.html"] post_commit_status( - gh, pr_info.sha, check_name_with_group, message, status, report_url + commit, status, report_url, message, check_name_with_group, pr_info ) if status == "error": diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index ddeb070b2b9..86d4985c6b2 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -6,7 +6,7 @@ from typing import Dict, List, Set, Union from unidiff import PatchSet # type: ignore -from build_download_helper import get_with_retries +from build_download_helper import get_gh_api from env_helper import ( GITHUB_REPOSITORY, GITHUB_SERVER_URL, @@ -45,7 +45,7 @@ def get_pr_for_commit(sha, ref): f"https://api.github.com/repos/{GITHUB_REPOSITORY}/commits/{sha}/pulls" ) try: - response = get_with_retries(try_get_pr_url, sleep=RETRY_SLEEP) + response = get_gh_api(try_get_pr_url, sleep=RETRY_SLEEP) data = response.json() our_prs = [] # type: List[Dict] if len(data) > 1: @@ -105,7 +105,7 @@ class PRInfo: # workflow completed event, used for PRs only if "action" in github_event and github_event["action"] == "completed": self.sha = github_event["workflow_run"]["head_sha"] - prs_for_sha = get_with_retries( + prs_for_sha = get_gh_api( f"https://api.github.com/repos/{GITHUB_REPOSITORY}/commits/{self.sha}" "/pulls", sleep=RETRY_SLEEP, @@ -117,7 +117,7 @@ class PRInfo: self.number = github_event["pull_request"]["number"] if pr_event_from_api: try: - response = get_with_retries( + response = get_gh_api( f"https://api.github.com/repos/{GITHUB_REPOSITORY}" f"/pulls/{self.number}", sleep=RETRY_SLEEP, @@ -159,7 +159,7 @@ class PRInfo: self.user_login = github_event["pull_request"]["user"]["login"] self.user_orgs = set([]) if need_orgs: - user_orgs_response = get_with_retries( + user_orgs_response = get_gh_api( github_event["pull_request"]["user"]["organizations_url"], sleep=RETRY_SLEEP, ) @@ -255,7 +255,7 @@ class PRInfo: raise TypeError("The event does not have diff URLs") for diff_url in self.diff_urls: - response = get_with_retries( + response = get_gh_api( diff_url, sleep=RETRY_SLEEP, ) diff --git a/tests/ci/report.py b/tests/ci/report.py index 15d8ff9010e..cdef8409e7e 100644 --- a/tests/ci/report.py +++ b/tests/ci/report.py @@ -262,17 +262,20 @@ class ReportColorTheme: ColorTheme = Tuple[str, str, str] -def _format_header(header, branch_name, branch_url=None): - result = " ".join([w.capitalize() for w in header.split(" ")]) +def _format_header( + header: str, branch_name: str, branch_url: Optional[str] = None +) -> str: + # Following line does not lower CI->Ci and SQLancer->Sqlancer. It only + # capitalizes the first letter and doesn't touch the rest of the word + result = " ".join([w[0].upper() + w[1:] for w in header.split(" ") if w]) result = result.replace("Clickhouse", "ClickHouse") result = result.replace("clickhouse", "ClickHouse") if "ClickHouse" not in result: - result = "ClickHouse " + result - result += " for " + result = f"ClickHouse {result}" if branch_url: - result += f'{branch_name}' + result = f'{result} for {branch_name}' else: - result += branch_name + result = f"{result} for {branch_name}" return result @@ -367,6 +370,7 @@ def create_test_html_report( colspan += 1 if test_result.log_urls is not None: + has_log_urls = True test_logs_html = "
".join( [_get_html_url(url) for url in test_result.log_urls] ) diff --git a/tests/ci/rerun_helper.py b/tests/ci/rerun_helper.py deleted file mode 100644 index fa73256d759..00000000000 --- a/tests/ci/rerun_helper.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python3 -from typing import Optional - -from commit_status_helper import get_commit, get_commit_filtered_statuses -from github import Github -from github.CommitStatus import CommitStatus -from pr_info import PRInfo - - -# TODO: move it to commit_status_helper -class RerunHelper: - def __init__(self, gh: Github, pr_info: PRInfo, check_name: str): - self.gh = gh - self.pr_info = pr_info - self.check_name = check_name - commit = get_commit(gh, self.pr_info.sha) - if commit is None: - raise ValueError(f"unable to receive commit for {pr_info.sha}") - self.pygh_commit = commit - self.statuses = get_commit_filtered_statuses(commit) - - def is_already_finished_by_status(self) -> bool: - # currently we agree even for failed statuses - for status in self.statuses: - if self.check_name in status.context and status.state in ( - "success", - "failure", - ): - return True - return False - - def get_finished_status(self) -> Optional[CommitStatus]: - for status in self.statuses: - if self.check_name in status.context: - return status - return None diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index 44e1e4132c8..9849f19a1e4 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -1,25 +1,28 @@ #!/usr/bin/env python3 import sys import logging -import re from typing import Tuple from github import Github from commit_status_helper import ( + CI_STATUS_NAME, + NotSet, + create_ci_report, format_description, get_commit, + post_commit_status, post_labels, remove_labels, set_mergeable_check, ) from docs_check import NAME as DOCS_NAME -from env_helper import GITHUB_RUN_URL, GITHUB_REPOSITORY, GITHUB_SERVER_URL +from env_helper import GITHUB_REPOSITORY, GITHUB_SERVER_URL from get_robot_token import get_best_robot_token from pr_info import FORCE_TESTS_LABEL, PRInfo -from workflow_approve_rerun_lambda.app import TRUSTED_CONTRIBUTORS -NAME = "Run Check" +from cancel_and_rerun_workflow_lambda.app import CATEGORY_TO_LABEL, check_pr_description +from workflow_approve_rerun_lambda.app import TRUSTED_CONTRIBUTORS TRUSTED_ORG_IDS = { 54801242, # clickhouse @@ -31,40 +34,6 @@ DO_NOT_TEST_LABEL = "do not test" FEATURE_LABEL = "pr-feature" SUBMODULE_CHANGED_LABEL = "submodule changed" -# They are used in .github/PULL_REQUEST_TEMPLATE.md, keep comments there -# updated accordingly -# The following lists are append only, try to avoid editing them -# They atill could be cleaned out after the decent time though. -LABELS = { - "pr-backward-incompatible": ["Backward Incompatible Change"], - "pr-bugfix": [ - "Bug Fix", - "Bug Fix (user-visible misbehavior in an official stable release)", - "Bug Fix (user-visible misbehaviour in official stable or prestable release)", - "Bug Fix (user-visible misbehavior in official stable or prestable release)", - ], - "pr-build": [ - "Build/Testing/Packaging Improvement", - "Build Improvement", - "Build/Testing Improvement", - "Build", - "Packaging Improvement", - ], - "pr-documentation": [ - "Documentation (changelog entry is not required)", - "Documentation", - ], - "pr-feature": ["New Feature"], - "pr-improvement": ["Improvement"], - "pr-not-for-changelog": [ - "Not for changelog (changelog entry is not required)", - "Not for changelog", - ], - "pr-performance": ["Performance Improvement"], -} - -CATEGORY_TO_LABEL = {c: lb for lb, categories in LABELS.items() for c in categories} - def pr_is_by_trusted_user(pr_user_login, pr_user_orgs): if pr_user_login.lower() in TRUSTED_CONTRIBUTORS: @@ -89,7 +58,7 @@ def pr_is_by_trusted_user(pr_user_login, pr_user_orgs): # Returns whether we should look into individual checks for this PR. If not, it # can be skipped entirely. # Returns can_run, description, labels_state -def should_run_checks_for_pr(pr_info: PRInfo) -> Tuple[bool, str, str]: +def should_run_ci_for_pr(pr_info: PRInfo) -> Tuple[bool, str, str]: # Consider the labels and whether the user is trusted. print("Got labels", pr_info.labels) if FORCE_TESTS_LABEL in pr_info.labels: @@ -118,92 +87,7 @@ def should_run_checks_for_pr(pr_info: PRInfo) -> Tuple[bool, str, str]: return True, "No special conditions apply", "pending" -def check_pr_description(pr_info: PRInfo) -> Tuple[str, str]: - lines = list( - map(lambda x: x.strip(), pr_info.body.split("\n") if pr_info.body else []) - ) - lines = [re.sub(r"\s+", " ", line) for line in lines] - - # Check if body contains "Reverts ClickHouse/ClickHouse#36337" - if [ - True - for line in lines - if re.match(rf"\AReverts {GITHUB_REPOSITORY}#[\d]+\Z", line) - ]: - return "", LABELS["pr-not-for-changelog"][0] - - category = "" - entry = "" - description_error = "" - - i = 0 - while i < len(lines): - if re.match(r"(?i)^[#>*_ ]*change\s*log\s*category", lines[i]): - i += 1 - if i >= len(lines): - break - # Can have one empty line between header and the category - # itself. Filter it out. - if not lines[i]: - i += 1 - if i >= len(lines): - break - category = re.sub(r"^[-*\s]*", "", lines[i]) - i += 1 - - # Should not have more than one category. Require empty line - # after the first found category. - if i >= len(lines): - break - if lines[i]: - second_category = re.sub(r"^[-*\s]*", "", lines[i]) - result_status = ( - "More than one changelog category specified: '" - + category - + "', '" - + second_category - + "'" - ) - return result_status, category - - elif re.match( - r"(?i)^[#>*_ ]*(short\s*description|change\s*log\s*entry)", lines[i] - ): - i += 1 - # Can have one empty line between header and the entry itself. - # Filter it out. - if i < len(lines) and not lines[i]: - i += 1 - # All following lines until empty one are the changelog entry. - entry_lines = [] - while i < len(lines) and lines[i]: - entry_lines.append(lines[i]) - i += 1 - entry = " ".join(entry_lines) - # Don't accept changelog entries like '...'. - entry = re.sub(r"[#>*_.\- ]", "", entry) - # Don't accept changelog entries like 'Close #12345'. - entry = re.sub(r"^[\w\-\s]{0,10}#?\d{5,6}\.?$", "", entry) - else: - i += 1 - - if not category: - description_error = "Changelog category is empty" - # Filter out the PR categories that are not for changelog. - elif re.match( - r"(?i)doc|((non|in|not|un)[-\s]*significant)|(not[ ]*for[ ]*changelog)", - category, - ): - pass # to not check the rest of the conditions - elif category not in CATEGORY_TO_LABEL: - description_error, category = f"Category '{category}' is not valid", "" - elif not entry: - description_error = f"Changelog entry required for category '{category}'" - - return description_error, category - - -if __name__ == "__main__": +def main(): logging.basicConfig(level=logging.INFO) pr_info = PRInfo(need_orgs=True, pr_event_from_api=True, need_changed_files=True) @@ -213,7 +97,7 @@ if __name__ == "__main__": print("::notice ::Cannot run, no PR exists for the commit") sys.exit(1) - can_run, description, labels_state = should_run_checks_for_pr(pr_info) + can_run, description, labels_state = should_run_ci_for_pr(pr_info) if can_run and OK_SKIP_LABELS.intersection(pr_info.labels): print("::notice :: Early finish the check, running in a special PR") sys.exit(0) @@ -222,7 +106,7 @@ if __name__ == "__main__": gh = Github(get_best_robot_token(), per_page=100) commit = get_commit(gh, pr_info.sha) - description_error, category = check_pr_description(pr_info) + description_error, category = check_pr_description(pr_info.body) pr_labels_to_add = [] pr_labels_to_remove = [] if ( @@ -253,12 +137,14 @@ if __name__ == "__main__": if FEATURE_LABEL in pr_info.labels: print(f"The '{FEATURE_LABEL}' in the labels, expect the 'Docs Check' status") - commit.create_status( - context=DOCS_NAME, - description=f"expect adding docs for {FEATURE_LABEL}", - state="pending", + post_commit_status( # do not pass pr_info here intentionally + commit, + "pending", + NotSet, + f"expect adding docs for {FEATURE_LABEL}", + DOCS_NAME, ) - else: + elif not description_error: set_mergeable_check(commit, "skipped") if description_error: @@ -267,7 +153,7 @@ if __name__ == "__main__": f"{description_error}" ) logging.info( - "PR body doesn't match the template: (start)\n%s\n(end)\n" "Reason: %s", + "PR body doesn't match the template: (start)\n%s\n(end)\nReason: %s", pr_info.body, description_error, ) @@ -275,23 +161,29 @@ if __name__ == "__main__": f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/" "blob/master/.github/PULL_REQUEST_TEMPLATE.md?plain=1" ) - commit.create_status( - context=NAME, - description=format_description(description_error), - state="failure", - target_url=url, + post_commit_status( + commit, + "failure", + url, + format_description(description_error), + CI_STATUS_NAME, + pr_info, ) sys.exit(1) - url = GITHUB_RUN_URL + ci_report_url = create_ci_report(pr_info, []) if not can_run: print("::notice ::Cannot run") - commit.create_status( - context=NAME, description=description, state=labels_state, target_url=url + post_commit_status( + commit, labels_state, ci_report_url, description, CI_STATUS_NAME, pr_info ) sys.exit(1) else: print("::notice ::Can run") - commit.create_status( - context=NAME, description=description, state="pending", target_url=url + post_commit_status( + commit, "pending", ci_report_url, description, CI_STATUS_NAME, pr_info ) + + +if __name__ == "__main__": + main() diff --git a/tests/ci/s3_helper.py b/tests/ci/s3_helper.py index fbe9f33b49b..2af02d572c8 100644 --- a/tests/ci/s3_helper.py +++ b/tests/ci/s3_helper.py @@ -40,11 +40,11 @@ def _flatten_list(lst): class S3Helper: - def __init__(self, host=S3_URL, download_host=S3_DOWNLOAD): + def __init__(self): self.session = boto3.session.Session(region_name="us-east-1") - self.client = self.session.client("s3", endpoint_url=host) - self.host = host - self.download_host = download_host + self.client = self.session.client("s3", endpoint_url=S3_URL) + self.host = S3_URL + self.download_host = S3_DOWNLOAD def _upload_file_to_s3(self, bucket_name: str, file_path: str, s3_path: str) -> str: logging.debug( diff --git a/tests/ci/sqlancer_check.py b/tests/ci/sqlancer_check.py index 1a6c4d14616..144dea54133 100644 --- a/tests/ci/sqlancer_check.py +++ b/tests/ci/sqlancer_check.py @@ -10,10 +10,14 @@ from github import Github from build_download_helper import get_build_name_for_check, read_build_urls from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse -from commit_status_helper import format_description, post_commit_status +from commit_status_helper import ( + RerunHelper, + format_description, + get_commit, + post_commit_status, +) from docker_pull_helper import get_image_with_version from env_helper import ( - GITHUB_REPOSITORY, GITHUB_RUN_URL, REPORTS_PATH, TEMP_PATH, @@ -21,7 +25,6 @@ from env_helper import ( from get_robot_token import get_best_robot_token from pr_info import PRInfo from report import TestResults, TestResult -from rerun_helper import RerunHelper from s3_helper import S3Helper from stopwatch import Stopwatch from upload_result_helper import upload_results @@ -46,12 +49,6 @@ def get_run_command(download_url, workspace_path, image): ) -def get_commit(gh, commit_sha): - repo = gh.get_repo(GITHUB_REPOSITORY) - commit = repo.get_commit(commit_sha) - return commit - - def main(): logging.basicConfig(level=logging.INFO) @@ -68,8 +65,9 @@ def main(): pr_info = PRInfo() gh = Github(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) - rerun_helper = RerunHelper(gh, pr_info, check_name) + rerun_helper = RerunHelper(commit, check_name) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) @@ -187,12 +185,10 @@ def main(): check_name, ) - post_commit_status(gh, pr_info.sha, check_name, description, status, report_url) - + post_commit_status(commit, status, report_url, description, check_name, pr_info) print(f"::notice:: {check_name} Report url: {report_url}") ch_helper = ClickHouseHelper() - prepared_events = prepare_tests_results_for_clickhouse( pr_info, test_results, @@ -202,12 +198,8 @@ def main(): report_url, check_name, ) - ch_helper.insert_events_into(db="default", table="checks", events=prepared_events) - print(f"::notice Result: '{status}', '{description}', '{report_url}'") - post_commit_status(gh, pr_info.sha, check_name, description, status, report_url) - if __name__ == "__main__": main() diff --git a/tests/ci/sqllogic_test.py b/tests/ci/sqllogic_test.py index 9b41ff4680f..942c9c60ee8 100755 --- a/tests/ci/sqllogic_test.py +++ b/tests/ci/sqllogic_test.py @@ -17,11 +17,15 @@ from pr_info import FORCE_TESTS_LABEL, PRInfo from build_download_helper import download_all_deb_packages from upload_result_helper import upload_results from docker_pull_helper import get_image_with_version -from commit_status_helper import override_status, post_commit_status +from commit_status_helper import ( + RerunHelper, + get_commit, + override_status, + post_commit_status, +) from report import TestResults, read_test_results from stopwatch import Stopwatch -from rerun_helper import RerunHelper from tee_popen import TeePopen @@ -103,8 +107,9 @@ if __name__ == "__main__": pr_info = PRInfo() gh = Github(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) - rerun_helper = RerunHelper(gh, pr_info, check_name) + rerun_helper = RerunHelper(commit, check_name) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) @@ -203,7 +208,7 @@ if __name__ == "__main__": # Until it pass all tests, do not block CI, report "success" assert description is not None - post_commit_status(gh, pr_info.sha, check_name, description, "success", report_url) + post_commit_status(commit, "success", report_url, description, check_name, pr_info) if status != "success": if FORCE_TESTS_LABEL in pr_info.labels: diff --git a/tests/ci/stress_check.py b/tests/ci/stress_check.py index 7596a81ebc9..ac280916a2f 100644 --- a/tests/ci/stress_check.py +++ b/tests/ci/stress_check.py @@ -16,13 +16,12 @@ from clickhouse_helper import ( mark_flaky_tests, prepare_tests_results_for_clickhouse, ) -from commit_status_helper import post_commit_status +from commit_status_helper import RerunHelper, get_commit, post_commit_status from docker_pull_helper import get_image_with_version from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH from get_robot_token import get_best_robot_token from pr_info import PRInfo from report import TestResults, read_test_results -from rerun_helper import RerunHelper from s3_helper import S3Helper from stopwatch import Stopwatch from tee_popen import TeePopen @@ -125,8 +124,9 @@ def run_stress_test(docker_image_name): pr_info = PRInfo() gh = Github(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) - rerun_helper = RerunHelper(gh, pr_info, check_name) + rerun_helper = RerunHelper(commit, check_name) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) @@ -180,7 +180,7 @@ def run_stress_test(docker_image_name): ) print(f"::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, check_name, description, state, report_url) + post_commit_status(commit, state, report_url, description, check_name, pr_info) prepared_events = prepare_tests_results_for_clickhouse( pr_info, diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 89878990c2c..33a5cd21f39 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -15,7 +15,12 @@ from clickhouse_helper import ( mark_flaky_tests, prepare_tests_results_for_clickhouse, ) -from commit_status_helper import post_commit_status, update_mergeable_check +from commit_status_helper import ( + RerunHelper, + get_commit, + post_commit_status, + update_mergeable_check, +) from docker_pull_helper import get_image_with_version from env_helper import GITHUB_WORKSPACE, RUNNER_TEMP from get_robot_token import get_best_robot_token @@ -23,7 +28,6 @@ from github_helper import GitHub from git_helper import git_runner from pr_info import PRInfo from report import TestResults, read_test_results -from rerun_helper import RerunHelper from s3_helper import S3Helper from ssh import SSHKey from stopwatch import Stopwatch @@ -149,10 +153,11 @@ def main(): checkout_head(pr_info) gh = GitHub(get_best_robot_token(), create_cache_dir=False) + commit = get_commit(gh, pr_info.sha) atexit.register(update_mergeable_check, gh, pr_info, NAME) - rerun_helper = RerunHelper(gh, pr_info, NAME) + rerun_helper = RerunHelper(commit, NAME) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") # Finish with the same code as previous @@ -190,7 +195,7 @@ def main(): s3_helper, pr_info.number, pr_info.sha, test_results, additional_files, NAME ) print(f"::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, NAME, description, state, report_url) + post_commit_status(commit, state, report_url, description, NAME, pr_info) prepared_events = prepare_tests_results_for_clickhouse( pr_info, diff --git a/tests/ci/unit_tests_check.py b/tests/ci/unit_tests_check.py index edc096908f4..5279ccde492 100644 --- a/tests/ci/unit_tests_check.py +++ b/tests/ci/unit_tests_check.py @@ -15,13 +15,17 @@ from clickhouse_helper import ( mark_flaky_tests, prepare_tests_results_for_clickhouse, ) -from commit_status_helper import post_commit_status, update_mergeable_check +from commit_status_helper import ( + RerunHelper, + get_commit, + post_commit_status, + update_mergeable_check, +) from docker_pull_helper import get_image_with_version from env_helper import TEMP_PATH, REPORTS_PATH from get_robot_token import get_best_robot_token from pr_info import PRInfo from report import TestResults, TestResult -from rerun_helper import RerunHelper from s3_helper import S3Helper from stopwatch import Stopwatch from tee_popen import TeePopen @@ -116,10 +120,11 @@ def main(): pr_info = PRInfo() gh = Github(get_best_robot_token(), per_page=100) + commit = get_commit(gh, pr_info.sha) atexit.register(update_mergeable_check, gh, pr_info, check_name) - rerun_helper = RerunHelper(gh, pr_info, check_name) + rerun_helper = RerunHelper(commit, check_name) if rerun_helper.is_already_finished_by_status(): logging.info("Check is already finished according to github status, exiting") sys.exit(0) @@ -165,7 +170,7 @@ def main(): check_name, ) print(f"::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, check_name, description, state, report_url) + post_commit_status(commit, state, report_url, description, check_name, pr_info) prepared_events = prepare_tests_results_for_clickhouse( pr_info, diff --git a/tests/ci/upload_result_helper.py b/tests/ci/upload_result_helper.py index b988e240b0e..150af7aff4a 100644 --- a/tests/ci/upload_result_helper.py +++ b/tests/ci/upload_result_helper.py @@ -59,9 +59,10 @@ def upload_results( additional_files: List[str], check_name: str, ) -> str: - s3_path_prefix = f"{pr_number}/{commit_sha}/" + check_name.lower().replace( - " ", "_" - ).replace("(", "_").replace(")", "_").replace(",", "_") + normalized_check_name = check_name.lower() + for r in ((" ", "_"), ("(", "_"), (")", "_"), (",", "_"), ("/", "_")): + normalized_check_name = normalized_check_name.replace(*r) + s3_path_prefix = f"{pr_number}/{commit_sha}/{normalized_check_name}" additional_urls = process_logs( s3_client, additional_files, s3_path_prefix, test_results ) diff --git a/tests/ci/workflow_jobs_lambda/app.py b/tests/ci/workflow_jobs_lambda/app.py index 9436e01ad53..49d475d11dc 100644 --- a/tests/ci/workflow_jobs_lambda/app.py +++ b/tests/ci/workflow_jobs_lambda/app.py @@ -284,10 +284,12 @@ def handler(event: dict, _: Any) -> dict: wf_job["runner_group_name"] or "", # nullable repo["full_name"], ) + logging.info( + "Got the next event (private_repo=%s): %s", repo["private"], workflow_job + ) if repo["private"]: workflow_job.anonimyze() - logging.info("Got the next event: %s", workflow_job) send_event_workflow_job(workflow_job) return { diff --git a/tests/clickhouse-test b/tests/clickhouse-test index eb5faa1ffb5..acc8688cc4a 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -11,6 +11,7 @@ import shutil import sys import os import os.path +import platform import signal import re import copy @@ -542,7 +543,10 @@ class SettingsRandomizer: 0.2, 0.5, 1, 10 * 1024 * 1024 * 1024 ), "local_filesystem_read_method": lambda: random.choice( + # Allow to use uring only when running on Linux ["read", "pread", "mmap", "pread_threadpool", "io_uring"] + if platform.system().lower() == "linux" + else ["read", "pread", "mmap", "pread_threadpool"] ), "remote_filesystem_read_method": lambda: random.choice(["read", "threadpool"]), "local_filesystem_read_prefetch": lambda: random.randint(0, 1), @@ -579,10 +583,17 @@ class SettingsRandomizer: } @staticmethod - def get_random_settings(): + def get_random_settings(args): random_settings = [] + is_debug = BuildFlags.DEBUG in args.build_flags for setting, generator in SettingsRandomizer.settings.items(): - random_settings.append(f"{setting}={generator()}") + if ( + is_debug + and setting == "allow_prefetched_read_pool_for_remote_filesystem" + ): + random_settings.append(f"{setting}=0") + else: + random_settings.append(f"{setting}={generator()}") return random_settings @@ -817,7 +828,7 @@ class TestCase: ) if self.randomize_settings: - self.random_settings = SettingsRandomizer.get_random_settings() + self.random_settings = SettingsRandomizer.get_random_settings(args) if self.randomize_merge_tree_settings: self.merge_tree_random_settings = ( @@ -2106,7 +2117,14 @@ def reportLogStats(args): 'Column ''{}'' already exists', 'No macro {} in config', 'Invalid origin H3 index: {}', 'Invalid session timeout: ''{}''', 'Tuple cannot be empty', 'Database name is empty', 'Table {} is not a Dictionary', 'Expected function, got: {}', 'Unknown identifier: ''{}''', - 'Failed to {} input ''{}''', '{}.{} is not a VIEW', 'Cannot convert NULL to {}', 'Dictionary {} doesn''t exist' + 'Failed to {} input ''{}''', '{}.{} is not a VIEW', 'Cannot convert NULL to {}', 'Dictionary {} doesn''t exist', + 'Write file: {}', 'Unable to parse JSONPath', 'Host is empty in S3 URI.', 'Expected end of line', + 'inflate failed: {}{}', 'Center is not valid', 'Column ''{}'' is ambiguous', 'Cannot parse object', 'Invalid date: {}', + 'There is no cache by name: {}', 'No part {} in table', '`{}` should be a String', 'There are duplicate id {}', + 'Invalid replica name: {}', 'Unexpected value {} in enum', 'Unknown BSON type: {}', 'Point is not valid', + 'Invalid qualified name: {}', 'INTO OUTFILE is not allowed', 'Arguments must not be NaN', 'Cell is not valid', + 'brotli decode error{}', 'Invalid H3 index: {}', 'Too large node state size', 'No additional keys found.', + 'Attempt to read after EOF.', 'Replication was stopped', '{} building file infos', 'Cannot parse uuid {}' ) AS known_short_messages SELECT count() AS c, message_format_string, substr(any(message), 1, 120) FROM system.text_log @@ -2245,7 +2263,7 @@ def main(args): "\nFound hung queries in processlist:", args, "red", attrs=["bold"] ) ) - print(json.dumps(processlist, indent=4)) + print(processlist) print(get_transactions_list(args)) print_stacktraces() diff --git a/tests/config/config.d/merge_tree_old_dirs_cleanup.xml b/tests/config/config.d/merge_tree_old_dirs_cleanup.xml index 41932cb6d61..2b8ea63b63d 100644 --- a/tests/config/config.d/merge_tree_old_dirs_cleanup.xml +++ b/tests/config/config.d/merge_tree_old_dirs_cleanup.xml @@ -3,6 +3,6 @@ 1 - 10 + 5 diff --git a/tests/config/config.d/storage_conf.xml b/tests/config/config.d/storage_conf.xml index bc9269e6ec1..dee03307177 100644 --- a/tests/config/config.d/storage_conf.xml +++ b/tests/config/config.d/storage_conf.xml @@ -55,52 +55,58 @@ cache s3_disk s3_cache/ - 2147483648 + 128Mi 1 0 + 100 cache s3_disk_2 s3_cache_2/ - 2Gi + 128Mi 0 100Mi + 100 cache s3_disk_3 s3_disk_3_cache/ - 22548578304 + 128Mi 22548578304 1 1 0 + 100 cache s3_disk_4 s3_cache_4/ - 22548578304 + 128Mi 1 1 0 + 100 cache s3_disk_5 s3_cache_5/ - 22548578304 + 128Mi 0 + 100 cache s3_disk_6 s3_cache_6/ - 22548578304 + 128Mi 0 1 100 + 100 cache @@ -108,27 +114,29 @@ s3_cache_small/ 1000 1 + 100 cache s3_disk_6 s3_cache_small_segment_size/ - 22548578304 + 128Mi 10Ki 0 1 + 100 - local + local_blob_storage local_disk/ - local + local_blob_storage local_disk_2/ - local + local_blob_storage local_disk_3/ @@ -139,6 +147,7 @@ 22548578304 1 0 + 100 cache @@ -146,6 +155,7 @@ local_cache_2/ 22548578304 0 + 100 cache @@ -155,6 +165,7 @@ 1 1 0 + 100 @@ -163,6 +174,7 @@ s3_cache_multi/ 22548578304 0 + 100 cache @@ -170,6 +182,7 @@ s3_cache_multi_2/ 22548578304 0 + 100 diff --git a/tests/config/install.sh b/tests/config/install.sh index 77e8a8460ad..efa5a9c086e 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -79,6 +79,10 @@ ln -sf $SRC_PATH/users.d/marks.xml $DEST_SERVER_PATH/users.d/ ln -sf $SRC_PATH/users.d/insert_keeper_retries.xml $DEST_SERVER_PATH/users.d/ ln -sf $SRC_PATH/users.d/prefetch_settings.xml $DEST_SERVER_PATH/users.d/ +if [[ -n "$USE_NEW_ANALYZER" ]] && [[ "$USE_NEW_ANALYZER" -eq 1 ]]; then + ln -sf $SRC_PATH/users.d/analyzer.xml $DEST_SERVER_PATH/users.d/ +fi + # FIXME DataPartsExchange may hang for http_send_timeout seconds # when nobody is going to read from the other side of socket (due to "Fetching of part was cancelled"), # but socket is owned by HTTPSessionPool, so it's not closed. diff --git a/tests/config/users.d/analyzer.xml b/tests/config/users.d/analyzer.xml new file mode 100644 index 00000000000..aa374364ef0 --- /dev/null +++ b/tests/config/users.d/analyzer.xml @@ -0,0 +1,7 @@ + + + + 1 + + + diff --git a/tests/integration/helpers/external_sources.py b/tests/integration/helpers/external_sources.py index fd086fc4526..afb91083d57 100644 --- a/tests/integration/helpers/external_sources.py +++ b/tests/integration/helpers/external_sources.py @@ -161,6 +161,29 @@ class SourceMySQL(ExternalSource): class SourceMongo(ExternalSource): + def __init__( + self, + name, + internal_hostname, + internal_port, + docker_hostname, + docker_port, + user, + password, + secure=False, + ): + ExternalSource.__init__( + self, + name, + internal_hostname, + internal_port, + docker_hostname, + docker_port, + user, + password, + ) + self.secure = secure + def get_source_str(self, table_name): return """ @@ -170,6 +193,7 @@ class SourceMongo(ExternalSource): {password} test {tbl} + {options} """.format( host=self.docker_hostname, @@ -177,6 +201,7 @@ class SourceMongo(ExternalSource): user=self.user, password=self.password, tbl=table_name, + options="ssl=true" if self.secure else "", ) def prepare(self, structure, table_name, cluster): @@ -186,6 +211,8 @@ class SourceMongo(ExternalSource): user=self.user, password=self.password, ) + if self.secure: + connection_str += "/?tls=true&tlsAllowInvalidCertificates=true" self.connection = pymongo.MongoClient(connection_str) self.converters = {} for field in structure.get_all_fields(): @@ -228,7 +255,7 @@ class SourceMongoURI(SourceMongo): def get_source_str(self, table_name): return """ - mongodb://{user}:{password}@{host}:{port}/test + mongodb://{user}:{password}@{host}:{port}/test{options} {tbl} """.format( @@ -237,6 +264,7 @@ class SourceMongoURI(SourceMongo): user=self.user, password=self.password, tbl=table_name, + options="?ssl=true" if self.secure else "", ) diff --git a/tests/integration/helpers/postgres_utility.py b/tests/integration/helpers/postgres_utility.py index 838c22c8a7c..dfae37af434 100644 --- a/tests/integration/helpers/postgres_utility.py +++ b/tests/integration/helpers/postgres_utility.py @@ -204,7 +204,7 @@ class PostgresManager: assert materialized_database in self.instance.query("SHOW DATABASES") def drop_materialized_db(self, materialized_database="test_database"): - self.instance.query(f"DROP DATABASE IF EXISTS {materialized_database} NO DELAY") + self.instance.query(f"DROP DATABASE IF EXISTS {materialized_database} SYNC") if materialized_database in self.created_materialized_postgres_db_list: self.created_materialized_postgres_db_list.remove(materialized_database) assert materialized_database not in self.instance.query("SHOW DATABASES") diff --git a/tests/integration/test_alternative_keeper_config/test.py b/tests/integration/test_alternative_keeper_config/test.py index 2d59d2ee8b9..f1016ee1ae3 100644 --- a/tests/integration/test_alternative_keeper_config/test.py +++ b/tests/integration/test_alternative_keeper_config/test.py @@ -44,7 +44,7 @@ def started_cluster(): def test_create_insert(started_cluster): - node1.query("DROP TABLE IF EXISTS tbl ON CLUSTER 'test_cluster' NO DELAY") + node1.query("DROP TABLE IF EXISTS tbl ON CLUSTER 'test_cluster' SYNC") node1.query( """ CREATE TABLE tbl ON CLUSTER 'test_cluster' ( diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py index 41db2580c7d..53f1599a0d6 100644 --- a/tests/integration/test_backup_restore_new/test.py +++ b/tests/integration/test_backup_restore_new/test.py @@ -1482,23 +1482,23 @@ def test_tables_dependency(): # Drop everything in reversive order. def drop(): - instance.query(f"DROP TABLE {t15} NO DELAY") - instance.query(f"DROP TABLE {t14} NO DELAY") - instance.query(f"DROP TABLE {t13} NO DELAY") - instance.query(f"DROP TABLE {t12} NO DELAY") - instance.query(f"DROP TABLE {t11} NO DELAY") - instance.query(f"DROP TABLE {t10} NO DELAY") - instance.query(f"DROP TABLE {t9} NO DELAY") + instance.query(f"DROP TABLE {t15} SYNC") + instance.query(f"DROP TABLE {t14} SYNC") + instance.query(f"DROP TABLE {t13} SYNC") + instance.query(f"DROP TABLE {t12} SYNC") + instance.query(f"DROP TABLE {t11} SYNC") + instance.query(f"DROP TABLE {t10} SYNC") + instance.query(f"DROP TABLE {t9} SYNC") instance.query(f"DROP DICTIONARY {t8}") - instance.query(f"DROP TABLE {t7} NO DELAY") - instance.query(f"DROP TABLE {t6} NO DELAY") - instance.query(f"DROP TABLE {t5} NO DELAY") + instance.query(f"DROP TABLE {t7} SYNC") + instance.query(f"DROP TABLE {t6} SYNC") + instance.query(f"DROP TABLE {t5} SYNC") instance.query(f"DROP DICTIONARY {t4}") - instance.query(f"DROP TABLE {t3} NO DELAY") - instance.query(f"DROP TABLE {t2} NO DELAY") - instance.query(f"DROP TABLE {t1} NO DELAY") - instance.query("DROP DATABASE test NO DELAY") - instance.query("DROP DATABASE test2 NO DELAY") + instance.query(f"DROP TABLE {t3} SYNC") + instance.query(f"DROP TABLE {t2} SYNC") + instance.query(f"DROP TABLE {t1} SYNC") + instance.query("DROP DATABASE test SYNC") + instance.query("DROP DATABASE test2 SYNC") drop() diff --git a/tests/integration/test_backup_restore_on_cluster/test.py b/tests/integration/test_backup_restore_on_cluster/test.py index 9ed39627d82..5542eac856d 100644 --- a/tests/integration/test_backup_restore_on_cluster/test.py +++ b/tests/integration/test_backup_restore_on_cluster/test.py @@ -65,9 +65,9 @@ def drop_after_test(): try: yield finally: - node1.query("DROP TABLE IF EXISTS tbl ON CLUSTER 'cluster3' NO DELAY") - node1.query("DROP TABLE IF EXISTS tbl2 ON CLUSTER 'cluster3' NO DELAY") - node1.query("DROP DATABASE IF EXISTS mydb ON CLUSTER 'cluster3' NO DELAY") + node1.query("DROP TABLE IF EXISTS tbl ON CLUSTER 'cluster3' SYNC") + node1.query("DROP TABLE IF EXISTS tbl2 ON CLUSTER 'cluster3' SYNC") + node1.query("DROP DATABASE IF EXISTS mydb ON CLUSTER 'cluster3' SYNC") node1.query("DROP USER IF EXISTS u1, u2 ON CLUSTER 'cluster3'") @@ -107,7 +107,7 @@ def test_replicated_table(): ) # Drop table on both nodes. - node1.query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + node1.query(f"DROP TABLE tbl ON CLUSTER 'cluster' SYNC") # Restore from backup on node2. node2.query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}") @@ -138,7 +138,7 @@ def test_empty_replicated_table(): ) # Drop table on both nodes. - node1.query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + node1.query(f"DROP TABLE tbl ON CLUSTER 'cluster' SYNC") # Restore from backup on node2. node1.query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}") @@ -172,7 +172,7 @@ def test_replicated_database(): ) # Drop table on both nodes. - node1.query("DROP DATABASE mydb ON CLUSTER 'cluster' NO DELAY") + node1.query("DROP DATABASE mydb ON CLUSTER 'cluster' SYNC") # Restore from backup on node2. node1.query(f"RESTORE DATABASE mydb ON CLUSTER 'cluster' FROM {backup_name}") @@ -201,7 +201,7 @@ def test_different_tables_on_nodes(): backup_name = new_backup_name() node1.query(f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}") - node1.query("DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + node1.query("DROP TABLE tbl ON CLUSTER 'cluster' SYNC") node2.query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}") @@ -224,7 +224,7 @@ def test_backup_restore_on_single_replica(): backup_name = new_backup_name() node1.query(f"BACKUP DATABASE mydb TO {backup_name}") - node1.query("DROP DATABASE mydb NO DELAY") + node1.query("DROP DATABASE mydb SYNC") # Cannot restore table because it already contains data on other replicas. expected_error = "already contains some data" @@ -243,7 +243,7 @@ def test_backup_restore_on_single_replica(): ) # Can restore table with allow_non_empty_tables=true. - node1.query("DROP DATABASE mydb NO DELAY") + node1.query("DROP DATABASE mydb SYNC") node1.query( f"RESTORE DATABASE mydb FROM {backup_name} SETTINGS allow_non_empty_tables=true" ) @@ -266,7 +266,7 @@ def test_table_with_parts_in_queue_considered_non_empty(): backup_name = new_backup_name() node1.query(f"BACKUP DATABASE mydb TO {backup_name}") - node1.query("DROP DATABASE mydb NO DELAY") + node1.query("DROP DATABASE mydb SYNC") # Cannot restore table because it already contains data on other replicas. expected_error = "already contains some data" @@ -295,7 +295,7 @@ def test_replicated_table_with_not_synced_insert(): backup_name = new_backup_name() node1.query(f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}") - node1.query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + node1.query(f"DROP TABLE tbl ON CLUSTER 'cluster' SYNC") node1.query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}") node1.query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster' tbl") @@ -325,7 +325,7 @@ def test_replicated_table_with_not_synced_merge(): backup_name = new_backup_name() node1.query(f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}") - node1.query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + node1.query(f"DROP TABLE tbl ON CLUSTER 'cluster' SYNC") node1.query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}") node1.query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster' tbl") @@ -348,7 +348,7 @@ def test_replicated_table_restored_into_bigger_cluster(): backup_name = new_backup_name() node1.query(f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}") - node1.query("DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + node1.query("DROP TABLE tbl ON CLUSTER 'cluster' SYNC") node1.query(f"RESTORE TABLE tbl ON CLUSTER 'cluster3' FROM {backup_name}") node1.query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster3' tbl") @@ -372,7 +372,7 @@ def test_replicated_table_restored_into_smaller_cluster(): backup_name = new_backup_name() node1.query(f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}") - node1.query("DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + node1.query("DROP TABLE tbl ON CLUSTER 'cluster' SYNC") node1.query(f"RESTORE TABLE tbl ON CLUSTER 'cluster1' FROM {backup_name}") assert node1.query("SELECT * FROM tbl ORDER BY x") == TSV([111, 222]) @@ -410,7 +410,7 @@ def test_replicated_database_async(): TSV([["BACKUP_CREATED", ""]]), ) - node1.query("DROP DATABASE mydb ON CLUSTER 'cluster' NO DELAY") + node1.query("DROP DATABASE mydb ON CLUSTER 'cluster' SYNC") [id, status] = node1.query( f"RESTORE DATABASE mydb ON CLUSTER 'cluster' FROM {backup_name} ASYNC" @@ -454,7 +454,7 @@ def test_keeper_value_max_size(): settings={"backup_restore_keeper_value_max_size": 50}, ) - node1.query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + node1.query(f"DROP TABLE tbl ON CLUSTER 'cluster' SYNC") node1.query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}") node1.query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster' tbl") @@ -541,7 +541,7 @@ def test_async_backups_to_same_destination(interface, on_cluster): assert num_failed_backups == len(ids) - 1 # Check that the succeeded backup is all right. - node1.query("DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + node1.query("DROP TABLE tbl ON CLUSTER 'cluster' SYNC") node1.query(f"RESTORE TABLE tbl FROM {backup_name}") assert node1.query("SELECT * FROM tbl") == "1\n" @@ -568,7 +568,7 @@ def test_required_privileges(): node1.query("GRANT BACKUP ON tbl TO u1") node1.query(f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}", user="u1") - node1.query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + node1.query(f"DROP TABLE tbl ON CLUSTER 'cluster' SYNC") expected_error = "necessary to have grant INSERT, CREATE TABLE ON default.tbl2" assert expected_error in node1.query_and_get_error( @@ -582,7 +582,7 @@ def test_required_privileges(): assert node2.query("SELECT * FROM tbl2") == "100\n" - node1.query(f"DROP TABLE tbl2 ON CLUSTER 'cluster' NO DELAY") + node1.query(f"DROP TABLE tbl2 ON CLUSTER 'cluster' SYNC") node1.query("REVOKE ALL FROM u1") expected_error = "necessary to have grant INSERT, CREATE TABLE ON default.tbl" @@ -703,7 +703,7 @@ def test_projection(): backup_name = new_backup_name() node1.query(f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}") - node1.query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + node1.query(f"DROP TABLE tbl ON CLUSTER 'cluster' SYNC") assert ( node1.query( @@ -755,7 +755,7 @@ def test_replicated_table_with_not_synced_def(): backup_name = new_backup_name() node2.query(f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}") - node1.query("DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + node1.query("DROP TABLE tbl ON CLUSTER 'cluster' SYNC") # But synced after RESTORE anyway node1.query( @@ -768,7 +768,7 @@ def test_replicated_table_with_not_synced_def(): "SELECT name, type FROM system.columns WHERE database='default' AND table='tbl'" ) == TSV([["x", "String"], ["y", "String"]]) - node1.query("DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + node1.query("DROP TABLE tbl ON CLUSTER 'cluster' SYNC") node2.query( f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name} SETTINGS replica_num_in_backup=2" @@ -795,7 +795,7 @@ def test_table_in_replicated_database_with_not_synced_def(): backup_name = new_backup_name() node2.query(f"BACKUP DATABASE mydb ON CLUSTER 'cluster' TO {backup_name}") - node1.query("DROP DATABASE mydb ON CLUSTER 'cluster' NO DELAY") + node1.query("DROP DATABASE mydb ON CLUSTER 'cluster' SYNC") # But synced after RESTORE anyway node1.query( @@ -808,7 +808,7 @@ def test_table_in_replicated_database_with_not_synced_def(): "SELECT name, type FROM system.columns WHERE database='mydb' AND table='tbl'" ) == TSV([["x", "String"], ["y", "String"]]) - node1.query("DROP DATABASE mydb ON CLUSTER 'cluster' NO DELAY") + node1.query("DROP DATABASE mydb ON CLUSTER 'cluster' SYNC") node2.query( f"RESTORE DATABASE mydb ON CLUSTER 'cluster' FROM {backup_name} SETTINGS replica_num_in_backup=2" @@ -870,7 +870,7 @@ def test_mutation(): assert has_mutation_in_backup("0000000002", backup_name, "default", "tbl") assert not has_mutation_in_backup("0000000003", backup_name, "default", "tbl") - node1.query("DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + node1.query("DROP TABLE tbl ON CLUSTER 'cluster' SYNC") node1.query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}") @@ -1006,7 +1006,7 @@ def test_stop_other_host_during_backup(kill): node2.start_clickhouse() if status == "BACKUP_CREATED": - node1.query("DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + node1.query("DROP TABLE tbl ON CLUSTER 'cluster' SYNC") node1.query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}") assert node1.query("SELECT * FROM tbl ORDER BY x") == TSV([3, 5]) elif status == "BACKUP_FAILED": diff --git a/tests/integration/test_backup_restore_on_cluster/test_concurrency.py b/tests/integration/test_backup_restore_on_cluster/test_concurrency.py index a28a1fa142b..aea82c6b559 100644 --- a/tests/integration/test_backup_restore_on_cluster/test_concurrency.py +++ b/tests/integration/test_backup_restore_on_cluster/test_concurrency.py @@ -62,8 +62,8 @@ def drop_after_test(): try: yield finally: - node0.query("DROP TABLE IF EXISTS tbl ON CLUSTER 'cluster' NO DELAY") - node0.query("DROP DATABASE IF EXISTS mydb ON CLUSTER 'cluster' NO DELAY") + node0.query("DROP TABLE IF EXISTS tbl ON CLUSTER 'cluster' SYNC") + node0.query("DROP DATABASE IF EXISTS mydb ON CLUSTER 'cluster' SYNC") backup_id_counter = 0 @@ -95,7 +95,7 @@ def test_replicated_table(): backup_name = new_backup_name() node0.query(f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}") - node0.query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + node0.query(f"DROP TABLE tbl ON CLUSTER 'cluster' SYNC") node0.query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}") node0.query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster' tbl") @@ -131,7 +131,7 @@ def test_concurrent_backups_on_same_node(): ) == TSV([["BACKUP_CREATED", ""]] * num_concurrent_backups) for backup_name in backup_names: - node0.query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + node0.query(f"DROP TABLE tbl ON CLUSTER 'cluster' SYNC") node0.query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}") node0.query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster' tbl") for i in range(num_nodes): @@ -166,7 +166,7 @@ def test_concurrent_backups_on_different_nodes(): ) == TSV([["BACKUP_CREATED", ""]]) for i in range(num_concurrent_backups): - nodes[i].query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + nodes[i].query(f"DROP TABLE tbl ON CLUSTER 'cluster' SYNC") nodes[i].query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_names[i]}") nodes[i].query("SYSTEM SYNC REPLICA ON CLUSTER 'cluster' tbl") for j in range(num_nodes): @@ -214,7 +214,7 @@ def test_create_or_drop_tables_during_backup(db_engine, table_engine): while time.time() < end_time: table_name = f"mydb.tbl{randint(1, num_nodes)}" node = nodes[randint(0, num_nodes - 1)] - node.query(f"DROP TABLE IF EXISTS {table_name} NO DELAY") + node.query(f"DROP TABLE IF EXISTS {table_name} SYNC") def rename_tables(): while time.time() < end_time: @@ -229,7 +229,7 @@ def test_create_or_drop_tables_during_backup(db_engine, table_engine): while time.time() < end_time: table_name = f"mydb.tbl{randint(1, num_nodes)}" node = nodes[randint(0, num_nodes - 1)] - node.query(f"TRUNCATE TABLE IF EXISTS {table_name} NO DELAY") + node.query(f"TRUNCATE TABLE IF EXISTS {table_name} SYNC") def make_backups(): ids = [] @@ -320,8 +320,8 @@ def test_kill_mutation_during_backup(): TSV([["BACKUP_CREATED", ""]]), ) - node0.query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + node0.query(f"DROP TABLE tbl ON CLUSTER 'cluster' SYNC") node0.query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}") if n != repeat_count - 1: - node0.query(f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY") + node0.query(f"DROP TABLE tbl ON CLUSTER 'cluster' SYNC") diff --git a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py index 0d8fad96438..e5cd9ade68b 100644 --- a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py +++ b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py @@ -84,7 +84,7 @@ def drop_after_test(): yield finally: node0.query( - "DROP TABLE IF EXISTS tbl ON CLUSTER 'cluster' NO DELAY", + "DROP TABLE IF EXISTS tbl ON CLUSTER 'cluster' SYNC", settings={ "distributed_ddl_task_timeout": 360, }, @@ -154,7 +154,7 @@ def test_concurrent_backups_on_same_node(): # This restore part is added to confirm creating an internal backup & restore work # even when a concurrent backup is stopped nodes[0].query( - f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY", + f"DROP TABLE tbl ON CLUSTER 'cluster' SYNC", settings={ "distributed_ddl_task_timeout": 360, }, @@ -206,7 +206,7 @@ def test_concurrent_restores_on_same_node(): nodes[0].query(f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}") nodes[0].query( - f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY", + f"DROP TABLE tbl ON CLUSTER 'cluster' SYNC", settings={ "distributed_ddl_task_timeout": 360, }, @@ -251,7 +251,7 @@ def test_concurrent_restores_on_different_node(): nodes[0].query(f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}") nodes[0].query( - f"DROP TABLE tbl ON CLUSTER 'cluster' NO DELAY", + f"DROP TABLE tbl ON CLUSTER 'cluster' SYNC", settings={ "distributed_ddl_task_timeout": 360, }, diff --git a/tests/integration/test_backup_restore_s3/test.py b/tests/integration/test_backup_restore_s3/test.py index 0696d51136a..30c7bad7b83 100644 --- a/tests/integration/test_backup_restore_s3/test.py +++ b/tests/integration/test_backup_restore_s3/test.py @@ -37,7 +37,7 @@ def new_backup_name(): def check_backup_and_restore(storage_policy, backup_destination, size=1000): node.query( f""" - DROP TABLE IF EXISTS data NO DELAY; + DROP TABLE IF EXISTS data SYNC; CREATE TABLE data (key Int, value String, array Array(String)) Engine=MergeTree() ORDER BY tuple() SETTINGS storage_policy='{storage_policy}'; INSERT INTO data SELECT * FROM generateRandom('key Int, value String, array Array(String)') LIMIT {size}; BACKUP TABLE data TO {backup_destination}; @@ -47,8 +47,8 @@ def check_backup_and_restore(storage_policy, backup_destination, size=1000): (SELECT count(), sum(sipHash64(*)) FROM data_restored), 'Data does not matched after BACKUP/RESTORE' ); - DROP TABLE data NO DELAY; - DROP TABLE data_restored NO DELAY; + DROP TABLE data SYNC; + DROP TABLE data_restored SYNC; """ ) diff --git a/tests/integration/test_concurrent_backups_s3/test.py b/tests/integration/test_concurrent_backups_s3/test.py index 73692e52cce..b29058865c0 100644 --- a/tests/integration/test_concurrent_backups_s3/test.py +++ b/tests/integration/test_concurrent_backups_s3/test.py @@ -25,7 +25,7 @@ def start_cluster(): def test_concurrent_backups(start_cluster): - node.query("DROP TABLE IF EXISTS s3_test NO DELAY") + node.query("DROP TABLE IF EXISTS s3_test SYNC") columns = [f"column_{i} UInt64" for i in range(1000)] columns_str = ", ".join(columns) node.query( diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/configs/ssl_verification.xml b/tests/integration/test_dictionaries_all_layouts_separate_sources/configs/ssl_verification.xml new file mode 100644 index 00000000000..3efe98e7045 --- /dev/null +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/configs/ssl_verification.xml @@ -0,0 +1,8 @@ + + + + + none + + + diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py index 55639877ba0..973dbfc0429 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo.py @@ -17,14 +17,71 @@ ranged_tester = None test_name = "mongo" -def setup_module(module): - global cluster - global node - global simple_tester - global complex_tester - global ranged_tester +@pytest.fixture(scope="module") +def secure_connection(request): + return request.param - cluster = ClickHouseCluster(__file__) + +@pytest.fixture(scope="module") +def cluster(secure_connection): + return ClickHouseCluster(__file__) + + +@pytest.fixture(scope="module") +def source(secure_connection, cluster): + return SourceMongo( + "MongoDB", + "localhost", + cluster.mongo_port, + cluster.mongo_host, + "27017", + "root", + "clickhouse", + secure=secure_connection, + ) + + +@pytest.fixture(scope="module") +def simple_tester(source): + tester = SimpleLayoutTester(test_name) + tester.cleanup() + tester.create_dictionaries(source) + return tester + + +@pytest.fixture(scope="module") +def complex_tester(source): + tester = ComplexLayoutTester(test_name) + tester.create_dictionaries(source) + return tester + + +@pytest.fixture(scope="module") +def ranged_tester(source): + tester = RangedLayoutTester(test_name) + tester.create_dictionaries(source) + return tester + + +@pytest.fixture(scope="module") +def main_config(secure_connection): + main_config = [] + if secure_connection: + main_config.append(os.path.join("configs", "disable_ssl_verification.xml")) + else: + main_config.append(os.path.join("configs", "ssl_verification.xml")) + return main_config + + +@pytest.fixture(scope="module") +def started_cluster( + secure_connection, + cluster, + main_config, + simple_tester, + ranged_tester, + complex_tester, +): SOURCE = SourceMongo( "MongoDB", "localhost", @@ -33,35 +90,18 @@ def setup_module(module): "27017", "root", "clickhouse", + secure=secure_connection, ) - - simple_tester = SimpleLayoutTester(test_name) - simple_tester.cleanup() - simple_tester.create_dictionaries(SOURCE) - - complex_tester = ComplexLayoutTester(test_name) - complex_tester.create_dictionaries(SOURCE) - - ranged_tester = RangedLayoutTester(test_name) - ranged_tester.create_dictionaries(SOURCE) - # Since that all .xml configs were created - - main_configs = [] - main_configs.append(os.path.join("configs", "disable_ssl_verification.xml")) - dictionaries = simple_tester.list_dictionaries() node = cluster.add_instance( - "node", main_configs=main_configs, dictionaries=dictionaries, with_mongo=True + "node", + main_configs=main_config, + dictionaries=dictionaries, + with_mongo=True, + with_mongo_secure=secure_connection, ) - -def teardown_module(module): - simple_tester.cleanup() - - -@pytest.fixture(scope="module") -def started_cluster(): try: cluster.start() @@ -75,16 +115,25 @@ def started_cluster(): cluster.shutdown() +@pytest.mark.parametrize("secure_connection", [False], indirect=["secure_connection"]) @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_SIMPLE)) -def test_simple(started_cluster, layout_name): - simple_tester.execute(layout_name, node) +def test_simple(secure_connection, started_cluster, layout_name, simple_tester): + simple_tester.execute(layout_name, started_cluster.instances["node"]) +@pytest.mark.parametrize("secure_connection", [False], indirect=["secure_connection"]) @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_COMPLEX)) -def test_complex(started_cluster, layout_name): - complex_tester.execute(layout_name, node) +def test_complex(secure_connection, started_cluster, layout_name, complex_tester): + complex_tester.execute(layout_name, started_cluster.instances["node"]) +@pytest.mark.parametrize("secure_connection", [False], indirect=["secure_connection"]) @pytest.mark.parametrize("layout_name", sorted(LAYOUTS_RANGED)) -def test_ranged(started_cluster, layout_name): - ranged_tester.execute(layout_name, node) +def test_ranged(secure_connection, started_cluster, layout_name, ranged_tester): + ranged_tester.execute(layout_name, started_cluster.instances["node"]) + + +@pytest.mark.parametrize("secure_connection", [True], indirect=["secure_connection"]) +@pytest.mark.parametrize("layout_name", sorted(LAYOUTS_SIMPLE)) +def test_simple_ssl(secure_connection, started_cluster, layout_name, simple_tester): + simple_tester.execute(layout_name, started_cluster.instances["node"]) diff --git a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py index 84c547b7a6b..22541432259 100644 --- a/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py +++ b/tests/integration/test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py @@ -8,25 +8,22 @@ from helpers.cluster import ClickHouseCluster from helpers.dictionary import Field, Row, Dictionary, DictionaryStructure, Layout from helpers.external_sources import SourceMongoURI -SOURCE = None -cluster = None -node = None -simple_tester = None -complex_tester = None -ranged_tester = None test_name = "mongo_uri" -def setup_module(module): - global cluster - global node - global simple_tester - global complex_tester - global ranged_tester +@pytest.fixture(scope="module") +def secure_connection(request): + return request.param - cluster = ClickHouseCluster(__file__) - SOURCE = SourceMongoURI( +@pytest.fixture(scope="module") +def cluster(secure_connection): + return ClickHouseCluster(__file__) + + +@pytest.fixture(scope="module") +def source(secure_connection, cluster): + return SourceMongoURI( "MongoDB", "localhost", cluster.mongo_port, @@ -34,52 +31,55 @@ def setup_module(module): "27017", "root", "clickhouse", + secure=secure_connection, ) - simple_tester = SimpleLayoutTester(test_name) - simple_tester.cleanup() - simple_tester.create_dictionaries(SOURCE) - complex_tester = ComplexLayoutTester(test_name) - complex_tester.create_dictionaries(SOURCE) +@pytest.fixture(scope="module") +def simple_tester(source): + tester = SimpleLayoutTester(test_name) + tester.cleanup() + tester.create_dictionaries(source) + return tester - ranged_tester = RangedLayoutTester(test_name) - ranged_tester.create_dictionaries(SOURCE) - # Since that all .xml configs were created - main_configs = [] - main_configs.append(os.path.join("configs", "disable_ssl_verification.xml")) +@pytest.fixture(scope="module") +def main_config(secure_connection): + main_config = [] + if secure_connection: + main_config.append(os.path.join("configs", "disable_ssl_verification.xml")) + else: + main_config.append(os.path.join("configs", "ssl_verification.xml")) + return main_config + +@pytest.fixture(scope="module") +def started_cluster(secure_connection, cluster, main_config, simple_tester): dictionaries = simple_tester.list_dictionaries() node = cluster.add_instance( "uri_node", - main_configs=main_configs, + main_configs=main_config, dictionaries=dictionaries, with_mongo=True, + with_mongo_secure=secure_connection, ) - - -def teardown_module(module): - simple_tester.cleanup() - - -@pytest.fixture(scope="module") -def started_cluster(): try: cluster.start() - simple_tester.prepare(cluster) - complex_tester.prepare(cluster) - ranged_tester.prepare(cluster) - yield cluster - finally: cluster.shutdown() # See comment in SourceMongoURI +@pytest.mark.parametrize("secure_connection", [False], indirect=["secure_connection"]) @pytest.mark.parametrize("layout_name", ["flat"]) -def test_simple(started_cluster, layout_name): - simple_tester.execute(layout_name, node) +def test_simple(secure_connection, started_cluster, simple_tester, layout_name): + simple_tester.execute(layout_name, started_cluster.instances["uri_node"]) + + +@pytest.mark.parametrize("secure_connection", [True], indirect=["secure_connection"]) +@pytest.mark.parametrize("layout_name", ["flat"]) +def test_simple_ssl(secure_connection, started_cluster, simple_tester, layout_name): + simple_tester.execute(layout_name, started_cluster.instances["uri_node"]) diff --git a/tests/integration/test_disk_configuration/test.py b/tests/integration/test_disk_configuration/test.py index 6ebe994dc68..3fe8286fa43 100644 --- a/tests/integration/test_disk_configuration/test.py +++ b/tests/integration/test_disk_configuration/test.py @@ -390,7 +390,7 @@ def test_merge_tree_setting_override(start_cluster): node.query( f""" - DROP TABLE IF EXISTS {TABLE_NAME} NO DELAY; + DROP TABLE IF EXISTS {TABLE_NAME} SYNC; CREATE TABLE {TABLE_NAME} (a Int32) ENGINE = MergeTree() ORDER BY tuple() @@ -412,7 +412,7 @@ def test_merge_tree_setting_override(start_cluster): node.query( f""" - DROP TABLE IF EXISTS {TABLE_NAME} NO DELAY; + DROP TABLE IF EXISTS {TABLE_NAME} SYNC; CREATE TABLE {TABLE_NAME} (a Int32) ENGINE = MergeTree() ORDER BY tuple() diff --git a/tests/integration/test_encrypted_disk/test.py b/tests/integration/test_encrypted_disk/test.py index 681df89dd0f..8187f2ff6a8 100644 --- a/tests/integration/test_encrypted_disk/test.py +++ b/tests/integration/test_encrypted_disk/test.py @@ -30,7 +30,7 @@ def cleanup_after_test(): try: yield finally: - node.query("DROP TABLE IF EXISTS encrypted_test NO DELAY") + node.query("DROP TABLE IF EXISTS encrypted_test SYNC") @pytest.mark.parametrize( @@ -294,4 +294,4 @@ def test_restart(): assert node.query(select_query) == "(0,'data'),(1,'data')" - node.query("DROP TABLE encrypted_test NO DELAY;") + node.query("DROP TABLE encrypted_test SYNC;") diff --git a/tests/integration/test_encrypted_disk_replication/test.py b/tests/integration/test_encrypted_disk_replication/test.py index f68c534ed43..c2aa710ba91 100644 --- a/tests/integration/test_encrypted_disk_replication/test.py +++ b/tests/integration/test_encrypted_disk_replication/test.py @@ -44,7 +44,7 @@ def cleanup_after_test(): try: yield finally: - node1.query("DROP TABLE IF EXISTS encrypted_test ON CLUSTER 'cluster' NO DELAY") + node1.query("DROP TABLE IF EXISTS encrypted_test ON CLUSTER 'cluster' SYNC") def create_table( @@ -52,9 +52,7 @@ def create_table( ): engine = "ReplicatedMergeTree('/clickhouse/tables/encrypted_test/', '{replica}')" - settings = f"storage_policy='{storage_policy}'" - if zero_copy_replication: - settings += ", allow_remote_fs_zero_copy_replication=true" + settings = f"storage_policy='{storage_policy}', allow_remote_fs_zero_copy_replication={int(zero_copy_replication)}" node1.query( f""" diff --git a/tests/integration/test_failed_async_inserts/test.py b/tests/integration/test_failed_async_inserts/test.py index 6d66ac97006..ecb506c36bc 100644 --- a/tests/integration/test_failed_async_inserts/test.py +++ b/tests/integration/test_failed_async_inserts/test.py @@ -51,4 +51,4 @@ def test_failed_async_inserts(started_cluster): assert node.query(select_query) == "4\n" - node.query("DROP TABLE IF EXISTS async_insert_30_10_2022 NO DELAY") + node.query("DROP TABLE IF EXISTS async_insert_30_10_2022 SYNC") diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py index 4ad046fe5d2..ee5d4b5df93 100644 --- a/tests/integration/test_grant_and_revoke/test.py +++ b/tests/integration/test_grant_and_revoke/test.py @@ -20,6 +20,9 @@ def start_cluster(): instance.query( "CREATE TABLE test.table(x UInt32, y UInt32) ENGINE = MergeTree ORDER BY tuple()" ) + instance.query( + "CREATE TABLE test.table2(x UInt32, y UInt32) ENGINE = MergeTree ORDER BY tuple()" + ) instance.query("INSERT INTO test.table VALUES (1,5), (2,10)") yield cluster @@ -585,3 +588,134 @@ def test_grant_with_replace_option(): assert instance.query("SHOW GRANTS FOR B") == TSV( ["GRANT INSERT ON test.table TO B"] ) + + +def test_grant_current_grants(): + instance.query("CREATE USER A") + instance.query( + "GRANT SELECT, CREATE TABLE, CREATE VIEW ON test.* TO A WITH GRANT OPTION" + ) + assert instance.query("SHOW GRANTS FOR A") == TSV( + ["GRANT SELECT, CREATE TABLE, CREATE VIEW ON test.* TO A WITH GRANT OPTION"] + ) + + instance.query("CREATE USER B") + instance.query("GRANT CURRENT GRANTS ON *.* TO B", user="A") + assert instance.query("SHOW GRANTS FOR B") == TSV( + ["GRANT SELECT, CREATE TABLE, CREATE VIEW ON test.* TO B"] + ) + + instance.query("CREATE USER C") + instance.query("GRANT CURRENT GRANTS(CREATE ON test.*) TO C", user="A") + assert instance.query("SHOW GRANTS FOR C") == TSV( + ["GRANT CREATE TABLE, CREATE VIEW ON test.* TO C"] + ) + + instance.query("DROP USER IF EXISTS C") + instance.query("CREATE USER C") + instance.query("GRANT CURRENT GRANTS(NONE ON *.*) TO C", user="A") + assert instance.query("SHOW GRANTS FOR C") == TSV([]) + + +def test_grant_current_grants_with_partial_revoke(): + instance.query("CREATE USER A") + instance.query("GRANT CREATE TABLE ON *.* TO A") + instance.query("REVOKE CREATE TABLE ON test.* FROM A") + instance.query("GRANT CREATE TABLE ON test.table TO A WITH GRANT OPTION") + instance.query("GRANT SELECT ON *.* TO A WITH GRANT OPTION") + instance.query("REVOKE SELECT ON test.* FROM A") + instance.query("GRANT SELECT ON test.table TO A WITH GRANT OPTION") + instance.query("GRANT SELECT ON test.table2 TO A") + + assert instance.query("SHOW GRANTS FOR A") == TSV( + [ + "GRANT CREATE TABLE ON *.* TO A", + "GRANT SELECT ON *.* TO A WITH GRANT OPTION", + "REVOKE SELECT, CREATE TABLE ON test.* FROM A", + "GRANT SELECT, CREATE TABLE ON test.table TO A WITH GRANT OPTION", + "GRANT SELECT ON test.table2 TO A", + ] + ) + + instance.query("CREATE USER B") + instance.query("GRANT CURRENT GRANTS ON *.* TO B", user="A") + assert instance.query("SHOW GRANTS FOR B") == TSV( + [ + "GRANT SELECT ON *.* TO B", + "REVOKE SELECT ON test.* FROM B", + "GRANT SELECT, CREATE TABLE ON test.table TO B", + ] + ) + + instance.query("DROP USER IF EXISTS B") + instance.query("CREATE USER B") + instance.query("GRANT CURRENT GRANTS ON *.* TO B WITH GRANT OPTION", user="A") + assert instance.query("SHOW GRANTS FOR B") == TSV( + [ + "GRANT SELECT ON *.* TO B WITH GRANT OPTION", + "REVOKE SELECT ON test.* FROM B", + "GRANT SELECT, CREATE TABLE ON test.table TO B WITH GRANT OPTION", + ] + ) + + instance.query("DROP USER IF EXISTS C") + instance.query("CREATE USER C") + instance.query("GRANT SELECT ON test.* TO B") + instance.query("GRANT CURRENT GRANTS ON *.* TO C", user="B") + assert instance.query("SHOW GRANTS FOR C") == TSV( + [ + "GRANT SELECT ON *.* TO C", + "GRANT CREATE TABLE ON test.table TO C", + ] + ) + + instance.query("DROP USER IF EXISTS B") + instance.query("CREATE USER B") + instance.query("GRANT CURRENT GRANTS ON test.* TO B WITH GRANT OPTION", user="A") + assert instance.query("SHOW GRANTS FOR B") == TSV( + [ + "GRANT SELECT, CREATE TABLE ON test.table TO B WITH GRANT OPTION", + ] + ) + + +def test_current_grants_override(): + instance.query("CREATE USER A") + instance.query("GRANT SELECT ON *.* TO A WITH GRANT OPTION") + instance.query("REVOKE SELECT ON test.* FROM A") + assert instance.query("SHOW GRANTS FOR A") == TSV( + [ + "GRANT SELECT ON *.* TO A WITH GRANT OPTION", + "REVOKE SELECT ON test.* FROM A", + ] + ) + + instance.query("CREATE USER B") + instance.query("GRANT SELECT ON test.table TO B") + assert instance.query("SHOW GRANTS FOR B") == TSV( + ["GRANT SELECT ON test.table TO B"] + ) + + instance.query("GRANT CURRENT GRANTS ON *.* TO B", user="A") + assert instance.query("SHOW GRANTS FOR B") == TSV( + [ + "GRANT SELECT ON *.* TO B", + "REVOKE SELECT ON test.* FROM B", + "GRANT SELECT ON test.table TO B", + ] + ) + + instance.query("DROP USER IF EXISTS B") + instance.query("CREATE USER B") + instance.query("GRANT SELECT ON test.table TO B") + assert instance.query("SHOW GRANTS FOR B") == TSV( + ["GRANT SELECT ON test.table TO B"] + ) + + instance.query("GRANT CURRENT GRANTS ON *.* TO B WITH REPLACE OPTION", user="A") + assert instance.query("SHOW GRANTS FOR B") == TSV( + [ + "GRANT SELECT ON *.* TO B", + "REVOKE SELECT ON test.* FROM B", + ] + ) diff --git a/tests/integration/test_hedged_requests/test.py b/tests/integration/test_hedged_requests/test.py index 88371f6908d..2ca37fbb7ee 100644 --- a/tests/integration/test_hedged_requests/test.py +++ b/tests/integration/test_hedged_requests/test.py @@ -128,12 +128,12 @@ def check_changing_replica_events(expected_count): assert int(result) >= expected_count -def check_if_query_sending_was_suspended(minimum_count): +def check_if_query_sending_was_suspended(): result = NODES["node"].query( "SELECT value FROM system.events WHERE event='SuspendSendingQueryToShard'" ) - assert int(result) >= minimum_count + assert int(result) >= 1 def check_if_query_sending_was_not_suspended(): @@ -381,7 +381,7 @@ def test_async_connect(started_cluster): "SELECT hostName(), id FROM distributed_connect ORDER BY id LIMIT 1 SETTINGS prefer_localhost_replica = 0, connect_timeout_with_failover_ms=5000, async_query_sending_for_remote=1, max_threads=1" ) check_changing_replica_events(2) - check_if_query_sending_was_suspended(2) + check_if_query_sending_was_suspended() NODES["node"].query("DROP TABLE distributed_connect") @@ -406,7 +406,7 @@ def test_async_query_sending(started_cluster): NODES["node"].query("DROP TABLE IF EXISTS tmp") NODES["node"].query( "CREATE TEMPORARY TABLE tmp (number UInt64, s String) " - "as select number, randomString(number % 1000) from numbers(1000000)" + "as select number, randomString(number % 1000) from numbers(10000000)" ) NODES["node"].query( @@ -419,6 +419,6 @@ def test_async_query_sending(started_cluster): "SELECT hostName(), id FROM distributed_query_sending ORDER BY id LIMIT 1 SETTINGS" " prefer_localhost_replica = 0, async_query_sending_for_remote=1, max_threads = 1" ) - check_if_query_sending_was_suspended(3) + check_if_query_sending_was_suspended() NODES["node"].query("DROP TABLE distributed_query_sending") diff --git a/tests/integration/test_mask_sensitive_info/configs/named_collections.xml b/tests/integration/test_mask_sensitive_info/configs/named_collections.xml index a4b58f6f812..3d294874d68 100644 --- a/tests/integration/test_mask_sensitive_info/configs/named_collections.xml +++ b/tests/integration/test_mask_sensitive_info/configs/named_collections.xml @@ -1,4 +1,5 @@ + 1 diff --git a/tests/integration/test_mask_sensitive_info/test.py b/tests/integration/test_mask_sensitive_info/test.py index 92232f7e6a8..2131a76b5be 100644 --- a/tests/integration/test_mask_sensitive_info/test.py +++ b/tests/integration/test_mask_sensitive_info/test.py @@ -65,14 +65,15 @@ def system_query_log_contains_search_pattern(search_pattern): ) -# Generates a random string. def new_password(len=16): return "".join( random.choice(string.ascii_uppercase + string.digits) for _ in range(len) ) -# Passwords in CREATE/ALTER queries must be hidden in logs. +show_secrets = "SETTINGS format_display_secrets_in_show_and_select" + + def test_create_alter_user(): password = new_password() @@ -95,21 +96,37 @@ def test_create_alter_user(): check_logs( must_contain=[ - "CREATE USER u1 IDENTIFIED WITH sha256_password", - "ALTER USER u1 IDENTIFIED WITH sha256_password", + "CREATE USER u1 IDENTIFIED", + "ALTER USER u1 IDENTIFIED", "CREATE USER u2 IDENTIFIED WITH plaintext_password", ], must_not_contain=[ password, - "IDENTIFIED WITH sha256_password BY", - "IDENTIFIED WITH sha256_hash BY", + "IDENTIFIED BY", + "IDENTIFIED BY", "IDENTIFIED WITH plaintext_password BY", ], ) + assert "BY" in node.query(f"SHOW CREATE USER u1 {show_secrets}=1") + assert "BY" in node.query(f"SHOW CREATE USER u2 {show_secrets}=1") + node.query("DROP USER u1, u2") +def check_secrets_for_tables(tables, table_name_prefix, password): + for i, table in enumerate(tables): + table_name = table_name_prefix + str(i) + if password in table: + assert password in node.query( + f"SHOW CREATE TABLE {table_name} {show_secrets}=1" + ) + assert password in node.query( + f"SELECT create_table_query, engine_full FROM system.tables WHERE name = '{table_name}' " + f"{show_secrets}=1" + ) + + def test_create_table(): password = new_password() @@ -133,21 +150,25 @@ def test_create_table(): for i, table_engine in enumerate(table_engines): node.query(f"CREATE TABLE table{i} (x int) ENGINE = {table_engine}") - assert ( - node.query("SHOW CREATE TABLE table0") - == "CREATE TABLE default.table0\\n(\\n `x` Int32\\n)\\nENGINE = MySQL(\\'mysql57:3306\\', \\'mysql_db\\', \\'mysql_table\\', \\'mysql_user\\', \\'[HIDDEN]\\')\n" - ) + for toggle, secret in enumerate(["[HIDDEN]", password]): + assert ( + node.query(f"SHOW CREATE TABLE table0 {show_secrets}={toggle}") + == "CREATE TABLE default.table0\\n(\\n `x` Int32\\n)\\n" + "ENGINE = MySQL(\\'mysql57:3306\\', \\'mysql_db\\', " + f"\\'mysql_table\\', \\'mysql_user\\', \\'{secret}\\')\n" + ) - assert node.query( - "SELECT create_table_query, engine_full FROM system.tables WHERE name = 'table0'" - ) == TSV( - [ + assert node.query( + f"SELECT create_table_query, engine_full FROM system.tables WHERE name = 'table0' {show_secrets}={toggle}" + ) == TSV( [ - "CREATE TABLE default.table0 (`x` Int32) ENGINE = MySQL(\\'mysql57:3306\\', \\'mysql_db\\', \\'mysql_table\\', \\'mysql_user\\', \\'[HIDDEN]\\')", - "MySQL(\\'mysql57:3306\\', \\'mysql_db\\', \\'mysql_table\\', \\'mysql_user\\', \\'[HIDDEN]\\')", - ], - ] - ) + [ + "CREATE TABLE default.table0 (`x` Int32) ENGINE = MySQL(\\'mysql57:3306\\', \\'mysql_db\\', " + f"\\'mysql_table\\', \\'mysql_user\\', \\'{secret}\\')", + f"MySQL(\\'mysql57:3306\\', \\'mysql_db\\', \\'mysql_table\\', \\'mysql_user\\', \\'{secret}\\')", + ], + ] + ) check_logs( must_contain=[ @@ -169,7 +190,9 @@ def test_create_table(): must_not_contain=[password], ) - for i in range(0, len(table_engines)): + check_secrets_for_tables(table_engines, "table", password) + + for i in range(len(table_engines)): node.query(f"DROP TABLE table{i}") @@ -198,7 +221,7 @@ def test_create_database(): must_not_contain=[password], ) - for i in range(0, len(database_engines)): + for i in range(len(database_engines)): node.query(f"DROP DATABASE IF EXISTS database{i}") @@ -241,21 +264,26 @@ def test_table_functions(): for i, table_function in enumerate(table_functions): node.query(f"CREATE TABLE tablefunc{i} (x int) AS {table_function}") - assert ( - node.query("SHOW CREATE TABLE tablefunc0") - == "CREATE TABLE default.tablefunc0\\n(\\n `x` Int32\\n) AS mysql(\\'mysql57:3306\\', \\'mysql_db\\', \\'mysql_table\\', \\'mysql_user\\', \\'[HIDDEN]\\')\n" - ) + for toggle, secret in enumerate(["[HIDDEN]", password]): + assert ( + node.query(f"SHOW CREATE TABLE tablefunc0 {show_secrets}={toggle}") + == "CREATE TABLE default.tablefunc0\\n(\\n `x` Int32\\n) AS " + "mysql(\\'mysql57:3306\\', \\'mysql_db\\', \\'mysql_table\\', " + f"\\'mysql_user\\', \\'{secret}\\')\n" + ) - assert node.query( - "SELECT create_table_query, engine_full FROM system.tables WHERE name = 'tablefunc0'" - ) == TSV( - [ + assert node.query( + "SELECT create_table_query, engine_full FROM system.tables WHERE name = 'tablefunc0' " + f"{show_secrets}={toggle}" + ) == TSV( [ - "CREATE TABLE default.tablefunc0 (`x` Int32) AS mysql(\\'mysql57:3306\\', \\'mysql_db\\', \\'mysql_table\\', \\'mysql_user\\', \\'[HIDDEN]\\')", - "", - ], - ] - ) + [ + "CREATE TABLE default.tablefunc0 (`x` Int32) AS mysql(\\'mysql57:3306\\', " + f"\\'mysql_db\\', \\'mysql_table\\', \\'mysql_user\\', \\'{secret}\\')", + "", + ], + ] + ) check_logs( must_contain=[ @@ -293,7 +321,9 @@ def test_table_functions(): must_not_contain=[password], ) - for i in range(0, len(table_functions)): + check_secrets_for_tables(table_functions, "tablefunc", password) + + for i in range(len(table_functions)): node.query(f"DROP TABLE tablefunc{i}") @@ -369,15 +399,18 @@ def test_create_dictionary(): f"LIFETIME(MIN 0 MAX 10) LAYOUT(FLAT())" ) - assert ( - node.query("SHOW CREATE TABLE dict1") - == "CREATE DICTIONARY default.dict1\\n(\\n `n` int DEFAULT 0,\\n `m` int DEFAULT 1\\n)\\nPRIMARY KEY n\\nSOURCE(CLICKHOUSE(HOST \\'localhost\\' PORT 9000 USER \\'user1\\' TABLE \\'test\\' PASSWORD \\'[HIDDEN]\\' DB \\'default\\'))\\nLIFETIME(MIN 0 MAX 10)\\nLAYOUT(FLAT())\n" - ) + for toggle, secret in enumerate(["[HIDDEN]", password]): + assert ( + node.query(f"SHOW CREATE TABLE dict1 {show_secrets}={toggle}") + == f"CREATE DICTIONARY default.dict1\\n(\\n `n` int DEFAULT 0,\\n `m` int DEFAULT 1\\n)\\nPRIMARY KEY n\\nSOURCE(CLICKHOUSE(HOST \\'localhost\\' PORT 9000 USER \\'user1\\' TABLE \\'test\\' PASSWORD \\'{secret}\\' DB \\'default\\'))\\nLIFETIME(MIN 0 MAX 10)\\nLAYOUT(FLAT())\n" + ) - assert ( - node.query("SELECT create_table_query FROM system.tables WHERE name = 'dict1'") - == "CREATE DICTIONARY default.dict1 (`n` int DEFAULT 0, `m` int DEFAULT 1) PRIMARY KEY n SOURCE(CLICKHOUSE(HOST \\'localhost\\' PORT 9000 USER \\'user1\\' TABLE \\'test\\' PASSWORD \\'[HIDDEN]\\' DB \\'default\\')) LIFETIME(MIN 0 MAX 10) LAYOUT(FLAT())\n" - ) + assert ( + node.query( + f"SELECT create_table_query FROM system.tables WHERE name = 'dict1' {show_secrets}={toggle}" + ) + == f"CREATE DICTIONARY default.dict1 (`n` int DEFAULT 0, `m` int DEFAULT 1) PRIMARY KEY n SOURCE(CLICKHOUSE(HOST \\'localhost\\' PORT 9000 USER \\'user1\\' TABLE \\'test\\' PASSWORD \\'{secret}\\' DB \\'default\\')) LIFETIME(MIN 0 MAX 10) LAYOUT(FLAT())\n" + ) check_logs( must_contain=[ @@ -448,4 +481,4 @@ def test_on_cluster(): "%CREATE TABLE default.table_oncl UUID \\'%\\' (`x` Int32) ENGINE = MySQL(\\'mysql57:3307\\', \\'mysql_db\\', \\'mysql_table\\', \\'mysql_user\\', \\'[HIDDEN]\\')" ) - node.query(f"DROP TABLE table_oncl") + node.query("DROP TABLE table_oncl") diff --git a/tests/integration/test_max_rows_to_read_leaf_with_view/__init__.py b/tests/integration/test_max_rows_to_read_leaf_with_view/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_max_rows_to_read_leaf_with_view/configs/remote_servers.xml b/tests/integration/test_max_rows_to_read_leaf_with_view/configs/remote_servers.xml new file mode 100644 index 00000000000..9ce90edb727 --- /dev/null +++ b/tests/integration/test_max_rows_to_read_leaf_with_view/configs/remote_servers.xml @@ -0,0 +1,18 @@ + + + + + + node1 + 9000 + + + + + node2 + 9000 + + + + + diff --git a/tests/integration/test_max_rows_to_read_leaf_with_view/test.py b/tests/integration/test_max_rows_to_read_leaf_with_view/test.py new file mode 100755 index 00000000000..6957534ce0d --- /dev/null +++ b/tests/integration/test_max_rows_to_read_leaf_with_view/test.py @@ -0,0 +1,76 @@ +from contextlib import contextmanager + +import pytest +from helpers.cluster import ClickHouseCluster +from helpers.client import QueryRuntimeException + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance( + "node1", + main_configs=["configs/remote_servers.xml"], + with_zookeeper=True, +) + +node2 = cluster.add_instance( + "node2", + main_configs=["configs/remote_servers.xml"], + with_zookeeper=True, +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + for node in (node1, node2): + node.query( + f""" + CREATE TABLE local_table(id UInt32, d DateTime) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/max_rows_read_leaf', '{node}') PARTITION BY toYYYYMM(d) ORDER BY d; + + CREATE TABLE distributed_table(id UInt32, d DateTime) ENGINE = Distributed(two_shards, default, local_table); + + CREATE OR REPLACE VIEW test_view AS select id from distributed_table; +""" + ) + node1.query( + "INSERT INTO local_table (id) select * from system.numbers limit 200" + ) + node2.query( + "INSERT INTO local_table (id) select * from system.numbers limit 200" + ) + + yield cluster + + finally: + cluster.shutdown() + + +def test_max_rows_to_read_leaf_via_view(started_cluster): + """ + Asserts the expected behaviour that we should be able to select + the total amount of rows (400 - 200 from each shard) from a + view that selects from a distributed table. + """ + assert ( + node1.query( + "SELECT count() from test_view SETTINGS max_rows_to_read_leaf=200" + ).rstrip() + == "400" + ) + with pytest.raises( + QueryRuntimeException, match="controlled by 'max_rows_to_read_leaf'" + ): + # insert some more data and ensure we get a legitimate failure + node2.query( + "INSERT INTO local_table (id) select * from system.numbers limit 10" + ) + node2.query("SELECT count() from test_view SETTINGS max_rows_to_read_leaf=200") + + +if __name__ == "__main__": + with contextmanager(started_cluster)() as cluster: + for name, instance in list(cluster.instances.items()): + print(name, instance.ip_address) + input("Cluster created, press any key to destroy...") diff --git a/tests/integration/test_merge_tree_azure_blob_storage/test.py b/tests/integration/test_merge_tree_azure_blob_storage/test.py index d1b7b64b56f..bcb62c3181d 100644 --- a/tests/integration/test_merge_tree_azure_blob_storage/test.py +++ b/tests/integration/test_merge_tree_azure_blob_storage/test.py @@ -461,7 +461,7 @@ def test_move_replace_partition_to_another_table(cluster): == "(512)" ) - azure_query(node, f"DROP TABLE {table_clone_name} NO DELAY") + azure_query(node, f"DROP TABLE {table_clone_name} SYNC") assert azure_query(node, f"SELECT sum(id) FROM {TABLE_NAME} FORMAT Values") == "(0)" assert ( azure_query(node, f"SELECT count(*) FROM {TABLE_NAME} FORMAT Values") @@ -470,7 +470,7 @@ def test_move_replace_partition_to_another_table(cluster): azure_query(node, f"ALTER TABLE {TABLE_NAME} FREEZE") - azure_query(node, f"DROP TABLE {TABLE_NAME} NO DELAY") + azure_query(node, f"DROP TABLE {TABLE_NAME} SYNC") def test_freeze_unfreeze(cluster): diff --git a/tests/integration/test_merge_tree_hdfs/test.py b/tests/integration/test_merge_tree_hdfs/test.py index 782237539fa..c79986c34f0 100644 --- a/tests/integration/test_merge_tree_hdfs/test.py +++ b/tests/integration/test_merge_tree_hdfs/test.py @@ -224,14 +224,22 @@ def test_attach_detach_partition(cluster): wait_for_delete_empty_parts(node, "hdfs_test") wait_for_delete_inactive_parts(node, "hdfs_test") wait_for_delete_hdfs_objects( - cluster, FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 + cluster, + FILES_OVERHEAD + + FILES_OVERHEAD_PER_PART_WIDE * 2 + - FILES_OVERHEAD_METADATA_VERSION, ) node.query("ALTER TABLE hdfs_test ATTACH PARTITION '2020-01-03'") assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(8192)" hdfs_objects = fs.listdir("/clickhouse") - assert len(hdfs_objects) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 + assert ( + len(hdfs_objects) + == FILES_OVERHEAD + + FILES_OVERHEAD_PER_PART_WIDE * 2 + - FILES_OVERHEAD_METADATA_VERSION + ) node.query("ALTER TABLE hdfs_test DROP PARTITION '2020-01-03'") assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(4096)" @@ -355,7 +363,14 @@ def test_move_replace_partition_to_another_table(cluster): # Number of objects in HDFS should be unchanged. hdfs_objects = fs.listdir("/clickhouse") - assert len(hdfs_objects) == FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 4 + for obj in hdfs_objects: + print("Object in HDFS after move", obj) + wait_for_delete_hdfs_objects( + cluster, + FILES_OVERHEAD * 2 + + FILES_OVERHEAD_PER_PART_WIDE * 4 + - FILES_OVERHEAD_METADATA_VERSION * 2, + ) # Add new partitions to source table, but with different values and replace them from copied table. node.query( @@ -370,7 +385,15 @@ def test_move_replace_partition_to_another_table(cluster): assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(16384)" hdfs_objects = fs.listdir("/clickhouse") - assert len(hdfs_objects) == FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 6 + for obj in hdfs_objects: + print("Object in HDFS after insert", obj) + + wait_for_delete_hdfs_objects( + cluster, + FILES_OVERHEAD * 2 + + FILES_OVERHEAD_PER_PART_WIDE * 6 + - FILES_OVERHEAD_METADATA_VERSION * 2, + ) node.query("ALTER TABLE hdfs_test REPLACE PARTITION '2020-01-03' FROM hdfs_clone") node.query("ALTER TABLE hdfs_test REPLACE PARTITION '2020-01-05' FROM hdfs_clone") @@ -381,13 +404,25 @@ def test_move_replace_partition_to_another_table(cluster): # Wait for outdated partitions deletion. wait_for_delete_hdfs_objects( - cluster, FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 4 + cluster, + FILES_OVERHEAD * 2 + + FILES_OVERHEAD_PER_PART_WIDE * 4 + - FILES_OVERHEAD_METADATA_VERSION * 2, ) - node.query("DROP TABLE hdfs_clone NO DELAY") + node.query("DROP TABLE hdfs_clone SYNC") assert node.query("SELECT sum(id) FROM hdfs_test FORMAT Values") == "(0)" assert node.query("SELECT count(*) FROM hdfs_test FORMAT Values") == "(16384)" # Data should remain in hdfs hdfs_objects = fs.listdir("/clickhouse") - assert len(hdfs_objects) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 4 + + for obj in hdfs_objects: + print("Object in HDFS after drop", obj) + + wait_for_delete_hdfs_objects( + cluster, + FILES_OVERHEAD + + FILES_OVERHEAD_PER_PART_WIDE * 4 + - FILES_OVERHEAD_METADATA_VERSION * 2, + ) diff --git a/tests/integration/test_merge_tree_load_parts/test.py b/tests/integration/test_merge_tree_load_parts/test.py index dfbe00c8e28..049dd516647 100644 --- a/tests/integration/test_merge_tree_load_parts/test.py +++ b/tests/integration/test_merge_tree_load_parts/test.py @@ -6,12 +6,14 @@ from helpers.corrupt_part_data_on_disk import corrupt_part_data_on_disk cluster = helpers.cluster.ClickHouseCluster(__file__) + node1 = cluster.add_instance( "node1", main_configs=["configs/fast_background_pool.xml"], with_zookeeper=True, stay_alive=True, ) + node2 = cluster.add_instance( "node2", main_configs=["configs/fast_background_pool.xml"], @@ -19,6 +21,12 @@ node2 = cluster.add_instance( stay_alive=True, ) +node3 = cluster.add_instance( + "node3", + with_zookeeper=True, + stay_alive=True, +) + @pytest.fixture(scope="module") def started_cluster(): @@ -194,3 +202,54 @@ def test_merge_tree_load_parts_corrupted(started_cluster): ) == "111\t1\n222\t1\n333\t1\n" ) + + +def test_merge_tree_load_parts_filesystem_error(started_cluster): + if node3.is_built_with_sanitizer() or node3.is_debug_build(): + pytest.skip( + "Skip with debug build and sanitizers. \ + This test intentionally triggers LOGICAL_ERROR which leads to crash with those builds" + ) + + node3.query( + """ + CREATE TABLE mt_load_parts (id UInt32) + ENGINE = MergeTree ORDER BY id + SETTINGS index_granularity_bytes = 0""" + ) + + node3.query("SYSTEM STOP MERGES mt_load_parts") + + for i in range(2): + node3.query(f"INSERT INTO mt_load_parts VALUES ({i})") + + # We want to somehow check that exception thrown on part creation is handled during part loading. + # It can be a filesystem exception triggered at initialization of part storage but it hard + # to trigger it because it should be an exception on stat/listDirectory. + # The most easy way to trigger such exception is to use chmod but clickhouse server + # is run with root user in integration test and this won't work. So let's do some + # some stupid things: create a table without adaptive granularity and change mark + # extensions of data files in part to make clickhouse think that it's a compact part which + # cannot be created in such table. This will trigger a LOGICAL_ERROR on part creation. + + def corrupt_part(table, part_name): + part_path = node3.query( + "SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format( + table, part_name + ) + ).strip() + + node3.exec_in_container( + ["bash", "-c", f"mv {part_path}id.mrk {part_path}id.mrk3"], privileged=True + ) + + corrupt_part("mt_load_parts", "all_1_1_0") + node3.restart_clickhouse(kill=True) + + assert node3.query("SELECT * FROM mt_load_parts") == "1\n" + assert ( + node3.query( + "SELECT name FROM system.detached_parts WHERE table = 'mt_load_parts'" + ) + == "broken-on-start_all_1_1_0\n" + ) diff --git a/tests/integration/test_merge_tree_s3/test.py b/tests/integration/test_merge_tree_s3/test.py index c2e00dc0cb8..e4bfffd70cc 100644 --- a/tests/integration/test_merge_tree_s3/test.py +++ b/tests/integration/test_merge_tree_s3/test.py @@ -101,44 +101,45 @@ def run_s3_mocks(cluster): ) -def list_objects(cluster, path="data/"): +def list_objects(cluster, path="data/", hint="list_objects"): minio = cluster.minio_client objects = list(minio.list_objects(cluster.minio_bucket, path, recursive=True)) - logging.info(f"list_objects ({len(objects)}): {[x.object_name for x in objects]}") + logging.info(f"{hint} ({len(objects)}): {[x.object_name for x in objects]}") return objects def wait_for_delete_s3_objects(cluster, expected, timeout=30): - minio = cluster.minio_client while timeout > 0: - if ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - == expected - ): + if len(list_objects(cluster, "data/")) == expected: return timeout -= 1 time.sleep(1) - assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - == expected - ) + assert len(list_objects(cluster, "data/")) == expected -@pytest.fixture(autouse=True) -@pytest.mark.parametrize("node_name", ["node"]) -def drop_table(cluster, node_name): - yield - node = cluster.instances[node_name] +def remove_all_s3_objects(cluster): minio = cluster.minio_client + for obj in list_objects(cluster, "data/"): + minio.remove_object(cluster.minio_bucket, obj.object_name) - node.query("DROP TABLE IF EXISTS s3_test NO DELAY") +@pytest.fixture(autouse=True, scope="function") +def clear_minio(cluster): try: - wait_for_delete_s3_objects(cluster, 0) - finally: + # CH do some writes to the S3 at start. For example, file data/clickhouse_access_check_{server_uuid}. + # Set the timeout there as 10 sec in order to resolve the race with that file exists. + wait_for_delete_s3_objects(cluster, 0, timeout=10) + except: # Remove extra objects to prevent tests cascade failing - for obj in list_objects(cluster, "data/"): - minio.remove_object(cluster.minio_bucket, obj.object_name) + remove_all_s3_objects(cluster) + + yield + + +def check_no_objects_after_drop(cluster, table_name="s3_test", node_name="node"): + node = cluster.instances[node_name] + node.query(f"DROP TABLE IF EXISTS {table_name} SYNC") + wait_for_delete_s3_objects(cluster, 0, timeout=0) @pytest.mark.parametrize( @@ -158,10 +159,7 @@ def test_simple_insert_select( values1 = generate_values("2020-01-03", 4096) node.query("INSERT INTO s3_test VALUES {}".format(values1)) assert node.query("SELECT * FROM s3_test order by dt, id FORMAT Values") == values1 - assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - == FILES_OVERHEAD + files_per_part - ) + assert len(list_objects(cluster, "data/")) == FILES_OVERHEAD + files_per_part values2 = generate_values("2020-01-04", 4096) node.query("INSERT INTO s3_test VALUES {}".format(values2)) @@ -169,15 +167,14 @@ def test_simple_insert_select( node.query("SELECT * FROM s3_test ORDER BY dt, id FORMAT Values") == values1 + "," + values2 ) - assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - == FILES_OVERHEAD + files_per_part * 2 - ) + assert len(list_objects(cluster, "data/")) == FILES_OVERHEAD + files_per_part * 2 assert ( node.query("SELECT count(*) FROM s3_test where id = 1 FORMAT Values") == "(2)" ) + check_no_objects_after_drop(cluster) + @pytest.mark.parametrize("merge_vertical,node_name", [(True, "node"), (False, "node")]) def test_insert_same_partition_and_merge(cluster, merge_vertical, node_name): @@ -188,7 +185,6 @@ def test_insert_same_partition_and_merge(cluster, merge_vertical, node_name): node = cluster.instances[node_name] create_table(node, "s3_test", **settings) - minio = cluster.minio_client node.query("SYSTEM STOP MERGES s3_test") node.query( @@ -214,7 +210,7 @@ def test_insert_same_partition_and_merge(cluster, merge_vertical, node_name): node.query("SELECT count(distinct(id)) FROM s3_test FORMAT Values") == "(8192)" ) assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list_objects(cluster, "data/")) == FILES_OVERHEAD_PER_PART_WIDE * 6 + FILES_OVERHEAD ) @@ -242,6 +238,8 @@ def test_insert_same_partition_and_merge(cluster, merge_vertical, node_name): cluster, FILES_OVERHEAD_PER_PART_WIDE + FILES_OVERHEAD, timeout=45 ) + check_no_objects_after_drop(cluster) + @pytest.mark.parametrize("node_name", ["node"]) def test_alter_table_columns(cluster, node_name): @@ -287,12 +285,13 @@ def test_alter_table_columns(cluster, node_name): cluster, FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE + 2 ) + check_no_objects_after_drop(cluster) + @pytest.mark.parametrize("node_name", ["node"]) def test_attach_detach_partition(cluster, node_name): node = cluster.instances[node_name] create_table(node, "s3_test") - minio = cluster.minio_client node.query( "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-03", 4096)) @@ -312,14 +311,18 @@ def test_attach_detach_partition(cluster, node_name): assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(4096)" assert ( len(list_objects(cluster, "data/")) - == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 + == FILES_OVERHEAD + + FILES_OVERHEAD_PER_PART_WIDE * 2 + - FILES_OVERHEAD_METADATA_VERSION ) node.query("ALTER TABLE s3_test ATTACH PARTITION '2020-01-03'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" assert ( len(list_objects(cluster, "data/")) - == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 + == FILES_OVERHEAD + + FILES_OVERHEAD_PER_PART_WIDE * 2 + - FILES_OVERHEAD_METADATA_VERSION ) node.query("ALTER TABLE s3_test DROP PARTITION '2020-01-03'") @@ -337,7 +340,9 @@ def test_attach_detach_partition(cluster, node_name): assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(0)" assert ( len(list_objects(cluster, "data/")) - == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 1 + == FILES_OVERHEAD + + FILES_OVERHEAD_PER_PART_WIDE * 1 + - FILES_OVERHEAD_METADATA_VERSION ) node.query( "ALTER TABLE s3_test DROP DETACHED PARTITION '2020-01-04'", @@ -349,12 +354,13 @@ def test_attach_detach_partition(cluster, node_name): == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 0 ) + check_no_objects_after_drop(cluster) + @pytest.mark.parametrize("node_name", ["node"]) def test_move_partition_to_another_disk(cluster, node_name): node = cluster.instances[node_name] create_table(node, "s3_test") - minio = cluster.minio_client node.query( "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-03", 4096)) @@ -364,30 +370,31 @@ def test_move_partition_to_another_disk(cluster, node_name): ) assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list_objects(cluster, "data/")) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 ) node.query("ALTER TABLE s3_test MOVE PARTITION '2020-01-04' TO DISK 'hdd'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list_objects(cluster, "data/")) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE ) node.query("ALTER TABLE s3_test MOVE PARTITION '2020-01-04' TO DISK 's3'") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list_objects(cluster, "data/")) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 ) + check_no_objects_after_drop(cluster) + @pytest.mark.parametrize("node_name", ["node"]) def test_table_manipulations(cluster, node_name): node = cluster.instances[node_name] create_table(node, "s3_test") - minio = cluster.minio_client node.query( "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-03", 4096)) @@ -399,9 +406,10 @@ def test_table_manipulations(cluster, node_name): node.query("RENAME TABLE s3_test TO s3_renamed") assert node.query("SELECT count(*) FROM s3_renamed FORMAT Values") == "(8192)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list_objects(cluster, "data/")) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 ) + node.query("RENAME TABLE s3_renamed TO s3_test") assert node.query("CHECK TABLE s3_test FORMAT Values") == "(1)" @@ -410,7 +418,7 @@ def test_table_manipulations(cluster, node_name): node.query("ATTACH TABLE s3_test") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list_objects(cluster, "data/")) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 ) @@ -418,17 +426,15 @@ def test_table_manipulations(cluster, node_name): wait_for_delete_empty_parts(node, "s3_test") wait_for_delete_inactive_parts(node, "s3_test") assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(0)" - assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - == FILES_OVERHEAD - ) + assert len(list_objects(cluster, "data/")) == FILES_OVERHEAD + + check_no_objects_after_drop(cluster) @pytest.mark.parametrize("node_name", ["node"]) def test_move_replace_partition_to_another_table(cluster, node_name): node = cluster.instances[node_name] create_table(node, "s3_test") - minio = cluster.minio_client node.query( "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-03", 4096)) @@ -444,11 +450,11 @@ def test_move_replace_partition_to_another_table(cluster, node_name): ) assert node.query("SELECT sum(id) FROM s3_test FORMAT Values") == "(0)" assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(16384)" + assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) + len(list_objects(cluster, "data/", "Objects at start")) == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 4 ) - create_table(node, "s3_clone") node.query("ALTER TABLE s3_test MOVE PARTITION '2020-01-03' TO TABLE s3_clone") @@ -457,10 +463,14 @@ def test_move_replace_partition_to_another_table(cluster, node_name): assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(8192)" assert node.query("SELECT sum(id) FROM s3_clone FORMAT Values") == "(0)" assert node.query("SELECT count(*) FROM s3_clone FORMAT Values") == "(8192)" + + list_objects(cluster, "data/", "Object after move partition") # Number of objects in S3 should be unchanged. - assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - == FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 4 + wait_for_delete_s3_objects( + cluster, + FILES_OVERHEAD * 2 + + FILES_OVERHEAD_PER_PART_WIDE * 4 + - FILES_OVERHEAD_METADATA_VERSION * 2, ) # Add new partitions to source table, but with different values and replace them from copied table. @@ -472,9 +482,13 @@ def test_move_replace_partition_to_another_table(cluster, node_name): ) assert node.query("SELECT sum(id) FROM s3_test FORMAT Values") == "(0)" assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(16384)" - assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - == FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 6 + + list_objects(cluster, "data/", "Object after insert") + wait_for_delete_s3_objects( + cluster, + FILES_OVERHEAD * 2 + + FILES_OVERHEAD_PER_PART_WIDE * 6 + - FILES_OVERHEAD_METADATA_VERSION * 2, ) node.query("ALTER TABLE s3_test REPLACE PARTITION '2020-01-03' FROM s3_clone") @@ -486,39 +500,48 @@ def test_move_replace_partition_to_another_table(cluster, node_name): # Wait for outdated partitions deletion. wait_for_delete_s3_objects( - cluster, FILES_OVERHEAD * 2 + FILES_OVERHEAD_PER_PART_WIDE * 4 + cluster, + FILES_OVERHEAD * 2 + + FILES_OVERHEAD_PER_PART_WIDE * 4 + - FILES_OVERHEAD_METADATA_VERSION * 2, ) - node.query("DROP TABLE s3_clone NO DELAY") + node.query("DROP TABLE s3_clone SYNC") assert node.query("SELECT sum(id) FROM s3_test FORMAT Values") == "(0)" assert node.query("SELECT count(*) FROM s3_test FORMAT Values") == "(16384)" - # Data should remain in S3 - assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 4 + + list_objects(cluster, "data/", "Object after drop") + wait_for_delete_s3_objects( + cluster, + FILES_OVERHEAD + + FILES_OVERHEAD_PER_PART_WIDE * 4 + - FILES_OVERHEAD_METADATA_VERSION * 2, ) node.query("ALTER TABLE s3_test FREEZE") # Number S3 objects should be unchanged. - assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 4 + list_objects(cluster, "data/", "Object after freeze") + wait_for_delete_s3_objects( + cluster, + FILES_OVERHEAD + + FILES_OVERHEAD_PER_PART_WIDE * 4 + - FILES_OVERHEAD_METADATA_VERSION * 2, ) - node.query("DROP TABLE s3_test NO DELAY") + node.query("DROP TABLE s3_test SYNC") # Backup data should remain in S3. - wait_for_delete_s3_objects(cluster, FILES_OVERHEAD_PER_PART_WIDE * 4) + wait_for_delete_s3_objects( + cluster, FILES_OVERHEAD_PER_PART_WIDE * 4 - FILES_OVERHEAD_METADATA_VERSION * 4 + ) - for obj in list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True)): - minio.remove_object(cluster.minio_bucket, obj.object_name) + remove_all_s3_objects(cluster) @pytest.mark.parametrize("node_name", ["node"]) def test_freeze_unfreeze(cluster, node_name): node = cluster.instances[node_name] create_table(node, "s3_test") - minio = cluster.minio_client node.query( "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-03", 4096)) @@ -533,8 +556,9 @@ def test_freeze_unfreeze(cluster, node_name): wait_for_delete_empty_parts(node, "s3_test") wait_for_delete_inactive_parts(node, "s3_test") assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 + len(list_objects(cluster, "data/")) + == FILES_OVERHEAD + + (FILES_OVERHEAD_PER_PART_WIDE - FILES_OVERHEAD_METADATA_VERSION) * 2 ) # Unfreeze single partition from backup1. @@ -544,13 +568,10 @@ def test_freeze_unfreeze(cluster, node_name): # Unfreeze all partitions from backup2. node.query("ALTER TABLE s3_test UNFREEZE WITH NAME 'backup2'") + # Data should be removed from S3. wait_for_delete_s3_objects(cluster, FILES_OVERHEAD) - # Data should be removed from S3. - assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - == FILES_OVERHEAD - ) + check_no_objects_after_drop(cluster) @pytest.mark.parametrize("node_name", ["node"]) @@ -558,7 +579,6 @@ def test_freeze_system_unfreeze(cluster, node_name): node = cluster.instances[node_name] create_table(node, "s3_test") create_table(node, "s3_test_removed") - minio = cluster.minio_client node.query( "INSERT INTO s3_test VALUES {}".format(generate_values("2020-01-04", 4096)) @@ -572,22 +592,20 @@ def test_freeze_system_unfreeze(cluster, node_name): node.query("TRUNCATE TABLE s3_test") wait_for_delete_empty_parts(node, "s3_test") wait_for_delete_inactive_parts(node, "s3_test") - node.query("DROP TABLE s3_test_removed NO DELAY") + node.query("DROP TABLE s3_test_removed SYNC") assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - == FILES_OVERHEAD + FILES_OVERHEAD_PER_PART_WIDE * 2 + len(list_objects(cluster, "data/")) + == FILES_OVERHEAD + + (FILES_OVERHEAD_PER_PART_WIDE - FILES_OVERHEAD_METADATA_VERSION) * 2 ) # Unfreeze all data from backup3. node.query("SYSTEM UNFREEZE WITH NAME 'backup3'") + # Data should be removed from S3. wait_for_delete_s3_objects(cluster, FILES_OVERHEAD) - # Data should be removed from S3. - assert ( - len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))) - == FILES_OVERHEAD - ) + check_no_objects_after_drop(cluster) @pytest.mark.parametrize("node_name", ["node"]) @@ -633,6 +651,8 @@ def test_s3_disk_apply_new_settings(cluster, node_name): # There should be 3 times more S3 requests because multi-part upload mode uses 3 requests to upload object. assert get_s3_requests() - s3_requests_before == s3_requests_to_write_partition * 3 + check_no_objects_after_drop(cluster) + @pytest.mark.parametrize("node_name", ["node"]) def test_s3_no_delete_objects(cluster, node_name): @@ -641,6 +661,7 @@ def test_s3_no_delete_objects(cluster, node_name): node, "s3_test_no_delete_objects", storage_policy="no_delete_objects_s3" ) node.query("DROP TABLE s3_test_no_delete_objects SYNC") + remove_all_s3_objects(cluster) @pytest.mark.parametrize("node_name", ["node"]) @@ -655,32 +676,33 @@ def test_s3_disk_reads_on_unstable_connection(cluster, node_name): assert node.query("SELECT sum(id) FROM s3_test").splitlines() == [ "40499995500000" ] + check_no_objects_after_drop(cluster) @pytest.mark.parametrize("node_name", ["node"]) def test_lazy_seek_optimization_for_async_read(cluster, node_name): node = cluster.instances[node_name] - node.query("DROP TABLE IF EXISTS s3_test NO DELAY") + node.query("DROP TABLE IF EXISTS s3_test SYNC") node.query( "CREATE TABLE s3_test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3';" ) + node.query("SYSTEM STOP MERGES s3_test") node.query( "INSERT INTO s3_test SELECT * FROM generateRandom('key UInt32, value String') LIMIT 10000000" ) node.query("SELECT * FROM s3_test WHERE value LIKE '%abc%' ORDER BY value LIMIT 10") - node.query("DROP TABLE IF EXISTS s3_test NO DELAY") - minio = cluster.minio_client - for obj in list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True)): - minio.remove_object(cluster.minio_bucket, obj.object_name) + + check_no_objects_after_drop(cluster) @pytest.mark.parametrize("node_name", ["node_with_limited_disk"]) def test_cache_with_full_disk_space(cluster, node_name): node = cluster.instances[node_name] - node.query("DROP TABLE IF EXISTS s3_test NO DELAY") + node.query("DROP TABLE IF EXISTS s3_test SYNC") node.query( "CREATE TABLE s3_test (key UInt32, value String) Engine=MergeTree() ORDER BY value SETTINGS storage_policy='s3_with_cache_and_jbod';" ) + node.query("SYSTEM STOP MERGES s3_test") node.query( "INSERT INTO s3_test SELECT number, toString(number) FROM numbers(100000000)" ) @@ -699,7 +721,7 @@ def test_cache_with_full_disk_space(cluster, node_name): assert node.contains_in_log( "Insert into cache is skipped due to insufficient disk space" ) - node.query("DROP TABLE IF EXISTS s3_test NO DELAY") + check_no_objects_after_drop(cluster, node_name=node_name) @pytest.mark.parametrize("node_name", ["node"]) @@ -724,13 +746,14 @@ def test_store_cleanup_disk_s3(cluster, node_name): "CREATE TABLE s3_test UUID '00000000-1000-4000-8000-000000000001' (n UInt64) Engine=MergeTree() ORDER BY n SETTINGS storage_policy='s3';" ) node.query("INSERT INTO s3_test SELECT 1") + check_no_objects_after_drop(cluster) @pytest.mark.parametrize("node_name", ["node"]) def test_cache_setting_compatibility(cluster, node_name): node = cluster.instances[node_name] - node.query("DROP TABLE IF EXISTS s3_test NO DELAY") + node.query("DROP TABLE IF EXISTS s3_test SYNC") node.query( "CREATE TABLE s3_test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache_r', compress_marks=false, compress_primary_key=false;" @@ -800,3 +823,5 @@ def test_cache_setting_compatibility(cluster, node_name): node.query("SELECT * FROM s3_test FORMAT Null") assert not node.contains_in_log("No such file or directory: Cache info:") + + check_no_objects_after_drop(cluster) diff --git a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml index 976933b2d21..74af657c783 100644 --- a/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml +++ b/tests/integration/test_merge_tree_s3_failover/configs/config.d/storage_conf.xml @@ -11,6 +11,7 @@ true 0 + 20000 s3 @@ -20,6 +21,7 @@ minio123 true + 20000 s3 @@ -32,6 +34,7 @@ 1 1 + 20000 diff --git a/tests/integration/test_merge_tree_s3_failover/test.py b/tests/integration/test_merge_tree_s3_failover/test.py index 3cc2b17dce2..05aeeff2ec1 100644 --- a/tests/integration/test_merge_tree_s3_failover/test.py +++ b/tests/integration/test_merge_tree_s3_failover/test.py @@ -85,7 +85,7 @@ def cluster(): def drop_table(cluster): yield node = cluster.instances["node"] - node.query("DROP TABLE IF EXISTS s3_failover_test NO DELAY") + node.query("DROP TABLE IF EXISTS s3_failover_test SYNC") # S3 request will be failed for an appropriate part file write. diff --git a/tests/integration/test_merge_tree_s3_with_cache/test.py b/tests/integration/test_merge_tree_s3_with_cache/test.py index 89b5a400b1b..067ed4f9679 100644 --- a/tests/integration/test_merge_tree_s3_with_cache/test.py +++ b/tests/integration/test_merge_tree_s3_with_cache/test.py @@ -77,7 +77,7 @@ def test_write_is_cached(cluster, min_rows_for_wide_part, read_requests): # stat = get_query_stat(node, select_query) # assert stat["S3ReadRequestsCount"] == read_requests # Only .bin files should be accessed from S3. - node.query("DROP TABLE IF EXISTS s3_test NO DELAY") + node.query("DROP TABLE IF EXISTS s3_test SYNC") @pytest.mark.parametrize( @@ -126,4 +126,4 @@ def test_read_after_cache_is_wiped( # stat = get_query_stat(node, select_query) # assert stat["S3ReadRequestsCount"] == bin_files - node.query("DROP TABLE IF EXISTS s3_test NO DELAY") + node.query("DROP TABLE IF EXISTS s3_test SYNC") diff --git a/tests/integration/test_partition/test.py b/tests/integration/test_partition/test.py index 5a972b58f99..93f03f4420e 100644 --- a/tests/integration/test_partition/test.py +++ b/tests/integration/test_partition/test.py @@ -70,7 +70,7 @@ def partition_complex_assert_columns_txt(): ) -def partition_complex_assert_checksums(): +def partition_complex_assert_checksums(after_detach=False): # Do not check increment.txt - it can be changed by other tests with FREEZE cmd = [ "bash", @@ -80,36 +80,67 @@ def partition_complex_assert_checksums(): " | sed 's shadow/[0-9]*/data/[a-z0-9_-]*/ shadow/1/data/test/ g' | sort | uniq", ] - checksums = ( - "082814b5aa5109160d5c0c5aff10d4df\tshadow/1/data/test/partition_complex/19700102_2_2_0/k.bin\n" - "082814b5aa5109160d5c0c5aff10d4df\tshadow/1/data/test/partition_complex/19700201_1_1_0/v1.bin\n" - "13cae8e658e0ca4f75c56b1fc424e150\tshadow/1/data/test/partition_complex/19700102_2_2_0/minmax_p.idx\n" - "25daad3d9e60b45043a70c4ab7d3b1c6\tshadow/1/data/test/partition_complex/19700102_2_2_0/partition.dat\n" - "3726312af62aec86b64a7708d5751787\tshadow/1/data/test/partition_complex/19700201_1_1_0/partition.dat\n" - "37855b06a39b79a67ea4e86e4a3299aa\tshadow/1/data/test/partition_complex/19700102_2_2_0/checksums.txt\n" - "38e62ff37e1e5064e9a3f605dfe09d13\tshadow/1/data/test/partition_complex/19700102_2_2_0/v1.bin\n" - "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700102_2_2_0/k.mrk\n" - "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700102_2_2_0/p.mrk\n" - "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700102_2_2_0/v1.mrk\n" - "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700201_1_1_0/k.mrk\n" - "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700201_1_1_0/p.mrk\n" - "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700201_1_1_0/v1.mrk\n" - "55a54008ad1ba589aa210d2629c1df41\tshadow/1/data/test/partition_complex/19700201_1_1_0/primary.idx\n" - "5f087cb3e7071bf9407e095821e2af8f\tshadow/1/data/test/partition_complex/19700201_1_1_0/checksums.txt\n" - "77d5af402ada101574f4da114f242e02\tshadow/1/data/test/partition_complex/19700102_2_2_0/columns.txt\n" - "77d5af402ada101574f4da114f242e02\tshadow/1/data/test/partition_complex/19700201_1_1_0/columns.txt\n" - "88cdc31ded355e7572d68d8cde525d3a\tshadow/1/data/test/partition_complex/19700201_1_1_0/p.bin\n" - "9e688c58a5487b8eaf69c9e1005ad0bf\tshadow/1/data/test/partition_complex/19700102_2_2_0/primary.idx\n" - "c0904274faa8f3f06f35666cc9c5bd2f\tshadow/1/data/test/partition_complex/19700102_2_2_0/default_compression_codec.txt\n" - "c0904274faa8f3f06f35666cc9c5bd2f\tshadow/1/data/test/partition_complex/19700201_1_1_0/default_compression_codec.txt\n" - "c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition_complex/19700102_2_2_0/count.txt\n" - "c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition_complex/19700201_1_1_0/count.txt\n" - "cfcb770c3ecd0990dcceb1bde129e6c6\tshadow/1/data/test/partition_complex/19700102_2_2_0/p.bin\n" - "cfcd208495d565ef66e7dff9f98764da\tshadow/1/data/test/partition_complex/19700102_2_2_0/metadata_version.txt\n" - "cfcd208495d565ef66e7dff9f98764da\tshadow/1/data/test/partition_complex/19700201_1_1_0/metadata_version.txt\n" - "e2af3bef1fd129aea73a890ede1e7a30\tshadow/1/data/test/partition_complex/19700201_1_1_0/k.bin\n" - "f2312862cc01adf34a93151377be2ddf\tshadow/1/data/test/partition_complex/19700201_1_1_0/minmax_p.idx\n" - ) + # no metadata version + if after_detach: + checksums = ( + "082814b5aa5109160d5c0c5aff10d4df\tshadow/1/data/test/partition_complex/19700102_2_2_0/k.bin\n" + "082814b5aa5109160d5c0c5aff10d4df\tshadow/1/data/test/partition_complex/19700201_1_1_0/v1.bin\n" + "13cae8e658e0ca4f75c56b1fc424e150\tshadow/1/data/test/partition_complex/19700102_2_2_0/minmax_p.idx\n" + "25daad3d9e60b45043a70c4ab7d3b1c6\tshadow/1/data/test/partition_complex/19700102_2_2_0/partition.dat\n" + "3726312af62aec86b64a7708d5751787\tshadow/1/data/test/partition_complex/19700201_1_1_0/partition.dat\n" + "37855b06a39b79a67ea4e86e4a3299aa\tshadow/1/data/test/partition_complex/19700102_2_2_0/checksums.txt\n" + "38e62ff37e1e5064e9a3f605dfe09d13\tshadow/1/data/test/partition_complex/19700102_2_2_0/v1.bin\n" + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700102_2_2_0/k.mrk\n" + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700102_2_2_0/p.mrk\n" + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700102_2_2_0/v1.mrk\n" + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700201_1_1_0/k.mrk\n" + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700201_1_1_0/p.mrk\n" + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700201_1_1_0/v1.mrk\n" + "55a54008ad1ba589aa210d2629c1df41\tshadow/1/data/test/partition_complex/19700201_1_1_0/primary.idx\n" + "5f087cb3e7071bf9407e095821e2af8f\tshadow/1/data/test/partition_complex/19700201_1_1_0/checksums.txt\n" + "77d5af402ada101574f4da114f242e02\tshadow/1/data/test/partition_complex/19700102_2_2_0/columns.txt\n" + "77d5af402ada101574f4da114f242e02\tshadow/1/data/test/partition_complex/19700201_1_1_0/columns.txt\n" + "88cdc31ded355e7572d68d8cde525d3a\tshadow/1/data/test/partition_complex/19700201_1_1_0/p.bin\n" + "9e688c58a5487b8eaf69c9e1005ad0bf\tshadow/1/data/test/partition_complex/19700102_2_2_0/primary.idx\n" + "c0904274faa8f3f06f35666cc9c5bd2f\tshadow/1/data/test/partition_complex/19700102_2_2_0/default_compression_codec.txt\n" + "c0904274faa8f3f06f35666cc9c5bd2f\tshadow/1/data/test/partition_complex/19700201_1_1_0/default_compression_codec.txt\n" + "c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition_complex/19700102_2_2_0/count.txt\n" + "c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition_complex/19700201_1_1_0/count.txt\n" + "cfcb770c3ecd0990dcceb1bde129e6c6\tshadow/1/data/test/partition_complex/19700102_2_2_0/p.bin\n" + "e2af3bef1fd129aea73a890ede1e7a30\tshadow/1/data/test/partition_complex/19700201_1_1_0/k.bin\n" + "f2312862cc01adf34a93151377be2ddf\tshadow/1/data/test/partition_complex/19700201_1_1_0/minmax_p.idx\n" + ) + else: + checksums = ( + "082814b5aa5109160d5c0c5aff10d4df\tshadow/1/data/test/partition_complex/19700102_2_2_0/k.bin\n" + "082814b5aa5109160d5c0c5aff10d4df\tshadow/1/data/test/partition_complex/19700201_1_1_0/v1.bin\n" + "13cae8e658e0ca4f75c56b1fc424e150\tshadow/1/data/test/partition_complex/19700102_2_2_0/minmax_p.idx\n" + "25daad3d9e60b45043a70c4ab7d3b1c6\tshadow/1/data/test/partition_complex/19700102_2_2_0/partition.dat\n" + "3726312af62aec86b64a7708d5751787\tshadow/1/data/test/partition_complex/19700201_1_1_0/partition.dat\n" + "37855b06a39b79a67ea4e86e4a3299aa\tshadow/1/data/test/partition_complex/19700102_2_2_0/checksums.txt\n" + "38e62ff37e1e5064e9a3f605dfe09d13\tshadow/1/data/test/partition_complex/19700102_2_2_0/v1.bin\n" + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700102_2_2_0/k.mrk\n" + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700102_2_2_0/p.mrk\n" + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700102_2_2_0/v1.mrk\n" + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700201_1_1_0/k.mrk\n" + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700201_1_1_0/p.mrk\n" + "4ae71336e44bf9bf79d2752e234818a5\tshadow/1/data/test/partition_complex/19700201_1_1_0/v1.mrk\n" + "55a54008ad1ba589aa210d2629c1df41\tshadow/1/data/test/partition_complex/19700201_1_1_0/primary.idx\n" + "5f087cb3e7071bf9407e095821e2af8f\tshadow/1/data/test/partition_complex/19700201_1_1_0/checksums.txt\n" + "77d5af402ada101574f4da114f242e02\tshadow/1/data/test/partition_complex/19700102_2_2_0/columns.txt\n" + "77d5af402ada101574f4da114f242e02\tshadow/1/data/test/partition_complex/19700201_1_1_0/columns.txt\n" + "88cdc31ded355e7572d68d8cde525d3a\tshadow/1/data/test/partition_complex/19700201_1_1_0/p.bin\n" + "9e688c58a5487b8eaf69c9e1005ad0bf\tshadow/1/data/test/partition_complex/19700102_2_2_0/primary.idx\n" + "c0904274faa8f3f06f35666cc9c5bd2f\tshadow/1/data/test/partition_complex/19700102_2_2_0/default_compression_codec.txt\n" + "c0904274faa8f3f06f35666cc9c5bd2f\tshadow/1/data/test/partition_complex/19700201_1_1_0/default_compression_codec.txt\n" + "c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition_complex/19700102_2_2_0/count.txt\n" + "c4ca4238a0b923820dcc509a6f75849b\tshadow/1/data/test/partition_complex/19700201_1_1_0/count.txt\n" + "cfcb770c3ecd0990dcceb1bde129e6c6\tshadow/1/data/test/partition_complex/19700102_2_2_0/p.bin\n" + "cfcd208495d565ef66e7dff9f98764da\tshadow/1/data/test/partition_complex/19700102_2_2_0/metadata_version.txt\n" + "cfcd208495d565ef66e7dff9f98764da\tshadow/1/data/test/partition_complex/19700201_1_1_0/metadata_version.txt\n" + "e2af3bef1fd129aea73a890ede1e7a30\tshadow/1/data/test/partition_complex/19700201_1_1_0/k.bin\n" + "f2312862cc01adf34a93151377be2ddf\tshadow/1/data/test/partition_complex/19700201_1_1_0/minmax_p.idx\n" + ) assert TSV(instance.exec_in_container(cmd).replace(" ", "\t")) == TSV(checksums) @@ -134,7 +165,7 @@ def test_partition_complex(partition_table_complex): q("ALTER TABLE test.partition_complex FREEZE") - partition_complex_assert_checksums() + partition_complex_assert_checksums(True) q("ALTER TABLE test.partition_complex DETACH PARTITION 197001") q("ALTER TABLE test.partition_complex ATTACH PARTITION 197001") @@ -144,7 +175,7 @@ def test_partition_complex(partition_table_complex): q("ALTER TABLE test.partition_complex MODIFY COLUMN v1 Int8") # Check the backup hasn't changed - partition_complex_assert_checksums() + partition_complex_assert_checksums(True) q("OPTIMIZE TABLE test.partition_complex") diff --git a/tests/integration/test_password_constraints/configs/default_password_type.xml b/tests/integration/test_password_constraints/configs/default_password_type.xml new file mode 100644 index 00000000000..4b23ea31df0 --- /dev/null +++ b/tests/integration/test_password_constraints/configs/default_password_type.xml @@ -0,0 +1,3 @@ + + double_sha1_password + diff --git a/tests/integration/test_password_constraints/test.py b/tests/integration/test_password_constraints/test.py index 9cdff51caa1..94e10ed5f9e 100644 --- a/tests/integration/test_password_constraints/test.py +++ b/tests/integration/test_password_constraints/test.py @@ -5,6 +5,9 @@ from helpers.cluster import ClickHouseCluster cluster = ClickHouseCluster(__file__) node = cluster.add_instance("node", main_configs=["configs/complexity_rules.xml"]) +node2 = cluster.add_instance( + "node2", main_configs=["configs/default_password_type.xml"] +) @pytest.fixture(scope="module") @@ -39,3 +42,10 @@ def test_complexity_rules(start_cluster): node.query("CREATE USER u_5 IDENTIFIED WITH plaintext_password BY 'aA!000000000'") node.query("DROP USER u_5") + + +def test_default_password_type(start_cluster): + node2.query("CREATE USER u1 IDENTIFIED BY 'pwd'") + + required_type = "double_sha1_password" + assert required_type in node2.query("SHOW CREATE USER u1") diff --git a/tests/integration/test_postgresql_database_engine/test.py b/tests/integration/test_postgresql_database_engine/test.py index de6c9ad2cf9..63e85afb1d4 100644 --- a/tests/integration/test_postgresql_database_engine/test.py +++ b/tests/integration/test_postgresql_database_engine/test.py @@ -327,6 +327,17 @@ def test_predefined_connection_configuration(started_cluster): node1.query(f"SELECT count() FROM postgres_database.test_table").rstrip() == "100" ) + node1.query( + """ + DROP DATABASE postgres_database; + CREATE DATABASE postgres_database ENGINE = PostgreSQL(postgres1, use_tables_cache=1); + """ + ) + assert ( + node1.query(f"SELECT count() FROM postgres_database.test_table").rstrip() + == "100" + ) + assert node1.contains_in_log("Cached table `test_table`") node1.query("DROP DATABASE postgres_database") cursor.execute(f"DROP TABLE test_table ") diff --git a/tests/integration/test_postgresql_replica_database_engine_2/test.py b/tests/integration/test_postgresql_replica_database_engine_2/test.py index 1ba278b357e..90d19e9532c 100644 --- a/tests/integration/test_postgresql_replica_database_engine_2/test.py +++ b/tests/integration/test_postgresql_replica_database_engine_2/test.py @@ -624,7 +624,7 @@ def test_table_override(started_cluster): time.sleep(5) query = f"select * from {materialized_database}.{table_name} order by key" expected = instance.query(f"select * from {table_name} order by key") - instance.query(f"drop table {table_name} no delay") + instance.query(f"drop table {table_name} sync") assert_eq_with_retry(instance, query, expected) diff --git a/tests/integration/test_rename_column/test.py b/tests/integration/test_rename_column/test.py index 33343da8f6d..6bab0a28259 100644 --- a/tests/integration/test_rename_column/test.py +++ b/tests/integration/test_rename_column/test.py @@ -40,7 +40,7 @@ def started_cluster(): def drop_table(nodes, table_name): for node in nodes: - node.query("DROP TABLE IF EXISTS {} NO DELAY".format(table_name)) + node.query("DROP TABLE IF EXISTS {} SYNC".format(table_name)) def create_table( diff --git a/tests/integration/test_replicated_merge_tree_encrypted_disk/test.py b/tests/integration/test_replicated_merge_tree_encrypted_disk/test.py index aea41fc0684..05d7bbb7282 100644 --- a/tests/integration/test_replicated_merge_tree_encrypted_disk/test.py +++ b/tests/integration/test_replicated_merge_tree_encrypted_disk/test.py @@ -42,7 +42,7 @@ def copy_keys(instance, keys_file_name): def create_table(): - node1.query("DROP TABLE IF EXISTS tbl ON CLUSTER 'cluster' NO DELAY") + node1.query("DROP TABLE IF EXISTS tbl ON CLUSTER 'cluster' SYNC") node1.query( """ CREATE TABLE tbl ON CLUSTER 'cluster' ( diff --git a/tests/integration/test_replicated_merge_tree_encryption_codec/test.py b/tests/integration/test_replicated_merge_tree_encryption_codec/test.py index ffe9c056f1e..a50f8341ee7 100644 --- a/tests/integration/test_replicated_merge_tree_encryption_codec/test.py +++ b/tests/integration/test_replicated_merge_tree_encryption_codec/test.py @@ -40,7 +40,7 @@ def copy_keys(instance, keys_file_name): def create_table(): - node1.query("DROP TABLE IF EXISTS tbl ON CLUSTER 'cluster' NO DELAY") + node1.query("DROP TABLE IF EXISTS tbl ON CLUSTER 'cluster' SYNC") node1.query( """ CREATE TABLE tbl ON CLUSTER 'cluster' ( diff --git a/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/test.py b/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/test.py index 1f81421f93c..bd1c890950a 100644 --- a/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/test.py +++ b/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/test.py @@ -111,8 +111,8 @@ def test_hdfs_zero_copy_replication_insert(cluster): SHARDS * FILES_OVERHEAD_PER_TABLE + FILES_OVERHEAD_PER_PART_COMPACT, ) finally: - node1.query("DROP TABLE IF EXISTS hdfs_test NO DELAY") - node2.query("DROP TABLE IF EXISTS hdfs_test NO DELAY") + node1.query("DROP TABLE IF EXISTS hdfs_test SYNC") + node2.query("DROP TABLE IF EXISTS hdfs_test SYNC") @pytest.mark.parametrize( @@ -173,7 +173,7 @@ def test_hdfs_zero_copy_replication_single_move(cluster, storage_policy, init_ob == "(10),(11)" ) finally: - node1.query("DROP TABLE IF EXISTS single_node_move_test NO DELAY") + node1.query("DROP TABLE IF EXISTS single_node_move_test SYNC") @pytest.mark.parametrize( @@ -244,8 +244,8 @@ def test_hdfs_zero_copy_replication_move(cluster, storage_policy, init_objects): cluster, "/clickhouse1", init_objects + FILES_OVERHEAD_PER_PART_COMPACT ) finally: - node1.query("DROP TABLE IF EXISTS move_test NO DELAY") - node2.query("DROP TABLE IF EXISTS move_test NO DELAY") + node1.query("DROP TABLE IF EXISTS move_test SYNC") + node2.query("DROP TABLE IF EXISTS move_test SYNC") @pytest.mark.parametrize(("storage_policy"), ["hybrid", "tiered", "tiered_copy"]) @@ -282,8 +282,8 @@ def test_hdfs_zero_copy_with_ttl_move(cluster, storage_policy): == "(10),(11)" ) finally: - node1.query("DROP TABLE IF EXISTS ttl_move_test NO DELAY") - node2.query("DROP TABLE IF EXISTS ttl_move_test NO DELAY") + node1.query("DROP TABLE IF EXISTS ttl_move_test SYNC") + node2.query("DROP TABLE IF EXISTS ttl_move_test SYNC") def test_hdfs_zero_copy_with_ttl_delete(cluster): @@ -318,5 +318,5 @@ def test_hdfs_zero_copy_with_ttl_delete(cluster): == "(11)" ) finally: - node1.query("DROP TABLE IF EXISTS ttl_delete_test NO DELAY") - node2.query("DROP TABLE IF EXISTS ttl_delete_test NO DELAY") + node1.query("DROP TABLE IF EXISTS ttl_delete_test SYNC") + node2.query("DROP TABLE IF EXISTS ttl_delete_test SYNC") diff --git a/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/test.py b/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/test.py index cf76d47157a..5a514be58dd 100644 --- a/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/test.py +++ b/tests/integration/test_replicated_merge_tree_with_auxiliary_zookeepers/test.py @@ -37,7 +37,7 @@ def started_cluster(): def drop_table(nodes, table_name): for node in nodes: - node.query("DROP TABLE IF EXISTS {} NO DELAY".format(table_name)) + node.query("DROP TABLE IF EXISTS {} SYNC".format(table_name)) # Create table with default zookeeper. diff --git a/tests/integration/test_s3_cluster/test.py b/tests/integration/test_s3_cluster/test.py index 241b90cac3f..237a81da0f5 100644 --- a/tests/integration/test_s3_cluster/test.py +++ b/tests/integration/test_s3_cluster/test.py @@ -247,9 +247,10 @@ def test_skip_unavailable_shards(started_cluster): assert result == "10\n" -def test_unskip_unavailable_shards(started_cluster): +def test_unset_skip_unavailable_shards(started_cluster): + # Although skip_unavailable_shards is not set, cluster table functions should always skip unavailable shards. node = started_cluster.instances["s0_0_0"] - error = node.query_and_get_error( + result = node.query( """ SELECT count(*) from s3Cluster( 'cluster_non_existent_port', @@ -258,7 +259,7 @@ def test_unskip_unavailable_shards(started_cluster): """ ) - assert "NETWORK_ERROR" in error + assert result == "10\n" def test_distributed_insert_select_with_replicated(started_cluster): diff --git a/tests/integration/test_s3_with_https/test.py b/tests/integration/test_s3_with_https/test.py index 46e281251a0..6db5b7da930 100644 --- a/tests/integration/test_s3_with_https/test.py +++ b/tests/integration/test_s3_with_https/test.py @@ -56,7 +56,7 @@ def test_s3_with_https(cluster, policy): == "(0,'data'),(1,'data')" ) - node.query("DROP TABLE IF EXISTS s3_test NO DELAY") + node.query("DROP TABLE IF EXISTS s3_test SYNC") if policy.find("proxy") != -1: check_proxy_logs(cluster, "proxy1") diff --git a/tests/integration/test_s3_with_proxy/test.py b/tests/integration/test_s3_with_proxy/test.py index 1af040c3c30..e5624d4e056 100644 --- a/tests/integration/test_s3_with_proxy/test.py +++ b/tests/integration/test_s3_with_proxy/test.py @@ -72,7 +72,7 @@ def test_s3_with_proxy_list(cluster, policy): == "(0,'data'),(1,'data')" ) - node.query("DROP TABLE IF EXISTS s3_test NO DELAY") + node.query("DROP TABLE IF EXISTS s3_test SYNC") for proxy in ["proxy1", "proxy2"]: check_proxy_logs(cluster, proxy, ["PUT", "GET"]) diff --git a/tests/integration/test_s3_zero_copy_replication/test.py b/tests/integration/test_s3_zero_copy_replication/test.py index 1c559312105..100f062de2f 100644 --- a/tests/integration/test_s3_zero_copy_replication/test.py +++ b/tests/integration/test_s3_zero_copy_replication/test.py @@ -149,8 +149,8 @@ def test_s3_zero_copy_replication(started_cluster, policy): # Based on version 21.x - after cleanup - only one merged part wait_for_large_objects_count(cluster, 1, timeout=60) - node1.query("DROP TABLE IF EXISTS s3_test NO DELAY") - node2.query("DROP TABLE IF EXISTS s3_test NO DELAY") + node1.query("DROP TABLE IF EXISTS s3_test SYNC") + node2.query("DROP TABLE IF EXISTS s3_test SYNC") @pytest.mark.skip(reason="Test is flaky (and never was stable)") @@ -239,8 +239,8 @@ def test_s3_zero_copy_on_hybrid_storage(started_cluster): == "(0,'data'),(1,'data')" ) - node1.query("DROP TABLE IF EXISTS hybrid_test NO DELAY") - node2.query("DROP TABLE IF EXISTS hybrid_test NO DELAY") + node1.query("DROP TABLE IF EXISTS hybrid_test SYNC") + node2.query("DROP TABLE IF EXISTS hybrid_test SYNC") def insert_data_time(node, table, number_of_mb, time, start=0): @@ -275,8 +275,8 @@ def test_s3_zero_copy_with_ttl_move( node1 = cluster.instances["node1"] node2 = cluster.instances["node2"] - node1.query("DROP TABLE IF EXISTS ttl_move_test NO DELAY") - node2.query("DROP TABLE IF EXISTS ttl_move_test NO DELAY") + node1.query("DROP TABLE IF EXISTS ttl_move_test SYNC") + node2.query("DROP TABLE IF EXISTS ttl_move_test SYNC") for i in range(iterations): node1.query( @@ -325,8 +325,8 @@ def test_s3_zero_copy_with_ttl_move( == "(10),(11)" ) - node1.query("DROP TABLE IF EXISTS ttl_move_test NO DELAY") - node2.query("DROP TABLE IF EXISTS ttl_move_test NO DELAY") + node1.query("DROP TABLE IF EXISTS ttl_move_test SYNC") + node2.query("DROP TABLE IF EXISTS ttl_move_test SYNC") @pytest.mark.parametrize( @@ -340,8 +340,8 @@ def test_s3_zero_copy_with_ttl_delete(started_cluster, large_data, iterations): node1 = cluster.instances["node1"] node2 = cluster.instances["node2"] - node1.query("DROP TABLE IF EXISTS ttl_delete_test NO DELAY") - node2.query("DROP TABLE IF EXISTS ttl_delete_test NO DELAY") + node1.query("DROP TABLE IF EXISTS ttl_delete_test SYNC") + node2.query("DROP TABLE IF EXISTS ttl_delete_test SYNC") for i in range(iterations): node1.query( @@ -398,8 +398,8 @@ def test_s3_zero_copy_with_ttl_delete(started_cluster, large_data, iterations): == "(11)" ) - node1.query("DROP TABLE IF EXISTS ttl_delete_test NO DELAY") - node2.query("DROP TABLE IF EXISTS ttl_delete_test NO DELAY") + node1.query("DROP TABLE IF EXISTS ttl_delete_test SYNC") + node2.query("DROP TABLE IF EXISTS ttl_delete_test SYNC") def wait_mutations(node, table, seconds): @@ -438,8 +438,8 @@ def s3_zero_copy_unfreeze_base(cluster, unfreeze_query_template): node1 = cluster.instances["node1"] node2 = cluster.instances["node2"] - node1.query("DROP TABLE IF EXISTS unfreeze_test NO DELAY") - node2.query("DROP TABLE IF EXISTS unfreeze_test NO DELAY") + node1.query("DROP TABLE IF EXISTS unfreeze_test SYNC") + node2.query("DROP TABLE IF EXISTS unfreeze_test SYNC") node1.query( """ @@ -489,8 +489,8 @@ def s3_zero_copy_unfreeze_base(cluster, unfreeze_query_template): check_objects_not_exisis(cluster, objects12) - node1.query("DROP TABLE IF EXISTS unfreeze_test NO DELAY") - node2.query("DROP TABLE IF EXISTS unfreeze_test NO DELAY") + node1.query("DROP TABLE IF EXISTS unfreeze_test SYNC") + node2.query("DROP TABLE IF EXISTS unfreeze_test SYNC") def test_s3_zero_copy_unfreeze_alter(started_cluster): @@ -505,8 +505,8 @@ def s3_zero_copy_drop_detached(cluster, unfreeze_query_template): node1 = cluster.instances["node1"] node2 = cluster.instances["node2"] - node1.query("DROP TABLE IF EXISTS drop_detached_test NO DELAY") - node2.query("DROP TABLE IF EXISTS drop_detached_test NO DELAY") + node1.query("DROP TABLE IF EXISTS drop_detached_test SYNC") + node2.query("DROP TABLE IF EXISTS drop_detached_test SYNC") node1.query( """ @@ -600,8 +600,8 @@ def test_s3_zero_copy_concurrent_merge(started_cluster): node1 = cluster.instances["node1"] node2 = cluster.instances["node2"] - node1.query("DROP TABLE IF EXISTS concurrent_merge NO DELAY") - node2.query("DROP TABLE IF EXISTS concurrent_merge NO DELAY") + node1.query("DROP TABLE IF EXISTS concurrent_merge SYNC") + node2.query("DROP TABLE IF EXISTS concurrent_merge SYNC") for node in (node1, node2): node.query( @@ -647,8 +647,8 @@ def test_s3_zero_copy_keeps_data_after_mutation(started_cluster): node1 = cluster.instances["node1"] node2 = cluster.instances["node2"] - node1.query("DROP TABLE IF EXISTS zero_copy_mutation NO DELAY") - node2.query("DROP TABLE IF EXISTS zero_copy_mutation NO DELAY") + node1.query("DROP TABLE IF EXISTS zero_copy_mutation SYNC") + node2.query("DROP TABLE IF EXISTS zero_copy_mutation SYNC") node1.query( """ diff --git a/tests/integration/test_storage_hdfs/test.py b/tests/integration/test_storage_hdfs/test.py index d4752d6cf2e..edf5344e887 100644 --- a/tests/integration/test_storage_hdfs/test.py +++ b/tests/integration/test_storage_hdfs/test.py @@ -788,6 +788,7 @@ def test_schema_inference_cache(started_cluster): def test_hdfsCluster_skip_unavailable_shards(started_cluster): + # Although skip_unavailable_shards is not set, cluster table functions should always skip unavailable shards. hdfs_api = started_cluster.hdfs_api node = started_cluster.instances["node1"] data = "1\tSerialize\t555.222\n2\tData\t777.333\n" @@ -801,16 +802,18 @@ def test_hdfsCluster_skip_unavailable_shards(started_cluster): ) -def test_hdfsCluster_unskip_unavailable_shards(started_cluster): +def test_hdfsCluster_unset_skip_unavailable_shards(started_cluster): hdfs_api = started_cluster.hdfs_api node = started_cluster.instances["node1"] data = "1\tSerialize\t555.222\n2\tData\t777.333\n" hdfs_api.write_data("/unskip_unavailable_shards", data) - error = node.query_and_get_error( - "select * from hdfsCluster('cluster_non_existent_port', 'hdfs://hdfs1:9000/unskip_unavailable_shards', 'TSV', 'id UInt64, text String, number Float64')" - ) - assert "NETWORK_ERROR" in error + assert ( + node1.query( + "select * from hdfsCluster('cluster_non_existent_port', 'hdfs://hdfs1:9000/skip_unavailable_shards', 'TSV', 'id UInt64, text String, number Float64')" + ) + == data + ) if __name__ == "__main__": diff --git a/tests/integration/test_storage_nats/test.py b/tests/integration/test_storage_nats/test.py index 2988c67bf63..1d7e046864b 100644 --- a/tests/integration/test_storage_nats/test.py +++ b/tests/integration/test_storage_nats/test.py @@ -94,7 +94,7 @@ def nats_cluster(): def nats_setup_teardown(): print("NATS is available - running test") yield # run test - instance.query("DROP DATABASE test NO DELAY") + instance.query("DROP DATABASE test SYNC") instance.query("CREATE DATABASE test") diff --git a/tests/integration/test_storage_postgresql_replica/test.py b/tests/integration/test_storage_postgresql_replica/test.py index 8666d7ae58c..61a9dd5687b 100644 --- a/tests/integration/test_storage_postgresql_replica/test.py +++ b/tests/integration/test_storage_postgresql_replica/test.py @@ -179,7 +179,7 @@ def test_initial_load_from_snapshot(started_cluster): cursor.execute("DROP TABLE postgresql_replica;") postgresql_replica_check_result(result, True) - instance.query(f"DROP TABLE test.postgresql_replica NO DELAY") + instance.query(f"DROP TABLE test.postgresql_replica SYNC") @pytest.mark.timeout(320) @@ -216,7 +216,7 @@ def test_no_connection_at_startup(started_cluster): result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;") cursor.execute("DROP TABLE postgresql_replica;") postgresql_replica_check_result(result, True) - instance.query(f"DROP TABLE test.postgresql_replica NO DELAY") + instance.query(f"DROP TABLE test.postgresql_replica SYNC") @pytest.mark.timeout(320) @@ -255,7 +255,7 @@ def test_detach_attach_is_ok(started_cluster): cursor.execute("DROP TABLE postgresql_replica;") postgresql_replica_check_result(result, True) - instance.query(f"DROP TABLE test.postgresql_replica NO DELAY") + instance.query(f"DROP TABLE test.postgresql_replica SYNC") @pytest.mark.timeout(320) @@ -309,7 +309,7 @@ def test_replicating_insert_queries(started_cluster): result = instance.query("SELECT * FROM test.postgresql_replica ORDER BY key;") cursor.execute("DROP TABLE postgresql_replica;") postgresql_replica_check_result(result, True) - instance.query(f"DROP TABLE test.postgresql_replica NO DELAY") + instance.query(f"DROP TABLE test.postgresql_replica SYNC") @pytest.mark.timeout(320) @@ -667,7 +667,7 @@ def test_virtual_columns(started_cluster): ) print(result) cursor.execute("DROP TABLE postgresql_replica;") - instance.query(f"DROP TABLE test.postgresql_replica NO DELAY") + instance.query(f"DROP TABLE test.postgresql_replica SYNC") def test_abrupt_connection_loss_while_heavy_replication(started_cluster): @@ -702,7 +702,7 @@ def test_abrupt_connection_loss_while_heavy_replication(started_cluster): result = instance.query("SELECT count() FROM test.postgresql_replica") print(result) # Just debug - instance.query(f"DROP TABLE test.postgresql_replica NO DELAY") + instance.query(f"DROP TABLE test.postgresql_replica SYNC") def test_abrupt_server_restart_while_heavy_replication(started_cluster): @@ -720,7 +720,7 @@ def test_abrupt_server_restart_while_heavy_replication(started_cluster): create_postgres_table(cursor, table_name) instance.query(f"INSERT INTO postgres_database.{table_name} SELECT -1, 1") - instance.query(f"DROP TABLE IF EXISTS test.{table_name} NO DELAY") + instance.query(f"DROP TABLE IF EXISTS test.{table_name} SYNC") create_materialized_table( ip=started_cluster.postgres_ip, port=started_cluster.postgres_port, @@ -747,7 +747,7 @@ def test_abrupt_server_restart_while_heavy_replication(started_cluster): result = instance.query(f"SELECT count() FROM test.{table_name}") print(result) # Just debug - instance.query(f"DROP TABLE test.{table_name} NO DELAY") + instance.query(f"DROP TABLE test.{table_name} SYNC") def test_drop_table_immediately(started_cluster): @@ -771,7 +771,7 @@ def test_drop_table_immediately(started_cluster): ip=started_cluster.postgres_ip, port=started_cluster.postgres_port ) check_tables_are_synchronized("postgresql_replica") - instance.query(f"DROP TABLE test.postgresql_replica NO DELAY") + instance.query(f"DROP TABLE test.postgresql_replica SYNC") if __name__ == "__main__": diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 4ca49e32c55..4e1e28373e3 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -95,7 +95,7 @@ def rabbitmq_cluster(): def rabbitmq_setup_teardown(): print("RabbitMQ is available - running test") yield # run test - instance.query("DROP DATABASE test NO DELAY") + instance.query("DROP DATABASE test SYNC") instance.query("CREATE DATABASE test") @@ -1097,10 +1097,10 @@ def test_rabbitmq_overloaded_insert(rabbitmq_cluster): instance.query( """ - DROP TABLE test.consumer_overload NO DELAY; - DROP TABLE test.view_overload NO DELAY; - DROP TABLE test.rabbitmq_consume NO DELAY; - DROP TABLE test.rabbitmq_overload NO DELAY; + DROP TABLE test.consumer_overload SYNC; + DROP TABLE test.view_overload SYNC; + DROP TABLE test.rabbitmq_consume SYNC; + DROP TABLE test.rabbitmq_overload SYNC; """ ) @@ -2745,7 +2745,7 @@ def test_rabbitmq_drop_mv(rabbitmq_cluster): result = instance.query("SELECT * FROM test.view ORDER BY key") rabbitmq_check_result(result, True) - instance.query("DROP VIEW test.consumer NO DELAY") + instance.query("DROP VIEW test.consumer SYNC") time.sleep(10) for i in range(50, 60): channel.basic_publish( diff --git a/tests/integration/test_system_metrics/test.py b/tests/integration/test_system_metrics/test.py index 439e8b66db1..8539828a8b8 100644 --- a/tests/integration/test_system_metrics/test.py +++ b/tests/integration/test_system_metrics/test.py @@ -157,3 +157,57 @@ def test_metrics_storage_buffer_size(start_cluster): ) == "0\n" ) + + +def test_attach_without_zk_incr_readonly_metric(start_cluster): + assert ( + node1.query("SELECT value FROM system.metrics WHERE metric = 'ReadonlyReplica'") + == "0\n" + ) + + node1.query( + "ATTACH TABLE test.test_no_zk UUID 'a50b7933-59b2-49ce-8db6-59da3c9b4413' (i Int8, d Date) ENGINE = ReplicatedMergeTree('no_zk', 'replica') ORDER BY tuple()" + ) + assert_eq_with_retry( + node1, + "SELECT value FROM system.metrics WHERE metric = 'ReadonlyReplica'", + "1\n", + retry_count=300, + sleep_time=1, + ) + + node1.query("DETACH TABLE test.test_no_zk") + assert_eq_with_retry( + node1, + "SELECT value FROM system.metrics WHERE metric = 'ReadonlyReplica'", + "0\n", + retry_count=300, + sleep_time=1, + ) + + node1.query("ATTACH TABLE test.test_no_zk") + assert_eq_with_retry( + node1, + "SELECT value FROM system.metrics WHERE metric = 'ReadonlyReplica'", + "1\n", + retry_count=300, + sleep_time=1, + ) + + node1.query("SYSTEM RESTORE REPLICA test.test_no_zk") + assert_eq_with_retry( + node1, + "SELECT value FROM system.metrics WHERE metric = 'ReadonlyReplica'", + "0\n", + retry_count=300, + sleep_time=1, + ) + + node1.query("DROP TABLE test.test_no_zk") + assert_eq_with_retry( + node1, + "SELECT value FROM system.metrics WHERE metric = 'ReadonlyReplica'", + "0\n", + retry_count=300, + sleep_time=1, + ) diff --git a/tests/integration/test_temporary_data_in_cache/configs/config.d/storage_configuration.xml b/tests/integration/test_temporary_data_in_cache/configs/config.d/storage_configuration.xml index acf0f765c6c..b527c74e8de 100644 --- a/tests/integration/test_temporary_data_in_cache/configs/config.d/storage_configuration.xml +++ b/tests/integration/test_temporary_data_in_cache/configs/config.d/storage_configuration.xml @@ -2,7 +2,7 @@ - local + local_blob_storage /local_disk/ diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py index 89824293320..7635d784fef 100644 --- a/tests/integration/test_ttl_move/test.py +++ b/tests/integration/test_ttl_move/test.py @@ -151,7 +151,7 @@ def test_rule_with_invalid_destination(started_cluster, name, engine, alter): get_command("TTL d1 TO DISK 'unknown'", "small_jbod_with_external") ) - node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + node1.query("DROP TABLE IF EXISTS {} SYNC".format(name)) if alter: node1.query(get_command(None, "small_jbod_with_external")) @@ -161,7 +161,7 @@ def test_rule_with_invalid_destination(started_cluster, name, engine, alter): get_command("TTL d1 TO VOLUME 'unknown'", "small_jbod_with_external") ) - node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + node1.query("DROP TABLE IF EXISTS {} SYNC".format(name)) if alter: node1.query(get_command(None, "only_jbod2")) @@ -169,7 +169,7 @@ def test_rule_with_invalid_destination(started_cluster, name, engine, alter): with pytest.raises(QueryRuntimeException): node1.query(get_command("TTL d1 TO DISK 'jbod1'", "only_jbod2")) - node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + node1.query("DROP TABLE IF EXISTS {} SYNC".format(name)) if alter: node1.query(get_command(None, "only_jbod2")) @@ -178,7 +178,7 @@ def test_rule_with_invalid_destination(started_cluster, name, engine, alter): node1.query(get_command("TTL d1 TO VOLUME 'external'", "only_jbod2")) finally: - node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + node1.query("DROP TABLE IF EXISTS {} SYNC".format(name)) @pytest.mark.parametrize( @@ -253,7 +253,7 @@ def test_inserts_to_disk_work(started_cluster, name, engine, positive): finally: try: - node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + node1.query("DROP TABLE IF EXISTS {} SYNC".format(name)) except: pass @@ -330,7 +330,7 @@ def test_moves_work_after_storage_policy_change(started_cluster, name, engine): ) finally: - node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + node1.query("DROP TABLE IF EXISTS {} SYNC".format(name)) @pytest.mark.parametrize( @@ -418,7 +418,7 @@ def test_moves_to_disk_work(started_cluster, name, engine, positive): ) finally: - node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + node1.query("DROP TABLE IF EXISTS {} SYNC".format(name)) @pytest.mark.parametrize( @@ -489,7 +489,7 @@ def test_moves_to_volume_work(started_cluster, name, engine): ) finally: - node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + node1.query("DROP TABLE IF EXISTS {} SYNC".format(name)) @pytest.mark.parametrize( @@ -570,7 +570,7 @@ def test_inserts_to_volume_work(started_cluster, name, engine, positive): ) finally: - node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + node1.query("DROP TABLE IF EXISTS {} SYNC".format(name)) @pytest.mark.parametrize( @@ -649,7 +649,7 @@ def test_moves_to_disk_eventually_work(started_cluster, name, engine): used_disks = get_used_disks_for_table(node1, name) assert set(used_disks) == {"jbod1"} - node1.query("DROP TABLE {} NO DELAY".format(name_temp)) + node1.query("DROP TABLE {} SYNC".format(name_temp)) wait_parts_mover(node1, name) @@ -661,8 +661,8 @@ def test_moves_to_disk_eventually_work(started_cluster, name, engine): ) finally: - node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name_temp)) - node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + node1.query("DROP TABLE IF EXISTS {} SYNC".format(name_temp)) + node1.query("DROP TABLE IF EXISTS {} SYNC".format(name)) def test_replicated_download_ttl_info(started_cluster): @@ -702,7 +702,7 @@ def test_replicated_download_ttl_info(started_cluster): finally: for node in (node1, node2): try: - node.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + node.query("DROP TABLE IF EXISTS {} SYNC".format(name)) except: continue @@ -818,7 +818,7 @@ def test_merges_to_disk_work(started_cluster, name, engine, positive): ) finally: - node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + node1.query("DROP TABLE IF EXISTS {} SYNC".format(name)) @pytest.mark.parametrize( @@ -932,8 +932,8 @@ def test_merges_with_full_disk_work(started_cluster, name, engine): ) finally: - node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name_temp)) - node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + node1.query("DROP TABLE IF EXISTS {} SYNC".format(name_temp)) + node1.query("DROP TABLE IF EXISTS {} SYNC".format(name)) @pytest.mark.parametrize( @@ -1035,7 +1035,7 @@ def test_moves_after_merges_work(started_cluster, name, engine, positive): ) finally: - node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + node1.query("DROP TABLE IF EXISTS {} SYNC".format(name)) @pytest.mark.parametrize( @@ -1150,7 +1150,7 @@ def test_ttls_do_not_work_after_alter(started_cluster, name, engine, positive, b ) finally: - node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + node1.query("DROP TABLE IF EXISTS {} SYNC".format(name)) @pytest.mark.parametrize( @@ -1255,7 +1255,7 @@ def test_materialize_ttl_in_partition(started_cluster, name, engine): ).strip() == str(len(data)) finally: - node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + node1.query("DROP TABLE IF EXISTS {} SYNC".format(name)) @pytest.mark.parametrize( @@ -1378,7 +1378,7 @@ def test_alter_multiple_ttls(started_cluster, name, engine, positive): assert rows_count == 3 finally: - node1.query("DROP TABLE IF EXISTS {name} NO DELAY".format(name=name)) + node1.query("DROP TABLE IF EXISTS {name} SYNC".format(name=name)) @pytest.mark.parametrize( @@ -1526,7 +1526,7 @@ def test_concurrent_alter_with_ttl_move(started_cluster, name, engine): assert node1.query("SELECT 1") == "1\n" assert node1.query("SELECT COUNT() FROM {}".format(name)) == "150\n" finally: - node1.query("DROP TABLE IF EXISTS {name} NO DELAY".format(name=name)) + node1.query("DROP TABLE IF EXISTS {name} SYNC".format(name=name)) @pytest.mark.skip(reason="Flacky test") @@ -1626,7 +1626,7 @@ def test_double_move_while_select(started_cluster, name, positive): ).splitlines() == ["1", "2", "3", "4"] finally: - node1.query("DROP TABLE IF EXISTS {name} NO DELAY".format(name=name)) + node1.query("DROP TABLE IF EXISTS {name} SYNC".format(name=name)) @pytest.mark.parametrize( @@ -1745,7 +1745,7 @@ def test_alter_with_merge_work(started_cluster, name, engine, positive): assert node1.query("SELECT count() FROM {name}".format(name=name)) == "6\n" finally: - node1.query("DROP TABLE IF EXISTS {name} NO DELAY".format(name=name)) + node1.query("DROP TABLE IF EXISTS {name} SYNC".format(name=name)) @pytest.mark.parametrize( @@ -1826,7 +1826,7 @@ def test_disabled_ttl_move_on_insert(started_cluster, name, dest_type, engine): finally: try: - node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + node1.query("DROP TABLE IF EXISTS {} SYNC".format(name)) except: pass @@ -1909,7 +1909,7 @@ def test_ttl_move_if_exists(started_cluster, name, dest_type): finally: try: - node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) - node2.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + node1.query("DROP TABLE IF EXISTS {} SYNC".format(name)) + node2.query("DROP TABLE IF EXISTS {} SYNC".format(name)) except: pass diff --git a/tests/integration/test_ttl_replicated/test.py b/tests/integration/test_ttl_replicated/test.py index 13fea0455d0..a3e7d6e4b8b 100644 --- a/tests/integration/test_ttl_replicated/test.py +++ b/tests/integration/test_ttl_replicated/test.py @@ -55,7 +55,7 @@ def started_cluster(): def drop_table(nodes, table_name): for node in nodes: - node.query("DROP TABLE IF EXISTS {} NO DELAY".format(table_name)) + node.query("DROP TABLE IF EXISTS {} SYNC".format(table_name)) # Column TTL works only with wide parts, because it's very expensive to apply it for compact parts diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.reference b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.reference index cd9da983785..11b660b54a3 100644 --- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.reference +++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.reference @@ -2,7 +2,7 @@ runtime messages 0.001 runtime exceptions 0.05 messages shorter than 10 1 messages shorter than 16 3 -exceptions shorter than 30 30 +exceptions shorter than 30 3 noisy messages 0.3 noisy Trace messages 0.16 noisy Debug messages 0.09 diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql index 480effec065..7796785afb5 100644 --- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql +++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql @@ -49,7 +49,14 @@ create temporary table known_short_messages (s String) as select * from (select 'Column ''{}'' already exists', 'No macro {} in config', 'Invalid origin H3 index: {}', 'Invalid session timeout: ''{}''', 'Tuple cannot be empty', 'Database name is empty', 'Table {} is not a Dictionary', 'Expected function, got: {}', 'Unknown identifier: ''{}''', -'Failed to {} input ''{}''', '{}.{} is not a VIEW', 'Cannot convert NULL to {}', 'Dictionary {} doesn''t exist' +'Failed to {} input ''{}''', '{}.{} is not a VIEW', 'Cannot convert NULL to {}', 'Dictionary {} doesn''t exist', +'Write file: {}', 'Unable to parse JSONPath', 'Host is empty in S3 URI.', 'Expected end of line', +'inflate failed: {}{}', 'Center is not valid', 'Column ''{}'' is ambiguous', 'Cannot parse object', 'Invalid date: {}', +'There is no cache by name: {}', 'No part {} in table', '`{}` should be a String', 'There are duplicate id {}', +'Invalid replica name: {}', 'Unexpected value {} in enum', 'Unknown BSON type: {}', 'Point is not valid', +'Invalid qualified name: {}', 'INTO OUTFILE is not allowed', 'Arguments must not be NaN', 'Cell is not valid', +'brotli decode error{}', 'Invalid H3 index: {}', 'Too large node state size', 'No additional keys found.', +'Attempt to read after EOF.', 'Replication was stopped', '{} building file infos', 'Cannot parse uuid {}' ] as arr) array join arr; -- Check that we don't have too many short meaningless message patterns. @@ -59,7 +66,7 @@ select 'messages shorter than 10', max2(countDistinctOrDefault(message_format_st select 'messages shorter than 16', max2(countDistinctOrDefault(message_format_string), 3) from logs where length(message_format_string) < 16 and message_format_string not in known_short_messages; -- Same as above, but exceptions must be more informative. Feel free to update the threshold or remove this query if really necessary -select 'exceptions shorter than 30', max2(countDistinctOrDefault(message_format_string), 30) from logs where length(message_format_string) < 30 and message ilike '%DB::Exception%' and message_format_string not in known_short_messages; +select 'exceptions shorter than 30', max2(countDistinctOrDefault(message_format_string), 3) from logs where length(message_format_string) < 30 and message ilike '%DB::Exception%' and message_format_string not in known_short_messages; -- Avoid too noisy messages: top 1 message frequency must be less than 30%. We should reduce the threshold @@ -98,7 +105,9 @@ select 'incorrect patterns', max2(countDistinct(message_format_string), 15) from where ((rand() % 8) = 0) and message not like (replaceRegexpAll(message_format_string, '{[:.0-9dfx]*}', '%') as s) and message not like (s || ' (skipped % similar messages)') - and message not like ('%Exception: '||s||'%') group by message_format_string + and message not like ('%Exception: '||s||'%') + and message not like ('%(skipped % similar messages)%') + group by message_format_string ) where any_message not like '%Poco::Exception%'; drop table logs; diff --git a/tests/queries/0_stateless/00155_long_merges.sh b/tests/queries/0_stateless/00155_long_merges.sh index 83d89c57cfa..9ed0f2c6de1 100755 --- a/tests/queries/0_stateless/00155_long_merges.sh +++ b/tests/queries/0_stateless/00155_long_merges.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long +# Tags: long, no-debug CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/00189_time_zones_long.reference b/tests/queries/0_stateless/00189_time_zones_long.reference index 8717a662771..d41c925bbe5 100644 --- a/tests/queries/0_stateless/00189_time_zones_long.reference +++ b/tests/queries/0_stateless/00189_time_zones_long.reference @@ -246,18 +246,18 @@ toUnixTimestamp 1426415400 1426415400 date_trunc -2019-01-01 -2020-01-01 -2020-01-01 -2019-10-01 -2020-01-01 -2020-01-01 -2019-12-01 -2020-01-01 -2020-01-01 -2019-12-30 -2019-12-30 -2019-12-30 +2019-01-01 00:00:00 +2020-01-01 00:00:00 +2020-01-01 00:00:00 +2019-10-01 00:00:00 +2020-01-01 00:00:00 +2020-01-01 00:00:00 +2019-12-01 00:00:00 +2020-01-01 00:00:00 +2020-01-01 00:00:00 +2019-12-30 00:00:00 +2019-12-30 00:00:00 +2019-12-30 00:00:00 2019-12-31 00:00:00 2020-01-01 00:00:00 2020-01-02 00:00:00 @@ -270,18 +270,18 @@ date_trunc 2019-12-31 20:11:22 2020-01-01 12:11:22 2020-01-02 05:11:22 -2019-01-01 -2020-01-01 -2020-01-01 -2019-10-01 -2020-01-01 -2020-01-01 -2019-12-01 -2020-01-01 -2020-01-01 -2019-12-30 -2019-12-30 -2019-12-30 +2019-01-01 00:00:00 +2020-01-01 00:00:00 +2020-01-01 00:00:00 +2019-10-01 00:00:00 +2020-01-01 00:00:00 +2020-01-01 00:00:00 +2019-12-01 00:00:00 +2020-01-01 00:00:00 +2020-01-01 00:00:00 +2019-12-30 00:00:00 +2019-12-30 00:00:00 +2019-12-30 00:00:00 2019-12-31 00:00:00 2020-01-01 00:00:00 2020-01-02 00:00:00 @@ -294,8 +294,8 @@ date_trunc 2019-12-31 20:11:22 2020-01-01 12:11:22 2020-01-02 05:11:22 -2020-01-01 -2020-01-01 -2020-01-01 -2019-12-30 +2020-01-01 00:00:00 +2020-01-01 00:00:00 +2020-01-01 00:00:00 +2019-12-30 00:00:00 2020-01-01 00:00:00 diff --git a/tests/queries/0_stateless/00322_disable_checksumming.sh b/tests/queries/0_stateless/00322_disable_checksumming.sh index c044a5c6650..e04ec076f80 100755 --- a/tests/queries/0_stateless/00322_disable_checksumming.sh +++ b/tests/queries/0_stateless/00322_disable_checksumming.sh @@ -4,10 +4,5 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -# use big-endian version of binary data for s390x -if [[ $(uname -a | grep s390x) ]]; then -echo -ne '\xdb\x8a\xe9\x59\xf2\x32\x74\x50\x39\xc4\x22\xfb\xa7\x4a\xc6\x37''\x82\x13\x00\x00\x00\x09\x00\x00\x00''\x90SELECT 1\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&decompress=1" --data-binary @- -else echo -ne '\x50\x74\x32\xf2\x59\xe9\x8a\xdb\x37\xc6\x4a\xa7\xfb\x22\xc4\x39''\x82\x13\x00\x00\x00\x09\x00\x00\x00''\x90SELECT 1\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&decompress=1" --data-binary @- -fi echo -ne 'xxxxxxxxxxxxxxxx''\x82\x13\x00\x00\x00\x09\x00\x00\x00''\x90SELECT 1\n' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&decompress=1&http_native_compression_disable_checksumming_on_decompress=1" --data-binary @- diff --git a/tests/queries/0_stateless/00700_decimal_aggregates.reference b/tests/queries/0_stateless/00700_decimal_aggregates.reference index acf41546f5c..79195312867 100644 --- a/tests/queries/0_stateless/00700_decimal_aggregates.reference +++ b/tests/queries/0_stateless/00700_decimal_aggregates.reference @@ -5,7 +5,7 @@ -1275 -424.99999983 -255 -1275 -424.99999983 -255 101 101 101 101 101 101 -101 -101 -101 -101 -101 -101 -(101,101,101) (101,101,101) (101,101,101) (101,101,101) (102,100,101) +(101,101,101) (101,101,101) (101,101,101) (101,101,101) (1,1,1,1,1,1) 5 5 5 10 10 10 -50 -50 -16.66666666 -16.66666666 -10 -10 diff --git a/tests/queries/0_stateless/00700_decimal_aggregates.sql b/tests/queries/0_stateless/00700_decimal_aggregates.sql index a1814fc866f..6ca37e06918 100644 --- a/tests/queries/0_stateless/00700_decimal_aggregates.sql +++ b/tests/queries/0_stateless/00700_decimal_aggregates.sql @@ -24,7 +24,7 @@ SELECT (uniq(a), uniq(b), uniq(c)), (uniqCombined(a), uniqCombined(b), uniqCombined(c)), (uniqCombined(17)(a), uniqCombined(17)(b), uniqCombined(17)(c)), (uniqExact(a), uniqExact(b), uniqExact(c)), - (uniqHLL12(a), uniqHLL12(b), uniqHLL12(c)) + (102 - uniqHLL12(a) >= 0, 102 - uniqHLL12(b) >= 0, 102 - uniqHLL12(c) >= 0, uniqHLL12(a) - 99 >= 0, uniqHLL12(b) - 99 >= 0, uniqHLL12(c) - 99 >= 0) FROM (SELECT * FROM decimal ORDER BY a); SELECT uniqUpTo(10)(a), uniqUpTo(10)(b), uniqUpTo(10)(c) FROM decimal WHERE a >= 0 AND a < 5; diff --git a/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh b/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh index 19d9b006cd7..71acc11b971 100755 --- a/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh +++ b/tests/queries/0_stateless/00816_long_concurrent_alter_column.sh @@ -59,7 +59,7 @@ timeout $TIMEOUT bash -c thread4 2> /dev/null & wait -echo "DROP TABLE concurrent_alter_column NO DELAY" | ${CLICKHOUSE_CLIENT} # NO DELAY has effect only for Atomic database +echo "DROP TABLE concurrent_alter_column SYNC" | ${CLICKHOUSE_CLIENT} # SYNC has effect only for Atomic database # Wait for alters and check for deadlocks (in case of deadlock this loop will not finish) while true; do diff --git a/tests/queries/0_stateless/00921_datetime64_compatibility_long.reference b/tests/queries/0_stateless/00921_datetime64_compatibility_long.reference index 62de3a149a7..4f964f2478f 100644 --- a/tests/queries/0_stateless/00921_datetime64_compatibility_long.reference +++ b/tests/queries/0_stateless/00921_datetime64_compatibility_long.reference @@ -135,13 +135,13 @@ Code: 43 ------------------------------------------ SELECT date_trunc(\'year\', N, \'Asia/Istanbul\') Code: 43 -"Date","2019-01-01" -"Date","2019-01-01" +"DateTime('Asia/Istanbul')","2019-01-01 00:00:00" +"DateTime('Asia/Istanbul')","2019-01-01 00:00:00" ------------------------------------------ SELECT date_trunc(\'month\', N, \'Asia/Istanbul\') Code: 43 -"Date","2019-09-01" -"Date","2019-09-01" +"DateTime('Asia/Istanbul')","2019-09-01 00:00:00" +"DateTime('Asia/Istanbul')","2019-09-01 00:00:00" ------------------------------------------ SELECT date_trunc(\'day\', N, \'Asia/Istanbul\') "DateTime('Asia/Istanbul')","2019-09-16 00:00:00" diff --git a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh index d49f63e143d..5b1c50262bf 100755 --- a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh +++ b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh @@ -13,8 +13,8 @@ $CLICKHOUSE_CLIENT -n -q " DROP TABLE IF EXISTS alter_table0; DROP TABLE IF EXISTS alter_table1; - CREATE TABLE alter_table0 (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r1') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0, replicated_max_mutations_in_one_entry = $(($RANDOM / 50)); - CREATE TABLE alter_table1 (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r2') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0, replicated_max_mutations_in_one_entry = $(($RANDOM / 50)); + CREATE TABLE alter_table0 (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r1') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0, replicated_max_mutations_in_one_entry = $(($RANDOM / 50 + 100)); + CREATE TABLE alter_table1 (a UInt8, b Int16, c Float32, d String, e Array(UInt8), f Nullable(UUID), g Tuple(UInt8, UInt16)) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_table', 'r2') ORDER BY a PARTITION BY b % 10 SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 1, cleanup_delay_period_random_add = 0, replicated_max_mutations_in_one_entry = $(($RANDOM / 50 + 200)); " function thread1() diff --git a/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh b/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh index bceda77c7f8..f4f38ad9c83 100755 --- a/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh +++ b/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh @@ -63,7 +63,6 @@ function thread6() done } - # https://stackoverflow.com/questions/9954794/execute-a-shell-function-with-timeout export -f thread1; export -f thread2; diff --git a/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh index 4c0afc4c439..8ef03be02b6 100755 --- a/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh +++ b/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh @@ -70,8 +70,8 @@ timeout $TIMEOUT bash -c thread5 2> /dev/null & wait -echo "DROP TABLE src NO DELAY" | ${CLICKHOUSE_CLIENT} -echo "DROP TABLE dst NO DELAY" | ${CLICKHOUSE_CLIENT} +echo "DROP TABLE src SYNC" | ${CLICKHOUSE_CLIENT} +echo "DROP TABLE dst SYNC" | ${CLICKHOUSE_CLIENT} sleep 5 # Check for deadlocks diff --git a/tests/queries/0_stateless/01050_window_view_parser_tumble.sql b/tests/queries/0_stateless/01050_window_view_parser_tumble.sql index f49fbc251fd..c52a6fefacb 100644 --- a/tests/queries/0_stateless/01050_window_view_parser_tumble.sql +++ b/tests/queries/0_stateless/01050_window_view_parser_tumble.sql @@ -6,29 +6,29 @@ DROP TABLE IF EXISTS mt; CREATE TABLE mt(a Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple(); SELECT '---WATERMARK---'; -DROP TABLE IF EXISTS wv NO DELAY; +DROP TABLE IF EXISTS wv SYNC; CREATE WINDOW VIEW wv ENGINE Memory WATERMARK=INTERVAL '1' SECOND AS SELECT count(a), tumbleStart(wid) AS w_start, tumbleEnd(wid) AS w_end FROM mt GROUP BY tumble(timestamp, INTERVAL '3' SECOND) AS wid; SELECT '---With w_end---'; -DROP TABLE IF EXISTS wv NO DELAY; +DROP TABLE IF EXISTS wv SYNC; CREATE WINDOW VIEW wv ENGINE Memory AS SELECT count(a), tumbleStart(tumble(timestamp, INTERVAL '3' SECOND)) AS w_start, tumbleEnd(wid) AS w_end FROM mt GROUP BY tumble(timestamp, INTERVAL '3' SECOND) AS wid; SELECT '---WithOut w_end---'; -DROP TABLE IF EXISTS wv NO DELAY; +DROP TABLE IF EXISTS wv SYNC; CREATE WINDOW VIEW wv ENGINE Memory AS SELECT count(a), tumbleStart(wid) AS w_start FROM mt GROUP BY tumble(timestamp, INTERVAL '3' SECOND) AS wid; SELECT '---WITH---'; -DROP TABLE IF EXISTS wv NO DELAY; +DROP TABLE IF EXISTS wv SYNC; CREATE WINDOW VIEW wv ENGINE Memory AS WITH toDateTime('2018-01-01 00:00:00') AS date_time SELECT count(a), tumbleStart(wid) AS w_start, tumbleEnd(wid) AS w_end, date_time FROM mt GROUP BY tumble(timestamp, INTERVAL '3' SECOND) AS wid; SELECT '---WHERE---'; -DROP TABLE IF EXISTS wv NO DELAY; +DROP TABLE IF EXISTS wv SYNC; CREATE WINDOW VIEW wv ENGINE Memory AS SELECT count(a), tumbleStart(wid) AS w_start FROM mt WHERE a != 1 GROUP BY tumble(timestamp, INTERVAL '3' SECOND) AS wid; SELECT '---ORDER_BY---'; -DROP TABLE IF EXISTS wv NO DELAY; +DROP TABLE IF EXISTS wv SYNC; CREATE WINDOW VIEW wv ENGINE Memory AS SELECT count(a), tumbleStart(wid) AS w_start FROM mt WHERE a != 1 GROUP BY tumble(timestamp, INTERVAL '3' SECOND) AS wid ORDER BY w_start; SELECT '---With now---'; -DROP TABLE IF EXISTS wv NO DELAY; +DROP TABLE IF EXISTS wv SYNC; CREATE WINDOW VIEW wv ENGINE Memory AS SELECT count(a), tumbleStart(wid) AS w_start, tumbleEnd(tumble(now(), INTERVAL '3' SECOND)) AS w_end FROM mt GROUP BY tumble(now(), INTERVAL '3' SECOND) AS wid; diff --git a/tests/queries/0_stateless/01051_window_view_parser_hop.sql b/tests/queries/0_stateless/01051_window_view_parser_hop.sql index 45877cf0647..b37e4ed3095 100644 --- a/tests/queries/0_stateless/01051_window_view_parser_hop.sql +++ b/tests/queries/0_stateless/01051_window_view_parser_hop.sql @@ -6,29 +6,29 @@ DROP TABLE IF EXISTS mt; CREATE TABLE mt(a Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple(); SELECT '---WATERMARK---'; -DROP TABLE IF EXISTS wv NO DELAY; +DROP TABLE IF EXISTS wv SYNC; CREATE WINDOW VIEW wv ENGINE Memory WATERMARK=INTERVAL '1' SECOND AS SELECT count(a), hopStart(wid) AS w_start, hopEnd(wid) AS w_end FROM mt GROUP BY hop(timestamp, INTERVAL '3' SECOND, INTERVAL '5' SECOND) AS wid; SELECT '---With w_end---'; -DROP TABLE IF EXISTS wv NO DELAY; +DROP TABLE IF EXISTS wv SYNC; CREATE WINDOW VIEW wv ENGINE Memory AS SELECT count(a), hopStart(wid) AS w_start, hopEnd(wid) AS w_end FROM mt GROUP BY hop(timestamp, INTERVAL '3' SECOND, INTERVAL '5' SECOND) AS wid; SELECT '---WithOut w_end---'; -DROP TABLE IF EXISTS wv NO DELAY; +DROP TABLE IF EXISTS wv SYNC; CREATE WINDOW VIEW wv ENGINE Memory AS SELECT count(a), hopStart(wid) AS w_start FROM mt GROUP BY hop(timestamp, INTERVAL '3' SECOND, INTERVAL '5' SECOND) AS wid; SELECT '---WITH---'; -DROP TABLE IF EXISTS wv NO DELAY; +DROP TABLE IF EXISTS wv SYNC; CREATE WINDOW VIEW wv ENGINE Memory AS WITH toDateTime('2018-01-01 00:00:00') AS date_time SELECT count(a), hopStart(wid) AS w_start, hopEnd(wid) AS w_end, date_time FROM mt GROUP BY hop(timestamp, INTERVAL '3' SECOND, INTERVAL '5' SECOND) AS wid; SELECT '---WHERE---'; -DROP TABLE IF EXISTS wv NO DELAY; +DROP TABLE IF EXISTS wv SYNC; CREATE WINDOW VIEW wv ENGINE Memory AS SELECT count(a), hopStart(wid) AS w_start FROM mt WHERE a != 1 GROUP BY hop(timestamp, INTERVAL '3' SECOND, INTERVAL '5' SECOND) AS wid; SELECT '---ORDER_BY---'; -DROP TABLE IF EXISTS wv NO DELAY; +DROP TABLE IF EXISTS wv SYNC; CREATE WINDOW VIEW wv ENGINE Memory AS SELECT count(a), hopStart(wid) AS w_start FROM mt WHERE a != 1 GROUP BY hop(timestamp, INTERVAL '3' SECOND, INTERVAL '5' SECOND) AS wid ORDER BY w_start; SELECT '---With now---'; -DROP TABLE IF EXISTS wv NO DELAY; +DROP TABLE IF EXISTS wv SYNC; CREATE WINDOW VIEW wv ENGINE Memory AS SELECT count(a), hopStart(wid) AS w_start, hopEnd(hop(now(), INTERVAL '1' SECOND, INTERVAL '3' SECOND)) as w_end FROM mt GROUP BY hop(now(), INTERVAL '1' SECOND, INTERVAL '3' SECOND) AS wid; diff --git a/tests/queries/0_stateless/01059_window_view_event_hop_watch_strict_asc.py b/tests/queries/0_stateless/01059_window_view_event_hop_watch_strict_asc.py index f9a465be2c8..9adff06442e 100755 --- a/tests/queries/0_stateless/01059_window_view_event_hop_watch_strict_asc.py +++ b/tests/queries/0_stateless/01059_window_view_event_hop_watch_strict_asc.py @@ -31,7 +31,7 @@ with client(name="client1>", log=log) as client1, client( client1.expect(prompt) client1.send("DROP TABLE IF EXISTS db_01059_event_hop_watch_strict_asc.mt") client1.expect(prompt) - client1.send("DROP TABLE IF EXISTS db_01059_event_hop_watch_strict_asc.wv NO DELAY") + client1.send("DROP TABLE IF EXISTS db_01059_event_hop_watch_strict_asc.wv SYNC") client1.expect(prompt) client1.send( @@ -71,7 +71,7 @@ with client(name="client1>", log=log) as client1, client( if match.groups()[1]: client1.send(client1.command) client1.expect(prompt) - client1.send("DROP TABLE db_01059_event_hop_watch_strict_asc.wv NO DELAY") + client1.send("DROP TABLE db_01059_event_hop_watch_strict_asc.wv SYNC") client1.expect(prompt) client1.send("DROP TABLE db_01059_event_hop_watch_strict_asc.mt") client1.expect(prompt) diff --git a/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py b/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py index 92298fa7ad6..bb40b1df2f0 100755 --- a/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py +++ b/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py @@ -33,9 +33,7 @@ with client(name="client1>", log=log) as client1, client( client1.expect(prompt) client1.send("DROP TABLE IF EXISTS 01062_window_view_event_hop_watch_asc.mt") client1.expect(prompt) - client1.send( - "DROP TABLE IF EXISTS 01062_window_view_event_hop_watch_asc.wv NO DELAY" - ) + client1.send("DROP TABLE IF EXISTS 01062_window_view_event_hop_watch_asc.wv SYNC") client1.expect(prompt) client1.send( @@ -77,7 +75,7 @@ with client(name="client1>", log=log) as client1, client( if match.groups()[1]: client1.send(client1.command) client1.expect(prompt) - client1.send("DROP TABLE 01062_window_view_event_hop_watch_asc.wv NO DELAY") + client1.send("DROP TABLE 01062_window_view_event_hop_watch_asc.wv SYNC") client1.expect(prompt) client1.send("DROP TABLE 01062_window_view_event_hop_watch_asc.mt") client1.expect(prompt) diff --git a/tests/queries/0_stateless/01066_bit_count.reference b/tests/queries/0_stateless/01066_bit_count.reference index 4a3b084b4a2..9a1a9a69216 100644 --- a/tests/queries/0_stateless/01066_bit_count.reference +++ b/tests/queries/0_stateless/01066_bit_count.reference @@ -19,3 +19,7 @@ 1 10 000000000000F03F -1 11 000000000000F0BF inf 11 000000000000F07F +Hello, world!!!! 55 +67 +67 +1 diff --git a/tests/queries/0_stateless/01066_bit_count.sql b/tests/queries/0_stateless/01066_bit_count.sql index d50b2657542..0b1b2dc8247 100644 --- a/tests/queries/0_stateless/01066_bit_count.sql +++ b/tests/queries/0_stateless/01066_bit_count.sql @@ -11,3 +11,9 @@ SELECT bitCount(toInt16(-1)); SELECT bitCount(toInt8(-1)); SELECT x, bitCount(x), hex(reinterpretAsString(x)) FROM VALUES ('x Float64', (1), (-1), (inf)); + +SELECT toFixedString('Hello, world!!!!', 16) AS x, bitCount(x); + +SELECT length(replaceAll(bin('clickhouse cloud'), '0', '')); +SELECT bitCount('clickhouse cloud'); +SELECT length(replaceAll(bin('clickhouse cloud'), '0', '')) = bitCount('clickhouse cloud'); diff --git a/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py b/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py index ff15f14cbc3..eb31b2ccbcf 100755 --- a/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py +++ b/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py @@ -33,7 +33,7 @@ with client(name="client1>", log=log) as client1, client( client1.expect(prompt) client1.send("DROP TABLE IF EXISTS 01069_window_view_proc_tumble_watch.mt") client1.expect(prompt) - client1.send("DROP TABLE IF EXISTS 01069_window_view_proc_tumble_watch.wv NO DELAY") + client1.send("DROP TABLE IF EXISTS 01069_window_view_proc_tumble_watch.wv SYNC") client1.expect(prompt) client1.send( @@ -67,7 +67,7 @@ with client(name="client1>", log=log) as client1, client( if match.groups()[1]: client1.send(client1.command) client1.expect(prompt) - client1.send("DROP TABLE 01069_window_view_proc_tumble_watch.wv NO DELAY") + client1.send("DROP TABLE 01069_window_view_proc_tumble_watch.wv SYNC") client1.expect(prompt) client1.send("DROP TABLE 01069_window_view_proc_tumble_watch.mt") client1.expect(prompt) diff --git a/tests/queries/0_stateless/01070_window_view_watch_events.py b/tests/queries/0_stateless/01070_window_view_watch_events.py index bf9f437b8ad..8aeff041cc1 100755 --- a/tests/queries/0_stateless/01070_window_view_watch_events.py +++ b/tests/queries/0_stateless/01070_window_view_watch_events.py @@ -31,9 +31,9 @@ with client(name="client1>", log=log) as client1, client( client1.send("CREATE DATABASE IF NOT EXISTS 01070_window_view_watch_events") client1.expect(prompt) - client1.send("DROP TABLE IF EXISTS 01070_window_view_watch_events.mt NO DELAY") + client1.send("DROP TABLE IF EXISTS 01070_window_view_watch_events.mt SYNC") client1.expect(prompt) - client1.send("DROP TABLE IF EXISTS 01070_window_view_watch_events.wv NO DELAY") + client1.send("DROP TABLE IF EXISTS 01070_window_view_watch_events.wv SYNC") client1.expect(prompt) client1.send( @@ -65,7 +65,7 @@ with client(name="client1>", log=log) as client1, client( if match.groups()[1]: client1.send(client1.command) client1.expect(prompt) - client1.send("DROP TABLE 01070_window_view_watch_events.wv NO DELAY;") + client1.send("DROP TABLE 01070_window_view_watch_events.wv SYNC;") client1.expect(prompt) client1.send("DROP TABLE 01070_window_view_watch_events.mt;") client1.expect(prompt) diff --git a/tests/queries/0_stateless/01078_window_view_alter_query_watch.py b/tests/queries/0_stateless/01078_window_view_alter_query_watch.py index 62a2a4bb4f3..c32e508c5a5 100755 --- a/tests/queries/0_stateless/01078_window_view_alter_query_watch.py +++ b/tests/queries/0_stateless/01078_window_view_alter_query_watch.py @@ -35,9 +35,9 @@ with client(name="client1>", log=log) as client1, client( client1.send("CREATE DATABASE IF NOT EXISTS 01078_window_view_alter_query_watch") client1.expect(prompt) - client1.send("DROP TABLE IF EXISTS 01078_window_view_alter_query_watch.mt NO DELAY") + client1.send("DROP TABLE IF EXISTS 01078_window_view_alter_query_watch.mt SYNC") client1.expect(prompt) - client1.send("DROP TABLE IF EXISTS 01078_window_view_alter_query_watch.wv NO DELAY") + client1.send("DROP TABLE IF EXISTS 01078_window_view_alter_query_watch.wv SYNC") client1.expect(prompt) client1.send( @@ -89,7 +89,7 @@ with client(name="client1>", log=log) as client1, client( if match.groups()[1]: client3.send(client3.command) client3.expect(prompt) - client3.send("DROP TABLE 01078_window_view_alter_query_watch.wv NO DELAY;") + client3.send("DROP TABLE 01078_window_view_alter_query_watch.wv SYNC;") client3.expect(prompt) client3.send("DROP TABLE 01078_window_view_alter_query_watch.mt;") client3.expect(prompt) diff --git a/tests/queries/0_stateless/01082_window_view_watch_limit.py b/tests/queries/0_stateless/01082_window_view_watch_limit.py index 7c2e855ef72..12c8d295591 100755 --- a/tests/queries/0_stateless/01082_window_view_watch_limit.py +++ b/tests/queries/0_stateless/01082_window_view_watch_limit.py @@ -32,7 +32,7 @@ with client(name="client1>", log=log) as client1, client( client1.expect(prompt) client1.send("DROP TABLE IF EXISTS 01082_window_view_watch_limit.mt") client1.expect(prompt) - client1.send("DROP TABLE IF EXISTS 01082_window_view_watch_limit.wv NO DELAY") + client1.send("DROP TABLE IF EXISTS 01082_window_view_watch_limit.wv SYNC") client1.expect(prompt) client1.send( @@ -61,7 +61,7 @@ with client(name="client1>", log=log) as client1, client( client1.expect("1 row" + end_of_block) client1.expect(prompt) - client1.send("DROP TABLE 01082_window_view_watch_limit.wv NO DELAY") + client1.send("DROP TABLE 01082_window_view_watch_limit.wv SYNC") client1.expect(prompt) client1.send("DROP TABLE 01082_window_view_watch_limit.mt") client1.expect(prompt) diff --git a/tests/queries/0_stateless/01085_window_view_attach.sql b/tests/queries/0_stateless/01085_window_view_attach.sql index 0e4f24dc465..051557a6a76 100644 --- a/tests/queries/0_stateless/01085_window_view_attach.sql +++ b/tests/queries/0_stateless/01085_window_view_attach.sql @@ -15,7 +15,7 @@ CREATE WINDOW VIEW test_01085.wv ENGINE Memory WATERMARK=ASCENDING AS SELECT cou SHOW tables FROM test_01085; -DROP TABLE test_01085.wv NO DELAY; +DROP TABLE test_01085.wv SYNC; SHOW tables FROM test_01085; CREATE WINDOW VIEW test_01085.wv ENGINE Memory WATERMARK=ASCENDING AS SELECT count(a) AS count, market, tumbleEnd(wid) AS w_end FROM test_01085.mt GROUP BY tumble(timestamp, INTERVAL '5' SECOND) AS wid, market; @@ -26,5 +26,5 @@ SHOW tables FROM test_01085; ATTACH TABLE test_01085.wv; SHOW tables FROM test_01085; -DROP TABLE test_01085.wv NO DELAY; +DROP TABLE test_01085.wv SYNC; SHOW tables FROM test_01085; diff --git a/tests/queries/0_stateless/01086_window_view_cleanup.sh b/tests/queries/0_stateless/01086_window_view_cleanup.sh index a25cacd45ec..b078b4718c0 100755 --- a/tests/queries/0_stateless/01086_window_view_cleanup.sh +++ b/tests/queries/0_stateless/01086_window_view_cleanup.sh @@ -40,7 +40,7 @@ while true; do done $CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT market, wid FROM test_01086.\`.inner.wv\` ORDER BY market, \`windowID(timestamp, toIntervalSecond('5'), 'US/Samoa')\` as wid"; -$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP TABLE test_01086.wv NO DELAY;" -$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP TABLE test_01086.mt NO DELAY;" -$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP TABLE test_01086.dst NO DELAY;" -$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP DATABASE test_01086 NO DELAY;" +$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP TABLE test_01086.wv SYNC;" +$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP TABLE test_01086.mt SYNC;" +$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP TABLE test_01086.dst SYNC;" +$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP DATABASE test_01086 SYNC;" diff --git a/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.reference b/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.reference index 01efbb7c64b..6d32c20909b 100644 --- a/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.reference +++ b/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.reference @@ -3,15 +3,27 @@ 1 l \N Nullable(String) 2 \N Nullable(String) - +1 l Nullable(String) \N Nullable(String) +0 \N Nullable(String) \N Nullable(String) +0 \N Nullable(String) \N Nullable(String) +1 l Nullable(String) \N Nullable(String) +- +1 l LowCardinality(String) \N Nullable(String) +0 LowCardinality(String) \N Nullable(String) +0 LowCardinality(String) \N Nullable(String) +1 l LowCardinality(String) \N Nullable(String) +- +1 l \N Nullable(String) +0 \N \N Nullable(String) +0 \N \N Nullable(String) +1 l \N Nullable(String) +- 1 l \N Nullable(String) 0 \N Nullable(String) 0 \N Nullable(String) 1 l \N Nullable(String) - -1 l \N Nullable(String) -0 \N Nullable(String) -0 \N Nullable(String) -1 l \N Nullable(String) +0 \N - 0 - diff --git a/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.sql b/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.sql index 38b72837174..2464b7a57cf 100644 --- a/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.sql +++ b/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.sql @@ -15,19 +15,37 @@ SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (x) ORD SELECT '-'; -SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x; -SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x; -SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x; +-- lc should be supertype for l.lc and r.lc, so expect Nullable(String) +SELECT x, lc, toTypeName(lc), r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1; +SELECT x, lc, toTypeName(lc), r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1; +SELECT x, lc, toTypeName(lc), r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1; SELECT '-'; -SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x; -SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x; -SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x; +-- old behavior is different +SELECT x, lc, toTypeName(lc), r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0; +SELECT x, lc, toTypeName(lc), r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0; +SELECT x, lc, toTypeName(lc), r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0; SELECT '-'; -SELECT x, lc FROM t AS l RIGHT JOIN nr AS r USING (lc); +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1; +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1; +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1; + +SELECT '-'; + +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0; +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0; +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0; + +SELECT '-'; + +SELECT x, lc FROM t AS l RIGHT JOIN nr AS r USING (lc) SETTINGS allow_experimental_analyzer = 1; + +SELECT '-'; + +SELECT x, lc FROM t AS l RIGHT JOIN nr AS r USING (lc) SETTINGS allow_experimental_analyzer = 0; SELECT '-'; diff --git a/tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.reference b/tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.reference index c6bdcb773b2..bb29ec9becd 100644 --- a/tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.reference +++ b/tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.reference @@ -4,6 +4,16 @@ 2 \N Nullable(String) - 1 l \N Nullable(String) +0 \N \N Nullable(String) +0 \N \N Nullable(String) +1 l \N Nullable(String) +- +1 l \N Nullable(String) +0 \N \N Nullable(String) +0 \N \N Nullable(String) +1 l \N Nullable(String) +- +1 l \N Nullable(String) 0 \N Nullable(String) 0 \N Nullable(String) 1 l \N Nullable(String) diff --git a/tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.sql b/tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.sql index dbc2d7c9f5d..718e8358c64 100644 --- a/tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.sql +++ b/tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.sql @@ -17,15 +17,27 @@ SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (x) ORD SELECT '-'; -SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x; -SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x; -SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1; SELECT '-'; -SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x; -SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x; -SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x; +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1; +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1; +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1; + +SELECT '-'; + +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0; +SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0; + +SELECT '-'; + +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0; +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0; +SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0; SELECT '-'; diff --git a/tests/queries/0_stateless/01158_zookeeper_log_long.reference b/tests/queries/0_stateless/01158_zookeeper_log_long.reference index a0088610c9d..7ec52cb3366 100644 --- a/tests/queries/0_stateless/01158_zookeeper_log_long.reference +++ b/tests/queries/0_stateless/01158_zookeeper_log_long.reference @@ -18,22 +18,18 @@ Response 0 Create /test/01158/default/rmt/replicas/1/parts/all_0_0_0 0 0 \N 0 4 Request 0 Exists /test/01158/default/rmt/replicas/1/parts/all_0_0_0 0 0 \N 0 0 \N \N \N 0 0 0 0 Response 0 Exists /test/01158/default/rmt/replicas/1/parts/all_0_0_0 0 0 \N 0 0 ZOK \N \N 0 0 96 0 blocks -Request 0 Multi 0 0 \N 3 0 \N \N \N 0 0 0 0 -Request 0 Create /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 \N 0 1 \N \N \N 0 0 0 0 -Request 0 Remove /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 -1 0 2 \N \N \N 0 0 0 0 -Request 0 Create /test/01158/default/rmt/block_numbers/all/block- 1 1 \N 0 3 \N \N \N 0 0 0 0 -Response 0 Multi 0 0 \N 3 0 ZOK \N \N 0 0 0 0 -Response 0 Create /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 \N 0 1 ZOK \N \N /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 0 0 -Response 0 Remove /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 -1 0 2 ZOK \N \N 0 0 0 0 -Response 0 Create /test/01158/default/rmt/block_numbers/all/block- 1 1 \N 0 3 ZOK \N \N /test/01158/default/rmt/block_numbers/all/block-0000000000 0 0 0 0 -Request 0 Multi 0 0 \N 3 0 \N \N \N 0 0 0 0 -Request 0 Create /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 \N 0 1 \N \N \N 0 0 0 0 -Request 0 Remove /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 -1 0 2 \N \N \N 0 0 0 0 -Request 0 Create /test/01158/default/rmt/block_numbers/all/block- 1 1 \N 0 3 \N \N \N 0 0 0 0 -Response 0 Multi 0 0 \N 3 0 ZNODEEXISTS \N \N 0 0 0 0 -Response 0 Error /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 \N 0 1 ZNODEEXISTS \N \N 0 0 0 0 -Response 0 Error /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 -1 0 2 ZRUNTIMEINCONSISTENCY \N \N 0 0 0 0 -Response 0 Error /test/01158/default/rmt/block_numbers/all/block- 1 1 \N 0 3 ZRUNTIMEINCONSISTENCY \N \N 0 0 0 0 +Request 0 Multi 0 0 \N 2 0 \N \N \N 0 0 0 0 +Request 0 CheckNotExists /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 -1 0 1 \N \N \N 0 0 0 0 +Request 0 Create /test/01158/default/rmt/block_numbers/all/block- 1 1 \N 0 2 \N \N \N 0 0 0 0 +Response 0 Multi 0 0 \N 2 0 ZOK \N \N 0 0 0 0 +Response 0 CheckNotExists /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 -1 0 1 ZOK \N \N 0 0 0 0 +Response 0 Create /test/01158/default/rmt/block_numbers/all/block- 1 1 \N 0 2 ZOK \N \N /test/01158/default/rmt/block_numbers/all/block-0000000000 0 0 0 0 +Request 0 Multi 0 0 \N 2 0 \N \N \N 0 0 0 0 +Request 0 CheckNotExists /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 -1 0 1 \N \N \N 0 0 0 0 +Request 0 Create /test/01158/default/rmt/block_numbers/all/block- 1 1 \N 0 2 \N \N \N 0 0 0 0 +Response 0 Multi 0 0 \N 2 0 ZNODEEXISTS \N \N 0 0 0 0 +Response 0 Error /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 -1 0 1 ZNODEEXISTS \N \N 0 0 0 0 +Response 0 Error /test/01158/default/rmt/block_numbers/all/block- 1 1 \N 0 2 ZRUNTIMEINCONSISTENCY \N \N 0 0 0 0 Request 0 Get /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 \N 0 0 \N \N \N 0 0 0 0 Response 0 Get /test/01158/default/rmt/blocks/all_6308706741995381342_2495791770474910886 0 0 \N 0 0 ZOK \N \N 0 0 9 0 duration_ms diff --git a/tests/queries/0_stateless/01181_db_atomic_drop_on_cluster.sql b/tests/queries/0_stateless/01181_db_atomic_drop_on_cluster.sql index fbb67a268ae..6edaaa5c602 100644 --- a/tests/queries/0_stateless/01181_db_atomic_drop_on_cluster.sql +++ b/tests/queries/0_stateless/01181_db_atomic_drop_on_cluster.sql @@ -1,8 +1,8 @@ -- Tags: no-replicated-database -- Tag no-replicated-database: ON CLUSTER is not allowed -DROP TABLE IF EXISTS test_repl ON CLUSTER test_shard_localhost SYNC; +DROP TABLE IF EXISTS test_repl ON CLUSTER test_shard_localhost NO DELAY; CREATE TABLE test_repl ON CLUSTER test_shard_localhost (n UInt64) ENGINE ReplicatedMergeTree('/clickhouse/test_01181/{database}/test_repl','r1') ORDER BY tuple(); -DETACH TABLE test_repl ON CLUSTER test_shard_localhost SYNC; +DETACH TABLE test_repl ON CLUSTER test_shard_localhost NO DELAY; ATTACH TABLE test_repl ON CLUSTER test_shard_localhost; -DROP TABLE test_repl ON CLUSTER test_shard_localhost SYNC; +DROP TABLE test_repl ON CLUSTER test_shard_localhost NO DELAY; diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index 553fee1f435..41c6a7bd709 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -138,6 +138,7 @@ SYSTEM THREAD FUZZER ['SYSTEM START THREAD FUZZER','SYSTEM STOP THREAD FUZZER',' SYSTEM UNFREEZE ['SYSTEM UNFREEZE'] GLOBAL SYSTEM SYSTEM [] \N ALL dictGet ['dictHas','dictGetHierarchy','dictIsIn'] DICTIONARY ALL +displaySecretsInShowAndSelect [] GLOBAL ALL addressToLine [] GLOBAL INTROSPECTION addressToLineWithInlines [] GLOBAL INTROSPECTION addressToSymbol [] GLOBAL INTROSPECTION diff --git a/tests/queries/0_stateless/01292_create_user.reference b/tests/queries/0_stateless/01292_create_user.reference index f723412c636..eb89a5ed38c 100644 --- a/tests/queries/0_stateless/01292_create_user.reference +++ b/tests/queries/0_stateless/01292_create_user.reference @@ -13,6 +13,8 @@ CREATE USER u4_01292 IDENTIFIED WITH sha256_password CREATE USER u5_01292 IDENTIFIED WITH sha256_password CREATE USER u6_01292 IDENTIFIED WITH double_sha1_password CREATE USER u7_01292 IDENTIFIED WITH double_sha1_password +CREATE USER u8_01292 IDENTIFIED WITH bcrypt_password +CREATE USER u9_01292 IDENTIFIED WITH bcrypt_password CREATE USER u1_01292 IDENTIFIED WITH sha256_password CREATE USER u2_01292 IDENTIFIED WITH sha256_password CREATE USER u3_01292 IDENTIFIED WITH sha256_password diff --git a/tests/queries/0_stateless/01292_create_user.sql b/tests/queries/0_stateless/01292_create_user.sql index d0f157d36b0..a283ce687e6 100644 --- a/tests/queries/0_stateless/01292_create_user.sql +++ b/tests/queries/0_stateless/01292_create_user.sql @@ -1,4 +1,4 @@ --- Tags: no-fasttest +-- Tags: no-fasttest, no-parallel DROP USER IF EXISTS u1_01292, u2_01292, u3_01292, u4_01292, u5_01292, u6_01292, u7_01292, u8_01292, u9_01292; DROP USER IF EXISTS u10_01292, u11_01292, u12_01292, u13_01292, u14_01292, u15_01292, u16_01292; @@ -31,6 +31,8 @@ CREATE USER u4_01292 IDENTIFIED WITH sha256_password BY 'qwe123'; CREATE USER u5_01292 IDENTIFIED WITH sha256_hash BY '18138372FAD4B94533CD4881F03DC6C69296DD897234E0CEE83F727E2E6B1F63'; CREATE USER u6_01292 IDENTIFIED WITH double_sha1_password BY 'qwe123'; CREATE USER u7_01292 IDENTIFIED WITH double_sha1_hash BY '8DCDD69CE7D121DE8013062AEAEB2A148910D50E'; +CREATE USER u8_01292 IDENTIFIED WITH bcrypt_password BY 'qwe123'; +CREATE USER u9_01292 IDENTIFIED WITH bcrypt_hash BY '$2a$12$rz5iy2LhuwBezsM88ZzWiemOVUeJ94xHTzwAlLMDhTzwUxOHaY64q'; SHOW CREATE USER u1_01292; SHOW CREATE USER u2_01292; SHOW CREATE USER u3_01292; @@ -38,6 +40,8 @@ SHOW CREATE USER u4_01292; SHOW CREATE USER u5_01292; SHOW CREATE USER u6_01292; SHOW CREATE USER u7_01292; +SHOW CREATE USER u8_01292; +SHOW CREATE USER u9_01292; ALTER USER u1_01292 IDENTIFIED BY '123qwe'; ALTER USER u2_01292 IDENTIFIED BY '123qwe'; ALTER USER u3_01292 IDENTIFIED BY '123qwe'; @@ -48,7 +52,7 @@ SHOW CREATE USER u2_01292; SHOW CREATE USER u3_01292; SHOW CREATE USER u4_01292; SHOW CREATE USER u5_01292; -DROP USER u1_01292, u2_01292, u3_01292, u4_01292, u5_01292, u6_01292, u7_01292; +DROP USER u1_01292, u2_01292, u3_01292, u4_01292, u5_01292, u6_01292, u7_01292, u8_01292, u9_01292; SELECT '-- host'; CREATE USER u1_01292 HOST ANY; diff --git a/tests/queries/0_stateless/01346_alter_enum_partition_key_replicated_zookeeper_long.reference b/tests/queries/0_stateless/01346_alter_enum_partition_key_replicated_zookeeper_long.reference index 60c6076aef0..a905ea97ae5 100644 --- a/tests/queries/0_stateless/01346_alter_enum_partition_key_replicated_zookeeper_long.reference +++ b/tests/queries/0_stateless/01346_alter_enum_partition_key_replicated_zookeeper_long.reference @@ -1,24 +1,24 @@ hello test hello test -1_0_0_0 hello 1 -1_0_0_0 hello 1 +0 0 hello 1 +0 0 hello 1 hello test goodbye test hello test goodbye test -3_0_0_1 goodbye 3 -1_0_0_1 hello 1 -3_0_0_1 goodbye 3 -1_0_0_1 hello 1 +0 0 goodbye 3 +0 0 hello 1 +0 0 goodbye 3 +0 0 hello 1 1 test 3 test 111 abc 1 test 3 test 111 abc -1_0_0_2 1 1 -111_0_0_1 111 111 -3_0_0_2 3 3 -1_0_0_2 1 1 -111_0_0_1 111 111 -3_0_0_2 3 3 +0 0 1 1 +0 0 111 111 +0 0 3 3 +0 0 1 1 +0 0 111 111 +0 0 3 3 diff --git a/tests/queries/0_stateless/01346_alter_enum_partition_key_replicated_zookeeper_long.sql b/tests/queries/0_stateless/01346_alter_enum_partition_key_replicated_zookeeper_long.sql index f20156fd9e3..d40bcc15e55 100644 --- a/tests/queries/0_stateless/01346_alter_enum_partition_key_replicated_zookeeper_long.sql +++ b/tests/queries/0_stateless/01346_alter_enum_partition_key_replicated_zookeeper_long.sql @@ -13,16 +13,17 @@ INSERT INTO test VALUES ('hello', 'test'); SELECT * FROM test; SYSTEM SYNC REPLICA test2; SELECT * FROM test2; -SELECT name, partition, partition_id FROM system.parts WHERE database = currentDatabase() AND table = 'test' AND active ORDER BY partition; -SELECT name, partition, partition_id FROM system.parts WHERE database = currentDatabase() AND table = 'test2' AND active ORDER BY partition; +SELECT min_block_number, max_block_number, partition, partition_id FROM system.parts WHERE database = currentDatabase() AND table = 'test' AND active ORDER BY partition; +SELECT min_block_number, max_block_number, partition, partition_id FROM system.parts WHERE database = currentDatabase() AND table = 'test2' AND active ORDER BY partition; ALTER TABLE test MODIFY COLUMN x Enum('hello' = 1, 'world' = 2, 'goodbye' = 3); INSERT INTO test VALUES ('goodbye', 'test'); OPTIMIZE TABLE test FINAL; SELECT * FROM test ORDER BY x; +SYSTEM SYNC REPLICA test2; SELECT * FROM test2 ORDER BY x; -SELECT name, partition, partition_id FROM system.parts WHERE database = currentDatabase() AND table = 'test' AND active ORDER BY partition; -SELECT name, partition, partition_id FROM system.parts WHERE database = currentDatabase() AND table = 'test2' AND active ORDER BY partition; +SELECT min_block_number, max_block_number, partition, partition_id FROM system.parts WHERE database = currentDatabase() AND table = 'test' AND active ORDER BY partition; +SELECT min_block_number, max_block_number, partition, partition_id FROM system.parts WHERE database = currentDatabase() AND table = 'test2' AND active ORDER BY partition; ALTER TABLE test MODIFY COLUMN x Enum('hello' = 1, 'world' = 2); -- { serverError 524 } ALTER TABLE test MODIFY COLUMN x Enum('hello' = 1, 'world' = 2, 'test' = 3); @@ -33,9 +34,10 @@ ALTER TABLE test MODIFY COLUMN x Int8; INSERT INTO test VALUES (111, 'abc'); OPTIMIZE TABLE test FINAL; SELECT * FROM test ORDER BY x; +SYSTEM SYNC REPLICA test2; SELECT * FROM test2 ORDER BY x; -SELECT name, partition, partition_id FROM system.parts WHERE database = currentDatabase() AND table = 'test' AND active ORDER BY partition; -SELECT name, partition, partition_id FROM system.parts WHERE database = currentDatabase() AND table = 'test2' AND active ORDER BY partition; +SELECT min_block_number, max_block_number, partition, partition_id FROM system.parts WHERE database = currentDatabase() AND table = 'test' AND active ORDER BY partition; +SELECT min_block_number, max_block_number, partition, partition_id FROM system.parts WHERE database = currentDatabase() AND table = 'test2' AND active ORDER BY partition; ALTER TABLE test MODIFY COLUMN x Enum8('' = 1); -- { serverError 524 } ALTER TABLE test MODIFY COLUMN x Enum16('' = 1); -- { serverError 524 } diff --git a/tests/queries/0_stateless/01442_merge_detach_attach_long.sh b/tests/queries/0_stateless/01442_merge_detach_attach_long.sh index b3f9fbb42dd..c080dded1c8 100755 --- a/tests/queries/0_stateless/01442_merge_detach_attach_long.sh +++ b/tests/queries/0_stateless/01442_merge_detach_attach_long.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long, no-parallel +# Tags: long, no-parallel, no-debug set -e diff --git a/tests/queries/0_stateless/01476_right_full_join_switch.reference b/tests/queries/0_stateless/01476_right_full_join_switch.reference index 1f839b86013..54f9909762f 100644 --- a/tests/queries/0_stateless/01476_right_full_join_switch.reference +++ b/tests/queries/0_stateless/01476_right_full_join_switch.reference @@ -3,6 +3,16 @@ 1 l \N LowCardinality(String) Nullable(String) 2 \N LowCardinality(String) Nullable(String) - +\N \N Nullable(String) LowCardinality(String) +1 \N l Nullable(String) LowCardinality(String) +1 \N l Nullable(String) LowCardinality(String) +\N \N Nullable(String) LowCardinality(String) +- +1 l \N LowCardinality(String) Nullable(String) +2 \N LowCardinality(String) Nullable(String) +1 l \N LowCardinality(String) Nullable(String) +2 \N LowCardinality(String) Nullable(String) +- 0 \N Nullable(String) LowCardinality(String) 1 \N l Nullable(String) LowCardinality(String) 0 \N Nullable(String) LowCardinality(String) diff --git a/tests/queries/0_stateless/01476_right_full_join_switch.sql b/tests/queries/0_stateless/01476_right_full_join_switch.sql index 5d041843ee2..dfbdec47e1f 100644 --- a/tests/queries/0_stateless/01476_right_full_join_switch.sql +++ b/tests/queries/0_stateless/01476_right_full_join_switch.sql @@ -10,8 +10,27 @@ CREATE TABLE nr (`x` Nullable(UInt32), `s` Nullable(String)) ENGINE = Memory; INSERT INTO t VALUES (1, 'l'); INSERT INTO nr VALUES (2, NULL); + SET join_use_nulls = 0; +SET allow_experimental_analyzer = 1; + +-- t.x is supertupe for `x` from left and right since `x` is inside `USING`. +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l LEFT JOIN nr AS r USING (x) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l RIGHT JOIN nr AS r USING (x) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l FULL JOIN nr AS r USING (x) ORDER BY t.x; + +SELECT '-'; + +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l LEFT JOIN t AS r USING (x) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l RIGHT JOIN t AS r USING (x) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l FULL JOIN t AS r USING (x) ORDER BY t.x; + +SELECT '-'; + +SET allow_experimental_analyzer = 0; + +-- t.x is supertupe for `x` from left and right since `x` is inside `USING`. SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l LEFT JOIN nr AS r USING (x) ORDER BY t.x; SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l RIGHT JOIN nr AS r USING (x) ORDER BY t.x; SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l FULL JOIN nr AS r USING (x) ORDER BY t.x; diff --git a/tests/queries/0_stateless/01477_lc_in_merge_join_left_key.reference b/tests/queries/0_stateless/01477_lc_in_merge_join_left_key.reference index ac4d0a3d21a..9b6890c01ee 100644 --- a/tests/queries/0_stateless/01477_lc_in_merge_join_left_key.reference +++ b/tests/queries/0_stateless/01477_lc_in_merge_join_left_key.reference @@ -17,7 +17,7 @@ 1 \N l Nullable(String) LowCardinality(String) 0 \N Nullable(String) LowCardinality(String) 1 \N l Nullable(String) LowCardinality(String) -- +- join_use_nulls - 1 l \N LowCardinality(String) Nullable(String) 2 \N \N LowCardinality(Nullable(String)) Nullable(String) 1 l \N LowCardinality(Nullable(String)) Nullable(String) @@ -33,3 +33,47 @@ 1 l \N LowCardinality(Nullable(String)) Nullable(String) \N \N \N LowCardinality(Nullable(String)) Nullable(String) - +\N \N \N Nullable(String) LowCardinality(Nullable(String)) +1 \N l Nullable(String) LowCardinality(String) +1 \N l Nullable(String) LowCardinality(Nullable(String)) +\N \N \N Nullable(String) LowCardinality(Nullable(String)) +- analyzer - +1 l \N LowCardinality(String) Nullable(String) +2 \N LowCardinality(String) Nullable(String) +1 l \N LowCardinality(String) Nullable(String) +2 \N LowCardinality(String) Nullable(String) +- +\N \N Nullable(String) LowCardinality(String) +1 \N l Nullable(String) LowCardinality(String) +1 \N l Nullable(String) LowCardinality(String) +\N \N Nullable(String) LowCardinality(String) +- +1 l \N Nullable(String) Nullable(String) +0 \N \N Nullable(String) Nullable(String) +0 \N \N Nullable(String) Nullable(String) +1 l \N Nullable(String) Nullable(String) +- +0 \N \N Nullable(String) Nullable(String) +1 \N l Nullable(String) Nullable(String) +0 \N \N Nullable(String) Nullable(String) +1 \N l Nullable(String) Nullable(String) +- join_use_nulls - +1 l \N LowCardinality(String) Nullable(String) +2 \N \N LowCardinality(Nullable(String)) Nullable(String) +1 l \N LowCardinality(Nullable(String)) Nullable(String) +2 \N \N LowCardinality(Nullable(String)) Nullable(String) +- +\N \N \N Nullable(String) LowCardinality(Nullable(String)) +1 \N l Nullable(String) LowCardinality(String) +1 \N l Nullable(String) LowCardinality(Nullable(String)) +\N \N \N Nullable(String) LowCardinality(Nullable(String)) +- +1 l \N Nullable(String) Nullable(String) +\N \N \N Nullable(String) Nullable(String) +1 l \N Nullable(String) Nullable(String) +\N \N \N Nullable(String) Nullable(String) +- +\N \N \N Nullable(String) Nullable(String) +1 \N l Nullable(String) Nullable(String) +1 \N l Nullable(String) Nullable(String) +\N \N \N Nullable(String) Nullable(String) diff --git a/tests/queries/0_stateless/01477_lc_in_merge_join_left_key.sql b/tests/queries/0_stateless/01477_lc_in_merge_join_left_key.sql.j2 similarity index 83% rename from tests/queries/0_stateless/01477_lc_in_merge_join_left_key.sql rename to tests/queries/0_stateless/01477_lc_in_merge_join_left_key.sql.j2 index 2507613f051..6eafd41b411 100644 --- a/tests/queries/0_stateless/01477_lc_in_merge_join_left_key.sql +++ b/tests/queries/0_stateless/01477_lc_in_merge_join_left_key.sql.j2 @@ -10,6 +10,14 @@ CREATE TABLE nr (`x` Nullable(UInt32), `s` Nullable(String)) ENGINE = Memory; INSERT INTO t VALUES (1, 'l'); INSERT INTO nr VALUES (2, NULL); +{% for allow_experimental_analyzer in [0, 1] -%} + +SET allow_experimental_analyzer = {{ allow_experimental_analyzer }}; + +{% if allow_experimental_analyzer -%} +SELECT '- analyzer -'; +{% endif -%} + SET join_use_nulls = 0; SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l LEFT JOIN nr AS r USING (x) ORDER BY t.x; @@ -36,7 +44,7 @@ SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l FULL JOIN t SET join_use_nulls = 1; -SELECT '-'; +SELECT '- join_use_nulls -'; SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l LEFT JOIN nr AS r USING (x) ORDER BY t.x; SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l RIGHT JOIN nr AS r USING (x) ORDER BY t.x; @@ -56,10 +64,11 @@ SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l FULL JOIN nr SELECT '-'; --- TODO --- SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l LEFT JOIN t AS r USING (s) ORDER BY t.x; --- SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l RIGHT JOIN t AS r USING (s) ORDER BY t.x; --- SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l FULL JOIN t AS r USING (s) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l LEFT JOIN t AS r USING (s) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l RIGHT JOIN t AS r USING (s) ORDER BY t.x; +SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l FULL JOIN t AS r USING (s) ORDER BY t.x; + +{% endfor %} DROP TABLE t; DROP TABLE nr; diff --git a/tests/queries/0_stateless/01532_execute_merges_on_single_replica_long.sql b/tests/queries/0_stateless/01532_execute_merges_on_single_replica_long.sql index f217b6094b2..4b907d5ebb6 100644 --- a/tests/queries/0_stateless/01532_execute_merges_on_single_replica_long.sql +++ b/tests/queries/0_stateless/01532_execute_merges_on_single_replica_long.sql @@ -4,8 +4,8 @@ SET insert_keeper_fault_injection_probability=0; -- disable fault injection; part ids are non-deterministic in case of insert retries -DROP TABLE IF EXISTS execute_on_single_replica_r1 NO DELAY; -DROP TABLE IF EXISTS execute_on_single_replica_r2 NO DELAY; +DROP TABLE IF EXISTS execute_on_single_replica_r1 SYNC; +DROP TABLE IF EXISTS execute_on_single_replica_r2 SYNC; /* that test requires fixed zookeeper path, so we cannot use ReplicatedMergeTree({database}) */ CREATE TABLE execute_on_single_replica_r1 (x UInt64) ENGINE=ReplicatedMergeTree('/clickhouse/tables/test_01532/execute_on_single_replica', 'r1') ORDER BY tuple() SETTINGS execute_merges_on_single_replica_time_threshold=10; @@ -130,5 +130,5 @@ GROUP BY part_name ORDER BY part_name FORMAT Vertical; -DROP TABLE execute_on_single_replica_r1 NO DELAY; -DROP TABLE execute_on_single_replica_r2 NO DELAY; +DROP TABLE execute_on_single_replica_r1 SYNC; +DROP TABLE execute_on_single_replica_r2 SYNC; diff --git a/tests/queries/0_stateless/01600_parts_states_metrics_long.sh b/tests/queries/0_stateless/01600_parts_states_metrics_long.sh index f47d0863e69..89ce84f6dbc 100755 --- a/tests/queries/0_stateless/01600_parts_states_metrics_long.sh +++ b/tests/queries/0_stateless/01600_parts_states_metrics_long.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash -# Tags: long CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -8,7 +7,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # NOTE: database = $CLICKHOUSE_DATABASE is unwanted verify_sql="SELECT (SELECT sumIf(value, metric = 'PartsActive'), sumIf(value, metric = 'PartsOutdated') FROM system.metrics) - = (SELECT sum(active), sum(NOT active) FROM system.parts)" + = (SELECT sum(active), sum(NOT active) FROM + (SELECT active FROM system.parts UNION ALL SELECT active FROM system.projection_parts))" # The query is not atomic - it can compare states between system.parts and system.metrics from different points in time. # So, there is inherent race condition. But it should get expected result eventually. diff --git a/tests/queries/0_stateless/01600_parts_types_metrics_long.sh b/tests/queries/0_stateless/01600_parts_types_metrics_long.sh index 05edf02f7ed..0b9afcf633e 100755 --- a/tests/queries/0_stateless/01600_parts_types_metrics_long.sh +++ b/tests/queries/0_stateless/01600_parts_types_metrics_long.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long, no-s3-storage +# Tags: no-s3-storage CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -11,7 +11,8 @@ set -o pipefail # NOTE: database = $CLICKHOUSE_DATABASE is unwanted verify_sql="SELECT (SELECT sumIf(value, metric = 'PartsInMemory'), sumIf(value, metric = 'PartsCompact'), sumIf(value, metric = 'PartsWide') FROM system.metrics) = - (SELECT countIf(part_type == 'InMemory'), countIf(part_type == 'Compact'), countIf(part_type == 'Wide') FROM system.parts)" + (SELECT countIf(part_type == 'InMemory'), countIf(part_type == 'Compact'), countIf(part_type == 'Wide') + FROM (SELECT part_type FROM system.parts UNION ALL SELECT part_type FROM system.projection_parts))" # The query is not atomic - it can compare states between system.parts and system.metrics from different points in time. # So, there is inherent race condition (especially in fasttest that runs tests in parallel). diff --git a/tests/queries/0_stateless/01686_rocksdb.sql b/tests/queries/0_stateless/01686_rocksdb.sql index ad6f56772b0..f3177ce140e 100644 --- a/tests/queries/0_stateless/01686_rocksdb.sql +++ b/tests/queries/0_stateless/01686_rocksdb.sql @@ -24,7 +24,7 @@ SELECT * FROM 01686_test WHERE key IN (123, 456, -123) ORDER BY key; SELECT '--'; SELECT * FROM 01686_test WHERE key = 'Hello'; -- { serverError 53 } -DETACH TABLE 01686_test NO DELAY; +DETACH TABLE 01686_test SYNC; ATTACH TABLE 01686_test; SELECT * FROM 01686_test WHERE key IN (99, 999, 9999, -123) ORDER BY key; diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index 5ac1ff501e5..cb8e19ea2a0 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -37,6 +37,20 @@ select JSON_VALUE('{"hello":{"world":"!"}}', '$.hello') settings function_json_v {"world":"!"} SELECT JSON_VALUE('{"hello":["world","world2"]}', '$.hello') settings function_json_value_return_type_allow_complex=true; ["world","world2"] +SELECT JSON_VALUE('{"1key":1}', '$.1key'); +1 +SELECT JSON_VALUE('{"hello":1}', '$[hello]'); +1 +SELECT JSON_VALUE('{"hello":1}', '$["hello"]'); +1 +SELECT JSON_VALUE('{"hello":1}', '$[\'hello\']'); +1 +SELECT JSON_VALUE('{"hello 1":1}', '$["hello 1"]'); +1 +SELECT JSON_VALUE('{"1key":1}', '$..1key'); -- { serverError 36 } +SELECT JSON_VALUE('{"1key":1}', '$1key'); -- { serverError 36 } +SELECT JSON_VALUE('{"1key":1}', '$key'); -- { serverError 36 } +SELECT JSON_VALUE('{"1key":1}', '$.[key]'); -- { serverError 36 } SELECT '--JSON_QUERY--'; --JSON_QUERY-- SELECT JSON_QUERY('{"hello":1}', '$'); @@ -61,6 +75,20 @@ SELECT JSON_QUERY('', '$.hello'); SELECT JSON_QUERY('{"array":[[0, 1, 2, 3, 4, 5], [0, -1, -2, -3, -4, -5]]}', '$.array[*][0 to 2, 4]'); [0, 1, 4, 0, -1, -4] +SELECT JSON_QUERY('{"1key":1}', '$.1key'); +[1] +SELECT JSON_QUERY('{"hello":1}', '$[hello]'); +[1] +SELECT JSON_QUERY('{"hello":1}', '$["hello"]'); +[1] +SELECT JSON_QUERY('{"hello":1}', '$[\'hello\']'); +[1] +SELECT JSON_QUERY('{"hello 1":1}', '$["hello 1"]'); +[1] +SELECT JSON_QUERY('{"1key":1}', '$..1key'); -- { serverError 36 } +SELECT JSON_QUERY('{"1key":1}', '$1key'); -- { serverError 36 } +SELECT JSON_QUERY('{"1key":1}', '$key'); -- { serverError 36 } +SELECT JSON_QUERY('{"1key":1}', '$.[key]'); -- { serverError 36 } SELECT '--JSON_EXISTS--'; --JSON_EXISTS-- SELECT JSON_EXISTS('{"hello":1}', '$'); diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql index f174d04933c..947b0171ec6 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.sql +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -20,6 +20,15 @@ select JSON_VALUE('{"a":"\\u263a"}', '$.a'); select JSON_VALUE('{"hello":"world"}', '$.b') settings function_json_value_return_type_allow_nullable=true; select JSON_VALUE('{"hello":{"world":"!"}}', '$.hello') settings function_json_value_return_type_allow_complex=true; SELECT JSON_VALUE('{"hello":["world","world2"]}', '$.hello') settings function_json_value_return_type_allow_complex=true; +SELECT JSON_VALUE('{"1key":1}', '$.1key'); +SELECT JSON_VALUE('{"hello":1}', '$[hello]'); +SELECT JSON_VALUE('{"hello":1}', '$["hello"]'); +SELECT JSON_VALUE('{"hello":1}', '$[\'hello\']'); +SELECT JSON_VALUE('{"hello 1":1}', '$["hello 1"]'); +SELECT JSON_VALUE('{"1key":1}', '$..1key'); -- { serverError 36 } +SELECT JSON_VALUE('{"1key":1}', '$1key'); -- { serverError 36 } +SELECT JSON_VALUE('{"1key":1}', '$key'); -- { serverError 36 } +SELECT JSON_VALUE('{"1key":1}', '$.[key]'); -- { serverError 36 } SELECT '--JSON_QUERY--'; SELECT JSON_QUERY('{"hello":1}', '$'); @@ -33,6 +42,15 @@ SELECT JSON_QUERY('{"hello":{"world":"!"}}', '$.hello'); SELECT JSON_QUERY( '{hello:{"world":"!"}}}', '$.hello'); -- invalid json => default value (empty string) SELECT JSON_QUERY('', '$.hello'); SELECT JSON_QUERY('{"array":[[0, 1, 2, 3, 4, 5], [0, -1, -2, -3, -4, -5]]}', '$.array[*][0 to 2, 4]'); +SELECT JSON_QUERY('{"1key":1}', '$.1key'); +SELECT JSON_QUERY('{"hello":1}', '$[hello]'); +SELECT JSON_QUERY('{"hello":1}', '$["hello"]'); +SELECT JSON_QUERY('{"hello":1}', '$[\'hello\']'); +SELECT JSON_QUERY('{"hello 1":1}', '$["hello 1"]'); +SELECT JSON_QUERY('{"1key":1}', '$..1key'); -- { serverError 36 } +SELECT JSON_QUERY('{"1key":1}', '$1key'); -- { serverError 36 } +SELECT JSON_QUERY('{"1key":1}', '$key'); -- { serverError 36 } +SELECT JSON_QUERY('{"1key":1}', '$.[key]'); -- { serverError 36 } SELECT '--JSON_EXISTS--'; SELECT JSON_EXISTS('{"hello":1}', '$'); diff --git a/tests/queries/0_stateless/01891_partition_hash.reference b/tests/queries/0_stateless/01891_partition_hash.reference index 56d11075e50..c5814777dfe 100644 --- a/tests/queries/0_stateless/01891_partition_hash.reference +++ b/tests/queries/0_stateless/01891_partition_hash.reference @@ -1 +1,2 @@ 6ba51fa36c625adab5d58007c96e32bf +ebc1c2f37455caea601feeb840757dd3 diff --git a/tests/queries/0_stateless/01891_partition_hash.sql b/tests/queries/0_stateless/01891_partition_hash.sql index f56ed6a4ff4..894594dd465 100644 --- a/tests/queries/0_stateless/01891_partition_hash.sql +++ b/tests/queries/0_stateless/01891_partition_hash.sql @@ -1,7 +1,32 @@ -drop table if exists tab; -create table tab (i8 Int8, i16 Int16, i32 Int32, i64 Int64, i128 Int128, i256 Int256, u8 UInt8, u16 UInt16, u32 UInt32, u64 UInt64, u128 UInt128, u256 UInt256, id UUID, s String, fs FixedString(33), a Array(UInt8), t Tuple(UInt16, UInt32), d Date, dt DateTime('Asia/Istanbul'), dt64 DateTime64(3, 'Asia/Istanbul'), dec128 Decimal128(3), dec256 Decimal256(4), lc LowCardinality(String)) engine = MergeTree PARTITION BY (i8, i16, i32, i64, i128, i256, u8, u16, u32, u64, u128, u256, id, s, fs, a, t, d, dt, dt64, dec128, dec256, lc) order by tuple(); -insert into tab values (-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, '61f0c404-5cb3-11e7-907b-a6006ad3dba0', 'a', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', [1, 2, 3], (-1, -2), '2020-01-01', '2020-01-01 01:01:01', '2020-01-01 01:01:01', '123.456', '78.9101', 'a'); +DROP TABLE IF EXISTS tab; +CREATE TABLE tab ( + i8 Int8, + i16 Int16, + i32 Int32, + i64 Int64, + i128 Int128, + i256 Int256, + u8 UInt8, + u16 UInt16, + u32 UInt32, + u64 UInt64, + u128 UInt128, + u256 UInt256, + id UUID, + s String, + fs FixedString(33), + a Array(UInt8), + t Tuple(UInt16, UInt32), + d Date, + dt DateTime('Asia/Istanbul'), + dt64 DateTime64(3, 'Asia/Istanbul'), + dec128 Decimal128(3), + dec256 Decimal256(4), + lc LowCardinality(String)) +engine = MergeTree PARTITION BY (i8, i16, i32, i64, i128, i256, u8, u16, u32, u64, u128, u256, id, s, fs, a, t, d, dt, dt64, dec128, dec256, lc) ORDER BY tuple(); +INSERT INTO tab VALUES (-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, '61f0c404-5cb3-11e7-907b-a6006ad3dba0', 'a', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', [1, 2, 3], (-1, -2), '2020-01-01', '2020-01-01 01:01:01', '2020-01-01 01:01:01', '123.456', '78.9101', 'a'); +INSERT INTO tab VALUES (123, 12345, 1234567890, 1234567890000000000, 123456789000000000000000000000000000000, 123456789000000000000000000000000000000000000000000000000000000000000000000000, 123, 12345, 1234567890, 1234567890000000000, 123456789000000000000000000000000000000, 123456789000000000000000000000000000000000000000000000000000000000000000000000, '61f0c404-5cb3-11e7-907b-a6006ad3dba0', 'a', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', [1, 2, 3], (-1, -2), '2020-01-01', '2020-01-01 01:01:01', '2020-01-01 01:01:01', '123.456', '78.9101', 'a'); -- Here we check that partition id did not change. -- Different result means Backward Incompatible Change. Old partitions will not be accepted by new server. -select partition_id from system.parts where table = 'tab' and database = currentDatabase(); -drop table if exists tab; +SELECT partition_id FROM system.parts WHERE table = 'tab' AND database = currentDatabase(); +DROP TABLE IF EXISTS tab; diff --git a/tests/queries/0_stateless/02020_alter_table_modify_comment.sh b/tests/queries/0_stateless/02020_alter_table_modify_comment.sh index c674f21034c..3448f052f51 100755 --- a/tests/queries/0_stateless/02020_alter_table_modify_comment.sh +++ b/tests/queries/0_stateless/02020_alter_table_modify_comment.sh @@ -43,7 +43,7 @@ EOF get_table_comment_info echo detach table - $CLICKHOUSE_CLIENT --query="DETACH TABLE comment_test_table NO DELAY;" + $CLICKHOUSE_CLIENT --query="DETACH TABLE comment_test_table SYNC;" get_table_comment_info echo re-attach table diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index f2524cac115..9cdc8182d67 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -47,7 +47,10 @@ CREATE TABLE system.clusters `default_database` String, `errors_count` UInt32, `slowdowns_count` UInt32, - `estimated_recovery_time` UInt32 + `estimated_recovery_time` UInt32, + `database_shard_name` String, + `database_replica_name` String, + `is_active` Nullable(UInt8) ) ENGINE = SystemClusters COMMENT 'SYSTEM TABLE is built on the fly.' @@ -281,7 +284,12 @@ CREATE TABLE system.functions `alias_to` String, `create_query` String, `origin` Enum8('System' = 0, 'SQLUserDefined' = 1, 'ExecutableUserDefined' = 2), - `description` String + `description` String, + `syntax` String, + `arguments` String, + `returned_value` String, + `examples` String, + `categories` String ) ENGINE = SystemFunctions COMMENT 'SYSTEM TABLE is built on the fly.' @@ -289,7 +297,7 @@ CREATE TABLE system.grants ( `user_name` Nullable(String), `role_name` Nullable(String), - `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'UNDROP TABLE' = 64, 'TRUNCATE' = 65, 'OPTIMIZE' = 66, 'BACKUP' = 67, 'KILL QUERY' = 68, 'KILL TRANSACTION' = 69, 'MOVE PARTITION BETWEEN SHARDS' = 70, 'CREATE USER' = 71, 'ALTER USER' = 72, 'DROP USER' = 73, 'CREATE ROLE' = 74, 'ALTER ROLE' = 75, 'DROP ROLE' = 76, 'ROLE ADMIN' = 77, 'CREATE ROW POLICY' = 78, 'ALTER ROW POLICY' = 79, 'DROP ROW POLICY' = 80, 'CREATE QUOTA' = 81, 'ALTER QUOTA' = 82, 'DROP QUOTA' = 83, 'CREATE SETTINGS PROFILE' = 84, 'ALTER SETTINGS PROFILE' = 85, 'DROP SETTINGS PROFILE' = 86, 'SHOW USERS' = 87, 'SHOW ROLES' = 88, 'SHOW ROW POLICIES' = 89, 'SHOW QUOTAS' = 90, 'SHOW SETTINGS PROFILES' = 91, 'SHOW ACCESS' = 92, 'ACCESS MANAGEMENT' = 93, 'SHOW NAMED COLLECTIONS' = 94, 'SHOW NAMED COLLECTIONS SECRETS' = 95, 'NAMED COLLECTION CONTROL' = 96, 'SYSTEM SHUTDOWN' = 97, 'SYSTEM DROP DNS CACHE' = 98, 'SYSTEM DROP MARK CACHE' = 99, 'SYSTEM DROP UNCOMPRESSED CACHE' = 100, 'SYSTEM DROP MMAP CACHE' = 101, 'SYSTEM DROP QUERY CACHE' = 102, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 103, 'SYSTEM DROP FILESYSTEM CACHE' = 104, 'SYSTEM DROP SCHEMA CACHE' = 105, 'SYSTEM DROP S3 CLIENT CACHE' = 106, 'SYSTEM DROP CACHE' = 107, 'SYSTEM RELOAD CONFIG' = 108, 'SYSTEM RELOAD USERS' = 109, 'SYSTEM RELOAD SYMBOLS' = 110, 'SYSTEM RELOAD DICTIONARY' = 111, 'SYSTEM RELOAD MODEL' = 112, 'SYSTEM RELOAD FUNCTION' = 113, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 114, 'SYSTEM RELOAD' = 115, 'SYSTEM RESTART DISK' = 116, 'SYSTEM MERGES' = 117, 'SYSTEM TTL MERGES' = 118, 'SYSTEM FETCHES' = 119, 'SYSTEM MOVES' = 120, 'SYSTEM DISTRIBUTED SENDS' = 121, 'SYSTEM REPLICATED SENDS' = 122, 'SYSTEM SENDS' = 123, 'SYSTEM REPLICATION QUEUES' = 124, 'SYSTEM DROP REPLICA' = 125, 'SYSTEM SYNC REPLICA' = 126, 'SYSTEM RESTART REPLICA' = 127, 'SYSTEM RESTORE REPLICA' = 128, 'SYSTEM WAIT LOADING PARTS' = 129, 'SYSTEM SYNC DATABASE REPLICA' = 130, 'SYSTEM SYNC TRANSACTION LOG' = 131, 'SYSTEM SYNC FILE CACHE' = 132, 'SYSTEM FLUSH DISTRIBUTED' = 133, 'SYSTEM FLUSH LOGS' = 134, 'SYSTEM FLUSH' = 135, 'SYSTEM THREAD FUZZER' = 136, 'SYSTEM UNFREEZE' = 137, 'SYSTEM' = 138, 'dictGet' = 139, 'addressToLine' = 140, 'addressToLineWithInlines' = 141, 'addressToSymbol' = 142, 'demangle' = 143, 'INTROSPECTION' = 144, 'FILE' = 145, 'URL' = 146, 'REMOTE' = 147, 'MONGO' = 148, 'MEILISEARCH' = 149, 'MYSQL' = 150, 'POSTGRES' = 151, 'SQLITE' = 152, 'ODBC' = 153, 'JDBC' = 154, 'HDFS' = 155, 'S3' = 156, 'HIVE' = 157, 'SOURCES' = 158, 'CLUSTER' = 159, 'ALL' = 160, 'NONE' = 161), + `access_type` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'UNDROP TABLE' = 64, 'TRUNCATE' = 65, 'OPTIMIZE' = 66, 'BACKUP' = 67, 'KILL QUERY' = 68, 'KILL TRANSACTION' = 69, 'MOVE PARTITION BETWEEN SHARDS' = 70, 'CREATE USER' = 71, 'ALTER USER' = 72, 'DROP USER' = 73, 'CREATE ROLE' = 74, 'ALTER ROLE' = 75, 'DROP ROLE' = 76, 'ROLE ADMIN' = 77, 'CREATE ROW POLICY' = 78, 'ALTER ROW POLICY' = 79, 'DROP ROW POLICY' = 80, 'CREATE QUOTA' = 81, 'ALTER QUOTA' = 82, 'DROP QUOTA' = 83, 'CREATE SETTINGS PROFILE' = 84, 'ALTER SETTINGS PROFILE' = 85, 'DROP SETTINGS PROFILE' = 86, 'SHOW USERS' = 87, 'SHOW ROLES' = 88, 'SHOW ROW POLICIES' = 89, 'SHOW QUOTAS' = 90, 'SHOW SETTINGS PROFILES' = 91, 'SHOW ACCESS' = 92, 'ACCESS MANAGEMENT' = 93, 'SHOW NAMED COLLECTIONS' = 94, 'SHOW NAMED COLLECTIONS SECRETS' = 95, 'NAMED COLLECTION CONTROL' = 96, 'SYSTEM SHUTDOWN' = 97, 'SYSTEM DROP DNS CACHE' = 98, 'SYSTEM DROP MARK CACHE' = 99, 'SYSTEM DROP UNCOMPRESSED CACHE' = 100, 'SYSTEM DROP MMAP CACHE' = 101, 'SYSTEM DROP QUERY CACHE' = 102, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 103, 'SYSTEM DROP FILESYSTEM CACHE' = 104, 'SYSTEM DROP SCHEMA CACHE' = 105, 'SYSTEM DROP S3 CLIENT CACHE' = 106, 'SYSTEM DROP CACHE' = 107, 'SYSTEM RELOAD CONFIG' = 108, 'SYSTEM RELOAD USERS' = 109, 'SYSTEM RELOAD SYMBOLS' = 110, 'SYSTEM RELOAD DICTIONARY' = 111, 'SYSTEM RELOAD MODEL' = 112, 'SYSTEM RELOAD FUNCTION' = 113, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 114, 'SYSTEM RELOAD' = 115, 'SYSTEM RESTART DISK' = 116, 'SYSTEM MERGES' = 117, 'SYSTEM TTL MERGES' = 118, 'SYSTEM FETCHES' = 119, 'SYSTEM MOVES' = 120, 'SYSTEM DISTRIBUTED SENDS' = 121, 'SYSTEM REPLICATED SENDS' = 122, 'SYSTEM SENDS' = 123, 'SYSTEM REPLICATION QUEUES' = 124, 'SYSTEM DROP REPLICA' = 125, 'SYSTEM SYNC REPLICA' = 126, 'SYSTEM RESTART REPLICA' = 127, 'SYSTEM RESTORE REPLICA' = 128, 'SYSTEM WAIT LOADING PARTS' = 129, 'SYSTEM SYNC DATABASE REPLICA' = 130, 'SYSTEM SYNC TRANSACTION LOG' = 131, 'SYSTEM SYNC FILE CACHE' = 132, 'SYSTEM FLUSH DISTRIBUTED' = 133, 'SYSTEM FLUSH LOGS' = 134, 'SYSTEM FLUSH' = 135, 'SYSTEM THREAD FUZZER' = 136, 'SYSTEM UNFREEZE' = 137, 'SYSTEM' = 138, 'dictGet' = 139, 'displaySecretsInShowAndSelect' = 140, 'addressToLine' = 141, 'addressToLineWithInlines' = 142, 'addressToSymbol' = 143, 'demangle' = 144, 'INTROSPECTION' = 145, 'FILE' = 146, 'URL' = 147, 'REMOTE' = 148, 'MONGO' = 149, 'MEILISEARCH' = 150, 'MYSQL' = 151, 'POSTGRES' = 152, 'SQLITE' = 153, 'ODBC' = 154, 'JDBC' = 155, 'HDFS' = 156, 'S3' = 157, 'HIVE' = 158, 'SOURCES' = 159, 'CLUSTER' = 160, 'ALL' = 161, 'NONE' = 162), `database` Nullable(String), `table` Nullable(String), `column` Nullable(String), @@ -356,6 +364,7 @@ CREATE TABLE system.merges `partition_id` String, `is_mutation` UInt8, `total_size_bytes_compressed` UInt64, + `total_size_bytes_uncompressed` UInt64, `total_size_marks` UInt64, `bytes_read_uncompressed` UInt64, `rows_read` UInt64, @@ -572,10 +581,10 @@ ENGINE = SystemPartsColumns COMMENT 'SYSTEM TABLE is built on the fly.' CREATE TABLE system.privileges ( - `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'UNDROP TABLE' = 64, 'TRUNCATE' = 65, 'OPTIMIZE' = 66, 'BACKUP' = 67, 'KILL QUERY' = 68, 'KILL TRANSACTION' = 69, 'MOVE PARTITION BETWEEN SHARDS' = 70, 'CREATE USER' = 71, 'ALTER USER' = 72, 'DROP USER' = 73, 'CREATE ROLE' = 74, 'ALTER ROLE' = 75, 'DROP ROLE' = 76, 'ROLE ADMIN' = 77, 'CREATE ROW POLICY' = 78, 'ALTER ROW POLICY' = 79, 'DROP ROW POLICY' = 80, 'CREATE QUOTA' = 81, 'ALTER QUOTA' = 82, 'DROP QUOTA' = 83, 'CREATE SETTINGS PROFILE' = 84, 'ALTER SETTINGS PROFILE' = 85, 'DROP SETTINGS PROFILE' = 86, 'SHOW USERS' = 87, 'SHOW ROLES' = 88, 'SHOW ROW POLICIES' = 89, 'SHOW QUOTAS' = 90, 'SHOW SETTINGS PROFILES' = 91, 'SHOW ACCESS' = 92, 'ACCESS MANAGEMENT' = 93, 'SHOW NAMED COLLECTIONS' = 94, 'SHOW NAMED COLLECTIONS SECRETS' = 95, 'NAMED COLLECTION CONTROL' = 96, 'SYSTEM SHUTDOWN' = 97, 'SYSTEM DROP DNS CACHE' = 98, 'SYSTEM DROP MARK CACHE' = 99, 'SYSTEM DROP UNCOMPRESSED CACHE' = 100, 'SYSTEM DROP MMAP CACHE' = 101, 'SYSTEM DROP QUERY CACHE' = 102, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 103, 'SYSTEM DROP FILESYSTEM CACHE' = 104, 'SYSTEM DROP SCHEMA CACHE' = 105, 'SYSTEM DROP S3 CLIENT CACHE' = 106, 'SYSTEM DROP CACHE' = 107, 'SYSTEM RELOAD CONFIG' = 108, 'SYSTEM RELOAD USERS' = 109, 'SYSTEM RELOAD SYMBOLS' = 110, 'SYSTEM RELOAD DICTIONARY' = 111, 'SYSTEM RELOAD MODEL' = 112, 'SYSTEM RELOAD FUNCTION' = 113, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 114, 'SYSTEM RELOAD' = 115, 'SYSTEM RESTART DISK' = 116, 'SYSTEM MERGES' = 117, 'SYSTEM TTL MERGES' = 118, 'SYSTEM FETCHES' = 119, 'SYSTEM MOVES' = 120, 'SYSTEM DISTRIBUTED SENDS' = 121, 'SYSTEM REPLICATED SENDS' = 122, 'SYSTEM SENDS' = 123, 'SYSTEM REPLICATION QUEUES' = 124, 'SYSTEM DROP REPLICA' = 125, 'SYSTEM SYNC REPLICA' = 126, 'SYSTEM RESTART REPLICA' = 127, 'SYSTEM RESTORE REPLICA' = 128, 'SYSTEM WAIT LOADING PARTS' = 129, 'SYSTEM SYNC DATABASE REPLICA' = 130, 'SYSTEM SYNC TRANSACTION LOG' = 131, 'SYSTEM SYNC FILE CACHE' = 132, 'SYSTEM FLUSH DISTRIBUTED' = 133, 'SYSTEM FLUSH LOGS' = 134, 'SYSTEM FLUSH' = 135, 'SYSTEM THREAD FUZZER' = 136, 'SYSTEM UNFREEZE' = 137, 'SYSTEM' = 138, 'dictGet' = 139, 'addressToLine' = 140, 'addressToLineWithInlines' = 141, 'addressToSymbol' = 142, 'demangle' = 143, 'INTROSPECTION' = 144, 'FILE' = 145, 'URL' = 146, 'REMOTE' = 147, 'MONGO' = 148, 'MEILISEARCH' = 149, 'MYSQL' = 150, 'POSTGRES' = 151, 'SQLITE' = 152, 'ODBC' = 153, 'JDBC' = 154, 'HDFS' = 155, 'S3' = 156, 'HIVE' = 157, 'SOURCES' = 158, 'CLUSTER' = 159, 'ALL' = 160, 'NONE' = 161), + `privilege` Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'UNDROP TABLE' = 64, 'TRUNCATE' = 65, 'OPTIMIZE' = 66, 'BACKUP' = 67, 'KILL QUERY' = 68, 'KILL TRANSACTION' = 69, 'MOVE PARTITION BETWEEN SHARDS' = 70, 'CREATE USER' = 71, 'ALTER USER' = 72, 'DROP USER' = 73, 'CREATE ROLE' = 74, 'ALTER ROLE' = 75, 'DROP ROLE' = 76, 'ROLE ADMIN' = 77, 'CREATE ROW POLICY' = 78, 'ALTER ROW POLICY' = 79, 'DROP ROW POLICY' = 80, 'CREATE QUOTA' = 81, 'ALTER QUOTA' = 82, 'DROP QUOTA' = 83, 'CREATE SETTINGS PROFILE' = 84, 'ALTER SETTINGS PROFILE' = 85, 'DROP SETTINGS PROFILE' = 86, 'SHOW USERS' = 87, 'SHOW ROLES' = 88, 'SHOW ROW POLICIES' = 89, 'SHOW QUOTAS' = 90, 'SHOW SETTINGS PROFILES' = 91, 'SHOW ACCESS' = 92, 'ACCESS MANAGEMENT' = 93, 'SHOW NAMED COLLECTIONS' = 94, 'SHOW NAMED COLLECTIONS SECRETS' = 95, 'NAMED COLLECTION CONTROL' = 96, 'SYSTEM SHUTDOWN' = 97, 'SYSTEM DROP DNS CACHE' = 98, 'SYSTEM DROP MARK CACHE' = 99, 'SYSTEM DROP UNCOMPRESSED CACHE' = 100, 'SYSTEM DROP MMAP CACHE' = 101, 'SYSTEM DROP QUERY CACHE' = 102, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 103, 'SYSTEM DROP FILESYSTEM CACHE' = 104, 'SYSTEM DROP SCHEMA CACHE' = 105, 'SYSTEM DROP S3 CLIENT CACHE' = 106, 'SYSTEM DROP CACHE' = 107, 'SYSTEM RELOAD CONFIG' = 108, 'SYSTEM RELOAD USERS' = 109, 'SYSTEM RELOAD SYMBOLS' = 110, 'SYSTEM RELOAD DICTIONARY' = 111, 'SYSTEM RELOAD MODEL' = 112, 'SYSTEM RELOAD FUNCTION' = 113, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 114, 'SYSTEM RELOAD' = 115, 'SYSTEM RESTART DISK' = 116, 'SYSTEM MERGES' = 117, 'SYSTEM TTL MERGES' = 118, 'SYSTEM FETCHES' = 119, 'SYSTEM MOVES' = 120, 'SYSTEM DISTRIBUTED SENDS' = 121, 'SYSTEM REPLICATED SENDS' = 122, 'SYSTEM SENDS' = 123, 'SYSTEM REPLICATION QUEUES' = 124, 'SYSTEM DROP REPLICA' = 125, 'SYSTEM SYNC REPLICA' = 126, 'SYSTEM RESTART REPLICA' = 127, 'SYSTEM RESTORE REPLICA' = 128, 'SYSTEM WAIT LOADING PARTS' = 129, 'SYSTEM SYNC DATABASE REPLICA' = 130, 'SYSTEM SYNC TRANSACTION LOG' = 131, 'SYSTEM SYNC FILE CACHE' = 132, 'SYSTEM FLUSH DISTRIBUTED' = 133, 'SYSTEM FLUSH LOGS' = 134, 'SYSTEM FLUSH' = 135, 'SYSTEM THREAD FUZZER' = 136, 'SYSTEM UNFREEZE' = 137, 'SYSTEM' = 138, 'dictGet' = 139, 'displaySecretsInShowAndSelect' = 140, 'addressToLine' = 141, 'addressToLineWithInlines' = 142, 'addressToSymbol' = 143, 'demangle' = 144, 'INTROSPECTION' = 145, 'FILE' = 146, 'URL' = 147, 'REMOTE' = 148, 'MONGO' = 149, 'MEILISEARCH' = 150, 'MYSQL' = 151, 'POSTGRES' = 152, 'SQLITE' = 153, 'ODBC' = 154, 'JDBC' = 155, 'HDFS' = 156, 'S3' = 157, 'HIVE' = 158, 'SOURCES' = 159, 'CLUSTER' = 160, 'ALL' = 161, 'NONE' = 162), `aliases` Array(String), `level` Nullable(Enum8('GLOBAL' = 0, 'DATABASE' = 1, 'TABLE' = 2, 'DICTIONARY' = 3, 'VIEW' = 4, 'COLUMN' = 5, 'NAMED_COLLECTION' = 6)), - `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'UNDROP TABLE' = 64, 'TRUNCATE' = 65, 'OPTIMIZE' = 66, 'BACKUP' = 67, 'KILL QUERY' = 68, 'KILL TRANSACTION' = 69, 'MOVE PARTITION BETWEEN SHARDS' = 70, 'CREATE USER' = 71, 'ALTER USER' = 72, 'DROP USER' = 73, 'CREATE ROLE' = 74, 'ALTER ROLE' = 75, 'DROP ROLE' = 76, 'ROLE ADMIN' = 77, 'CREATE ROW POLICY' = 78, 'ALTER ROW POLICY' = 79, 'DROP ROW POLICY' = 80, 'CREATE QUOTA' = 81, 'ALTER QUOTA' = 82, 'DROP QUOTA' = 83, 'CREATE SETTINGS PROFILE' = 84, 'ALTER SETTINGS PROFILE' = 85, 'DROP SETTINGS PROFILE' = 86, 'SHOW USERS' = 87, 'SHOW ROLES' = 88, 'SHOW ROW POLICIES' = 89, 'SHOW QUOTAS' = 90, 'SHOW SETTINGS PROFILES' = 91, 'SHOW ACCESS' = 92, 'ACCESS MANAGEMENT' = 93, 'SHOW NAMED COLLECTIONS' = 94, 'SHOW NAMED COLLECTIONS SECRETS' = 95, 'NAMED COLLECTION CONTROL' = 96, 'SYSTEM SHUTDOWN' = 97, 'SYSTEM DROP DNS CACHE' = 98, 'SYSTEM DROP MARK CACHE' = 99, 'SYSTEM DROP UNCOMPRESSED CACHE' = 100, 'SYSTEM DROP MMAP CACHE' = 101, 'SYSTEM DROP QUERY CACHE' = 102, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 103, 'SYSTEM DROP FILESYSTEM CACHE' = 104, 'SYSTEM DROP SCHEMA CACHE' = 105, 'SYSTEM DROP S3 CLIENT CACHE' = 106, 'SYSTEM DROP CACHE' = 107, 'SYSTEM RELOAD CONFIG' = 108, 'SYSTEM RELOAD USERS' = 109, 'SYSTEM RELOAD SYMBOLS' = 110, 'SYSTEM RELOAD DICTIONARY' = 111, 'SYSTEM RELOAD MODEL' = 112, 'SYSTEM RELOAD FUNCTION' = 113, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 114, 'SYSTEM RELOAD' = 115, 'SYSTEM RESTART DISK' = 116, 'SYSTEM MERGES' = 117, 'SYSTEM TTL MERGES' = 118, 'SYSTEM FETCHES' = 119, 'SYSTEM MOVES' = 120, 'SYSTEM DISTRIBUTED SENDS' = 121, 'SYSTEM REPLICATED SENDS' = 122, 'SYSTEM SENDS' = 123, 'SYSTEM REPLICATION QUEUES' = 124, 'SYSTEM DROP REPLICA' = 125, 'SYSTEM SYNC REPLICA' = 126, 'SYSTEM RESTART REPLICA' = 127, 'SYSTEM RESTORE REPLICA' = 128, 'SYSTEM WAIT LOADING PARTS' = 129, 'SYSTEM SYNC DATABASE REPLICA' = 130, 'SYSTEM SYNC TRANSACTION LOG' = 131, 'SYSTEM SYNC FILE CACHE' = 132, 'SYSTEM FLUSH DISTRIBUTED' = 133, 'SYSTEM FLUSH LOGS' = 134, 'SYSTEM FLUSH' = 135, 'SYSTEM THREAD FUZZER' = 136, 'SYSTEM UNFREEZE' = 137, 'SYSTEM' = 138, 'dictGet' = 139, 'addressToLine' = 140, 'addressToLineWithInlines' = 141, 'addressToSymbol' = 142, 'demangle' = 143, 'INTROSPECTION' = 144, 'FILE' = 145, 'URL' = 146, 'REMOTE' = 147, 'MONGO' = 148, 'MEILISEARCH' = 149, 'MYSQL' = 150, 'POSTGRES' = 151, 'SQLITE' = 152, 'ODBC' = 153, 'JDBC' = 154, 'HDFS' = 155, 'S3' = 156, 'HIVE' = 157, 'SOURCES' = 158, 'CLUSTER' = 159, 'ALL' = 160, 'NONE' = 161)) + `parent_group` Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2, 'SHOW DICTIONARIES' = 3, 'SHOW' = 4, 'SHOW FILESYSTEM CACHES' = 5, 'SELECT' = 6, 'INSERT' = 7, 'ALTER UPDATE' = 8, 'ALTER DELETE' = 9, 'ALTER ADD COLUMN' = 10, 'ALTER MODIFY COLUMN' = 11, 'ALTER DROP COLUMN' = 12, 'ALTER COMMENT COLUMN' = 13, 'ALTER CLEAR COLUMN' = 14, 'ALTER RENAME COLUMN' = 15, 'ALTER MATERIALIZE COLUMN' = 16, 'ALTER COLUMN' = 17, 'ALTER MODIFY COMMENT' = 18, 'ALTER ORDER BY' = 19, 'ALTER SAMPLE BY' = 20, 'ALTER ADD INDEX' = 21, 'ALTER DROP INDEX' = 22, 'ALTER MATERIALIZE INDEX' = 23, 'ALTER CLEAR INDEX' = 24, 'ALTER INDEX' = 25, 'ALTER ADD PROJECTION' = 26, 'ALTER DROP PROJECTION' = 27, 'ALTER MATERIALIZE PROJECTION' = 28, 'ALTER CLEAR PROJECTION' = 29, 'ALTER PROJECTION' = 30, 'ALTER ADD CONSTRAINT' = 31, 'ALTER DROP CONSTRAINT' = 32, 'ALTER CONSTRAINT' = 33, 'ALTER TTL' = 34, 'ALTER MATERIALIZE TTL' = 35, 'ALTER SETTINGS' = 36, 'ALTER MOVE PARTITION' = 37, 'ALTER FETCH PARTITION' = 38, 'ALTER FREEZE PARTITION' = 39, 'ALTER DATABASE SETTINGS' = 40, 'ALTER NAMED COLLECTION' = 41, 'ALTER TABLE' = 42, 'ALTER DATABASE' = 43, 'ALTER VIEW REFRESH' = 44, 'ALTER VIEW MODIFY QUERY' = 45, 'ALTER VIEW' = 46, 'ALTER' = 47, 'CREATE DATABASE' = 48, 'CREATE TABLE' = 49, 'CREATE VIEW' = 50, 'CREATE DICTIONARY' = 51, 'CREATE TEMPORARY TABLE' = 52, 'CREATE ARBITRARY TEMPORARY TABLE' = 53, 'CREATE FUNCTION' = 54, 'CREATE NAMED COLLECTION' = 55, 'CREATE' = 56, 'DROP DATABASE' = 57, 'DROP TABLE' = 58, 'DROP VIEW' = 59, 'DROP DICTIONARY' = 60, 'DROP FUNCTION' = 61, 'DROP NAMED COLLECTION' = 62, 'DROP' = 63, 'UNDROP TABLE' = 64, 'TRUNCATE' = 65, 'OPTIMIZE' = 66, 'BACKUP' = 67, 'KILL QUERY' = 68, 'KILL TRANSACTION' = 69, 'MOVE PARTITION BETWEEN SHARDS' = 70, 'CREATE USER' = 71, 'ALTER USER' = 72, 'DROP USER' = 73, 'CREATE ROLE' = 74, 'ALTER ROLE' = 75, 'DROP ROLE' = 76, 'ROLE ADMIN' = 77, 'CREATE ROW POLICY' = 78, 'ALTER ROW POLICY' = 79, 'DROP ROW POLICY' = 80, 'CREATE QUOTA' = 81, 'ALTER QUOTA' = 82, 'DROP QUOTA' = 83, 'CREATE SETTINGS PROFILE' = 84, 'ALTER SETTINGS PROFILE' = 85, 'DROP SETTINGS PROFILE' = 86, 'SHOW USERS' = 87, 'SHOW ROLES' = 88, 'SHOW ROW POLICIES' = 89, 'SHOW QUOTAS' = 90, 'SHOW SETTINGS PROFILES' = 91, 'SHOW ACCESS' = 92, 'ACCESS MANAGEMENT' = 93, 'SHOW NAMED COLLECTIONS' = 94, 'SHOW NAMED COLLECTIONS SECRETS' = 95, 'NAMED COLLECTION CONTROL' = 96, 'SYSTEM SHUTDOWN' = 97, 'SYSTEM DROP DNS CACHE' = 98, 'SYSTEM DROP MARK CACHE' = 99, 'SYSTEM DROP UNCOMPRESSED CACHE' = 100, 'SYSTEM DROP MMAP CACHE' = 101, 'SYSTEM DROP QUERY CACHE' = 102, 'SYSTEM DROP COMPILED EXPRESSION CACHE' = 103, 'SYSTEM DROP FILESYSTEM CACHE' = 104, 'SYSTEM DROP SCHEMA CACHE' = 105, 'SYSTEM DROP S3 CLIENT CACHE' = 106, 'SYSTEM DROP CACHE' = 107, 'SYSTEM RELOAD CONFIG' = 108, 'SYSTEM RELOAD USERS' = 109, 'SYSTEM RELOAD SYMBOLS' = 110, 'SYSTEM RELOAD DICTIONARY' = 111, 'SYSTEM RELOAD MODEL' = 112, 'SYSTEM RELOAD FUNCTION' = 113, 'SYSTEM RELOAD EMBEDDED DICTIONARIES' = 114, 'SYSTEM RELOAD' = 115, 'SYSTEM RESTART DISK' = 116, 'SYSTEM MERGES' = 117, 'SYSTEM TTL MERGES' = 118, 'SYSTEM FETCHES' = 119, 'SYSTEM MOVES' = 120, 'SYSTEM DISTRIBUTED SENDS' = 121, 'SYSTEM REPLICATED SENDS' = 122, 'SYSTEM SENDS' = 123, 'SYSTEM REPLICATION QUEUES' = 124, 'SYSTEM DROP REPLICA' = 125, 'SYSTEM SYNC REPLICA' = 126, 'SYSTEM RESTART REPLICA' = 127, 'SYSTEM RESTORE REPLICA' = 128, 'SYSTEM WAIT LOADING PARTS' = 129, 'SYSTEM SYNC DATABASE REPLICA' = 130, 'SYSTEM SYNC TRANSACTION LOG' = 131, 'SYSTEM SYNC FILE CACHE' = 132, 'SYSTEM FLUSH DISTRIBUTED' = 133, 'SYSTEM FLUSH LOGS' = 134, 'SYSTEM FLUSH' = 135, 'SYSTEM THREAD FUZZER' = 136, 'SYSTEM UNFREEZE' = 137, 'SYSTEM' = 138, 'dictGet' = 139, 'displaySecretsInShowAndSelect' = 140, 'addressToLine' = 141, 'addressToLineWithInlines' = 142, 'addressToSymbol' = 143, 'demangle' = 144, 'INTROSPECTION' = 145, 'FILE' = 146, 'URL' = 147, 'REMOTE' = 148, 'MONGO' = 149, 'MEILISEARCH' = 150, 'MYSQL' = 151, 'POSTGRES' = 152, 'SQLITE' = 153, 'ODBC' = 154, 'JDBC' = 155, 'HDFS' = 156, 'S3' = 157, 'HIVE' = 158, 'SOURCES' = 159, 'CLUSTER' = 160, 'ALL' = 161, 'NONE' = 162)) ) ENGINE = SystemPrivileges COMMENT 'SYSTEM TABLE is built on the fly.' @@ -850,6 +859,7 @@ CREATE TABLE system.replicas `is_session_expired` UInt8, `future_parts` UInt32, `parts_to_check` UInt32, + `zookeeper_name` String, `zookeeper_path` String, `replica_name` String, `replica_path` String, @@ -1117,7 +1127,7 @@ CREATE TABLE system.users `name` String, `id` UUID, `storage` String, - `auth_type` Enum8('no_password' = 0, 'plaintext_password' = 1, 'sha256_password' = 2, 'double_sha1_password' = 3, 'ldap' = 4, 'kerberos' = 5, 'ssl_certificate' = 6), + `auth_type` Enum8('no_password' = 0, 'plaintext_password' = 1, 'sha256_password' = 2, 'double_sha1_password' = 3, 'ldap' = 4, 'kerberos' = 5, 'ssl_certificate' = 6, 'bcrypt_password' = 7), `auth_params` String, `host_ip` Array(String), `host_names` Array(String), diff --git a/tests/queries/0_stateless/02125_transform_decimal_bug.reference b/tests/queries/0_stateless/02125_transform_decimal_bug.reference index 7f59d0ee7bf..d1bf333ec8e 100644 --- a/tests/queries/0_stateless/02125_transform_decimal_bug.reference +++ b/tests/queries/0_stateless/02125_transform_decimal_bug.reference @@ -1,3 +1,4 @@ +1 0 1 2 diff --git a/tests/queries/0_stateless/02125_transform_decimal_bug.sql b/tests/queries/0_stateless/02125_transform_decimal_bug.sql index 4ef471ea875..002f60076e9 100644 --- a/tests/queries/0_stateless/02125_transform_decimal_bug.sql +++ b/tests/queries/0_stateless/02125_transform_decimal_bug.sql @@ -1,4 +1,4 @@ -SELECT transform(1, [1], [toDecimal32(1, 2)]); -- { serverError 44 } +SELECT transform(1, [1], [toDecimal32(1, 2)]); SELECT transform(toDecimal32(number, 2), [toDecimal32(3, 2)], [toDecimal32(30, 2)]) FROM system.numbers LIMIT 10; SELECT transform(toDecimal32(number, 2), [toDecimal32(3, 2)], [toDecimal32(30, 2)], toDecimal32(1000, 2)) FROM system.numbers LIMIT 10; SELECT transform(number, [3, 5, 11], [toDecimal32(30, 2), toDecimal32(50, 2), toDecimal32(70,2)], toDecimal32(1000, 2)) FROM system.numbers LIMIT 10; diff --git a/tests/queries/0_stateless/02163_operators.sql b/tests/queries/0_stateless/02163_operators.sql index b2414bb197e..3f2d7d8bbb7 100644 --- a/tests/queries/0_stateless/02163_operators.sql +++ b/tests/queries/0_stateless/02163_operators.sql @@ -1,2 +1,2 @@ -WITH 2 AS `b.c`, [4, 5] AS a, 6 AS u, 3 AS v, 2 AS d, TRUE AS e, 1 AS f, 0 AS g, 2 AS h, 'Hello' AS i, 'World' AS j, TIMESTAMP '2022-02-02 02:02:02' AS w, [] AS k, (1, 2) AS l, 2 AS m, 3 AS n, [] AS o, [1] AS p, 1 AS q, q AS r, 1 AS s, 1 AS t +WITH 2 AS `b.c`, [4, 5] AS a, 6 AS u, 3 AS v, 2 AS d, TRUE AS e, 1 AS f, 0 AS g, 2 AS h, 'Hello' AS i, 'World' AS j, 'hi' AS w, NULL AS k, (1, 2) AS l, 2 AS m, 3 AS n, [] AS o, [1] AS p, 1 AS q, q AS r, 1 AS s, 1 AS t SELECT INTERVAL CASE CASE WHEN NOT -a[`b.c`] * u DIV v + d IS NOT NULL AND e OR f BETWEEN g AND h THEN i ELSE j END WHEN w THEN k END || [l, (m, n)] MINUTE IS NULL OR NOT o::Array(INT) = p <> q < r > s != t AS upyachka; diff --git a/tests/queries/0_stateless/02169_map_functions.reference b/tests/queries/0_stateless/02169_map_functions.reference index bec2eaec595..10746a70f06 100644 --- a/tests/queries/0_stateless/02169_map_functions.reference +++ b/tests/queries/0_stateless/02169_map_functions.reference @@ -40,6 +40,8 @@ {'key1':1111,'key2':2222,'key5':500,'key6':600} {'key1':1112,'key2':2224,'key5':500,'key6':600} {'key1':1113,'key2':2226,'key5':500,'key6':600} +{'key5':500,'key6':600} +{'key5':500,'key6':600} 1 1 1 diff --git a/tests/queries/0_stateless/02169_map_functions.sql b/tests/queries/0_stateless/02169_map_functions.sql index 27ceb252022..febaf2bd9d0 100644 --- a/tests/queries/0_stateless/02169_map_functions.sql +++ b/tests/queries/0_stateless/02169_map_functions.sql @@ -11,6 +11,8 @@ SELECT mapApply((k, v) -> tuple(v + 9223372036854775806), col) FROM table_map; - SELECT mapConcat(col, map('key5', 500), map('key6', 600)) FROM table_map ORDER BY id; SELECT mapConcat(col, materialize(map('key5', 500)), map('key6', 600)) FROM table_map ORDER BY id; +SELECT concat(map('key5', 500), map('key6', 600)); +SELECT map('key5', 500) || map('key6', 600); SELECT mapExists((k, v) -> k LIKE '%3', col) FROM table_map ORDER BY id; SELECT mapExists((k, v) -> k LIKE '%2' AND v < 1000, col) FROM table_map ORDER BY id; diff --git a/tests/queries/0_stateless/02226_filesystem_cache_profile_events.reference b/tests/queries/0_stateless/02226_filesystem_cache_profile_events.reference index d895040ef59..2ee0f256949 100644 --- a/tests/queries/0_stateless/02226_filesystem_cache_profile_events.reference +++ b/tests/queries/0_stateless/02226_filesystem_cache_profile_events.reference @@ -1,15 +1,15 @@ Using storage policy: s3_cache 1 0 1 0 1 0 -0 0 1 0 +0 Using storage policy: local_cache 1 0 1 0 1 0 -0 0 1 0 +0 Using storage policy: azure_cache 1 0 1 0 1 0 -0 0 1 0 +0 diff --git a/tests/queries/0_stateless/02226_filesystem_cache_profile_events.sh b/tests/queries/0_stateless/02226_filesystem_cache_profile_events.sh index 96e51a58cc4..f071a570243 100755 --- a/tests/queries/0_stateless/02226_filesystem_cache_profile_events.sh +++ b/tests/queries/0_stateless/02226_filesystem_cache_profile_events.sh @@ -64,19 +64,6 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do set remote_filesystem_read_method='threadpool'; """ - clickhouse client --multiquery --multiline --query """ - SELECT * FROM test_02226 WHERE value LIKE '%abc%' ORDER BY value LIMIT 10 FORMAT Null; - - SET enable_filesystem_cache_on_write_operations = 1; - - TRUNCATE TABLE test_02226; - SELECT count() FROM test_02226; - - SYSTEM DROP FILESYSTEM CACHE; - - INSERT INTO test_02226 SELECT * FROM generateRandom('key UInt32, value String') LIMIT 10000; - """ - query_id=$(clickhouse client --query "select queryID() from ($query) limit 1") clickhouse client --multiquery --multiline --query """ @@ -90,7 +77,20 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do AND current_database = currentDatabase() ORDER BY query_start_time DESC LIMIT 1; - - DROP TABLE test_02226; """ + + clickhouse client --multiquery --multiline --query """ + SELECT * FROM test_02226 WHERE value LIKE '%abc%' ORDER BY value LIMIT 10 FORMAT Null; + + SET enable_filesystem_cache_on_write_operations = 1; + + TRUNCATE TABLE test_02226; + SELECT count() FROM test_02226; + + SYSTEM DROP FILESYSTEM CACHE; + + INSERT INTO test_02226 SELECT * FROM generateRandom('key UInt32, value String') LIMIT 10000; + """ + + clickhouse client --query "DROP TABLE test_02226" done diff --git a/tests/queries/0_stateless/02240_system_filesystem_cache_table.reference b/tests/queries/0_stateless/02240_system_filesystem_cache_table.reference index d3be4855b36..cf2bf5fb521 100644 --- a/tests/queries/0_stateless/02240_system_filesystem_cache_table.reference +++ b/tests/queries/0_stateless/02240_system_filesystem_cache_table.reference @@ -1,10 +1,60 @@ Using storage policy: s3_cache -0 79 80 -0 745 746 -0 745 746 -0 745 746 +0 +Expect cache +DOWNLOADED 0 79 80 +DOWNLOADED 0 745 746 +2 +Expect cache +DOWNLOADED 0 79 80 +DOWNLOADED 0 745 746 +2 +Expect no cache +Expect cache +DOWNLOADED 0 79 80 +DOWNLOADED 0 745 746 +2 +Expect no cache +Expect cache +DOWNLOADED 0 79 80 +DOWNLOADED 0 745 746 +2 +Expect cache +DOWNLOADED 0 79 80 +DOWNLOADED 0 745 746 +2 +Expect no cache +Expect cache +DOWNLOADED 0 79 80 +DOWNLOADED 0 745 746 +2 +Expect no cache Using storage policy: local_cache -0 79 80 -0 745 746 -0 745 746 -0 745 746 +0 +Expect cache +DOWNLOADED 0 79 80 +DOWNLOADED 0 745 746 +2 +Expect cache +DOWNLOADED 0 79 80 +DOWNLOADED 0 745 746 +2 +Expect no cache +Expect cache +DOWNLOADED 0 79 80 +DOWNLOADED 0 745 746 +2 +Expect no cache +Expect cache +DOWNLOADED 0 79 80 +DOWNLOADED 0 745 746 +2 +Expect cache +DOWNLOADED 0 79 80 +DOWNLOADED 0 745 746 +2 +Expect no cache +Expect cache +DOWNLOADED 0 79 80 +DOWNLOADED 0 745 746 +2 +Expect no cache diff --git a/tests/queries/0_stateless/02240_system_filesystem_cache_table.sh b/tests/queries/0_stateless/02240_system_filesystem_cache_table.sh index a487f3ca739..c7dc9fbd961 100755 --- a/tests/queries/0_stateless/02240_system_filesystem_cache_table.sh +++ b/tests/queries/0_stateless/02240_system_filesystem_cache_table.sh @@ -9,34 +9,69 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) for STORAGE_POLICY in 's3_cache' 'local_cache'; do echo "Using storage policy: $STORAGE_POLICY" + ${CLICKHOUSE_CLIENT} --query "SYSTEM STOP MERGES" ${CLICKHOUSE_CLIENT} --query "SYSTEM DROP FILESYSTEM CACHE" + ${CLICKHOUSE_CLIENT} --query "SYSTEM DROP MARK CACHE" + ${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.filesystem_cache" ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test_02240_storage_policy" - ${CLICKHOUSE_CLIENT} --query "CREATE TABLE test_02240_storage_policy (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='${STORAGE_POLICY}', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false" + ${CLICKHOUSE_CLIENT} --query "CREATE TABLE test_02240_storage_policy (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='${STORAGE_POLICY}', min_bytes_for_wide_part = 1000000, compress_marks=false, compress_primary_key=false" ${CLICKHOUSE_CLIENT} --query "SYSTEM STOP MERGES test_02240_storage_policy" ${CLICKHOUSE_CLIENT} --enable_filesystem_cache_on_write_operations=0 --query "INSERT INTO test_02240_storage_policy SELECT number, toString(number) FROM numbers(100)" + + echo 'Expect cache' + ${CLICKHOUSE_CLIENT} --query "SYSTEM DROP MARK CACHE" ${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_02240_storage_policy FORMAT Null" - ${CLICKHOUSE_CLIENT} --query "SELECT file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache ORDER BY file_segment_range_end, size" + ${CLICKHOUSE_CLIENT} --query "SELECT state, file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache ORDER BY file_segment_range_begin, file_segment_range_end, size" + ${CLICKHOUSE_CLIENT} --query "SELECT uniqExact(key) FROM system.filesystem_cache"; + + echo 'Expect cache' + ${CLICKHOUSE_CLIENT} --query "SYSTEM DROP MARK CACHE" + ${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_02240_storage_policy FORMAT Null" + ${CLICKHOUSE_CLIENT} --query "SELECT state, file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache ORDER BY file_segment_range_begin, file_segment_range_end, size" + ${CLICKHOUSE_CLIENT} --query "SELECT uniqExact(key) FROM system.filesystem_cache"; ${CLICKHOUSE_CLIENT} --query "SYSTEM DROP FILESYSTEM CACHE" + echo 'Expect no cache' ${CLICKHOUSE_CLIENT} --query "SELECT file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache" + + echo 'Expect cache' + ${CLICKHOUSE_CLIENT} --query "SYSTEM DROP MARK CACHE" ${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_02240_storage_policy FORMAT Null" - ${CLICKHOUSE_CLIENT} --query "SELECT file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache" + ${CLICKHOUSE_CLIENT} --query "SELECT state, file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache ORDER BY file_segment_range_begin, file_segment_range_end, size" + ${CLICKHOUSE_CLIENT} --query "SELECT uniqExact(key) FROM system.filesystem_cache"; + ${CLICKHOUSE_CLIENT} --query "SYSTEM DROP FILESYSTEM CACHE" + echo 'Expect no cache' ${CLICKHOUSE_CLIENT} --query "SELECT file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache" ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test_02240_storage_policy_3" - ${CLICKHOUSE_CLIENT} --query "CREATE TABLE test_02240_storage_policy_3 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='${STORAGE_POLICY}_3', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false" + ${CLICKHOUSE_CLIENT} --query "CREATE TABLE test_02240_storage_policy_3 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='${STORAGE_POLICY}_3', min_bytes_for_wide_part = 1000000, compress_marks=false, compress_primary_key=false" ${CLICKHOUSE_CLIENT} --enable_filesystem_cache_on_write_operations=0 --query "INSERT INTO test_02240_storage_policy_3 SELECT number, toString(number) FROM numbers(100)" + + echo 'Expect cache' + ${CLICKHOUSE_CLIENT} --query "SYSTEM DROP MARK CACHE" ${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_02240_storage_policy_3 FORMAT Null" - ${CLICKHOUSE_CLIENT} --query "SELECT file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache ORDER BY file_segment_range_end, size" + ${CLICKHOUSE_CLIENT} --query "SELECT state, file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache ORDER BY file_segment_range_begin, file_segment_range_end, size" + ${CLICKHOUSE_CLIENT} --query "SELECT uniqExact(key) FROM system.filesystem_cache"; + + echo 'Expect cache' + ${CLICKHOUSE_CLIENT} --query "SYSTEM DROP MARK CACHE" ${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_02240_storage_policy_3 FORMAT Null" - ${CLICKHOUSE_CLIENT} --query "SELECT file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache ORDER BY file_segment_range_end, size" + ${CLICKHOUSE_CLIENT} --query "SELECT state, file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache ORDER BY file_segment_range_begin, file_segment_range_end, size" + ${CLICKHOUSE_CLIENT} --query "SELECT uniqExact(key) FROM system.filesystem_cache"; + + echo 'Expect no cache' + ${CLICKHOUSE_CLIENT} --query "SYSTEM DROP FILESYSTEM CACHE" + ${CLICKHOUSE_CLIENT} --query "SELECT file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache" + + echo 'Expect cache' + ${CLICKHOUSE_CLIENT} --query "SYSTEM DROP MARK CACHE" + ${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_02240_storage_policy_3 FORMAT Null" + ${CLICKHOUSE_CLIENT} --query "SELECT state, file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache ORDER BY file_segment_range_begin, file_segment_range_end, size" + ${CLICKHOUSE_CLIENT} --query "SELECT uniqExact(key) FROM system.filesystem_cache"; ${CLICKHOUSE_CLIENT} --query "SYSTEM DROP FILESYSTEM CACHE" - ${CLICKHOUSE_CLIENT} --query "SELECT file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache" - ${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_02240_storage_policy_3 FORMAT Null" - ${CLICKHOUSE_CLIENT} --query "SELECT file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache" - ${CLICKHOUSE_CLIENT} --query "SYSTEM DROP FILESYSTEM CACHE" + echo 'Expect no cache' ${CLICKHOUSE_CLIENT} --query "SELECT file_segment_range_begin, file_segment_range_end, size FROM system.filesystem_cache" done diff --git a/tests/queries/0_stateless/02286_drop_filesystem_cache.sh b/tests/queries/0_stateless/02286_drop_filesystem_cache.sh index 333be806de0..a6fa0457078 100755 --- a/tests/queries/0_stateless/02286_drop_filesystem_cache.sh +++ b/tests/queries/0_stateless/02286_drop_filesystem_cache.sh @@ -50,7 +50,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache'; do INNER JOIN system.filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path" - $CLICKHOUSE_CLIENT --query "DROP TABLE test_02286 NO DELAY" + $CLICKHOUSE_CLIENT --query "DROP TABLE test_02286 SYNC" $CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache" $CLICKHOUSE_CLIENT --query "SELECT cache_path FROM system.filesystem_cache" diff --git a/tests/queries/0_stateless/02344_describe_cache.reference b/tests/queries/0_stateless/02344_describe_cache.reference index c98e9d263ca..7561b32bae1 100644 --- a/tests/queries/0_stateless/02344_describe_cache.reference +++ b/tests/queries/0_stateless/02344_describe_cache.reference @@ -1,2 +1,2 @@ -2147483648 1048576 104857600 1 0 0 0 /var/lib/clickhouse/caches/s3_cache/ 0 -2147483648 1048576 104857600 0 0 0 0 /var/lib/clickhouse/caches/s3_cache_2/ 0 +134217728 1048576 104857600 1 0 0 0 /var/lib/clickhouse/caches/s3_cache/ 0 +134217728 1048576 104857600 0 0 0 0 /var/lib/clickhouse/caches/s3_cache_2/ 0 diff --git a/tests/queries/0_stateless/02344_show_caches.reference b/tests/queries/0_stateless/02344_show_caches.reference deleted file mode 100644 index 2ee4f902ba1..00000000000 --- a/tests/queries/0_stateless/02344_show_caches.reference +++ /dev/null @@ -1,14 +0,0 @@ -cached_azure -s3_cache_2 -s3_cache -s3_cache_3 -s3_cache_multi -s3_cache_4 -s3_cache_5 -s3_cache_small_segment_size -local_cache -s3_cache_6 -s3_cache_small -local_cache_2 -local_cache_3 -s3_cache_multi_2 diff --git a/tests/queries/0_stateless/02344_show_caches.sql b/tests/queries/0_stateless/02344_show_caches.sql deleted file mode 100644 index 56f00b89051..00000000000 --- a/tests/queries/0_stateless/02344_show_caches.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Tags: no-fasttest, no-replicated-database, no-cpu-aarch64 -SHOW FILESYSTEM CACHES; diff --git a/tests/queries/0_stateless/02378_part_log_profile_events_replicated.sql b/tests/queries/0_stateless/02378_part_log_profile_events_replicated.sql index d61b680bb87..4f52740c498 100644 --- a/tests/queries/0_stateless/02378_part_log_profile_events_replicated.sql +++ b/tests/queries/0_stateless/02378_part_log_profile_events_replicated.sql @@ -1,8 +1,8 @@ -- Tags: long, replica, no-replicated-database, no-parallel -DROP TABLE IF EXISTS part_log_profile_events_r1 NO DELAY; -DROP TABLE IF EXISTS part_log_profile_events_r2 NO DELAY; +DROP TABLE IF EXISTS part_log_profile_events_r1 SYNC; +DROP TABLE IF EXISTS part_log_profile_events_r2 SYNC; CREATE TABLE part_log_profile_events_r1 (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/test_02378/part_log_profile_events', 'r1') @@ -36,5 +36,5 @@ WHERE event_time > now() - INTERVAL 10 MINUTE AND event_type == 'DownloadPart' ; -DROP TABLE part_log_profile_events_r1 NO DELAY; -DROP TABLE part_log_profile_events_r2 NO DELAY; +DROP TABLE part_log_profile_events_r1 SYNC; +DROP TABLE part_log_profile_events_r2 SYNC; diff --git a/tests/queries/0_stateless/02432_s3_parallel_parts_cleanup.sql b/tests/queries/0_stateless/02432_s3_parallel_parts_cleanup.sql index 3688a649d5e..88fb2cdf9b1 100644 --- a/tests/queries/0_stateless/02432_s3_parallel_parts_cleanup.sql +++ b/tests/queries/0_stateless/02432_s3_parallel_parts_cleanup.sql @@ -1,5 +1,7 @@ -- Tags: no-fasttest +SET send_logs_level = 'fatal'; + drop table if exists rmt; drop table if exists rmt2; diff --git a/tests/queries/0_stateless/02447_drop_database_replica.reference b/tests/queries/0_stateless/02447_drop_database_replica.reference index 1d65fe66c6e..f2b41569540 100644 --- a/tests/queries/0_stateless/02447_drop_database_replica.reference +++ b/tests/queries/0_stateless/02447_drop_database_replica.reference @@ -6,10 +6,16 @@ t 2 2 2 -rdb_default 1 1 -rdb_default 1 2 2 2 2 +2 +rdb_default 1 1 s1 r1 1 +2 +2 +rdb_default 1 1 s1 r1 1 +rdb_default 1 2 s1 r2 0 +2 +2 t -rdb_default_3 1 1 +rdb_default_4 1 1 s1 r1 1 diff --git a/tests/queries/0_stateless/02447_drop_database_replica.sh b/tests/queries/0_stateless/02447_drop_database_replica.sh index 4bfd6243c2e..47a6cf10bda 100755 --- a/tests/queries/0_stateless/02447_drop_database_replica.sh +++ b/tests/queries/0_stateless/02447_drop_database_replica.sh @@ -13,35 +13,49 @@ $CLICKHOUSE_CLIENT -q "show tables from $db" $CLICKHOUSE_CLIENT -q "system drop database replica 's1|r1' from table t" 2>&1| grep -Fac "SYNTAX_ERROR" $CLICKHOUSE_CLIENT -q "system drop database replica 's1|r1' from database $db" 2>&1| grep -Fac "There is a local database" +$CLICKHOUSE_CLIENT -q "system drop database replica 'r1' from shard 's1' from database $db" 2>&1| grep -Fac "There is a local database" $CLICKHOUSE_CLIENT -q "system drop database replica 's1|r1' from zkpath '/test/$CLICKHOUSE_DATABASE/rdb'" 2>&1| grep -Fac "There is a local database" $CLICKHOUSE_CLIENT -q "system drop database replica 's1|r1' from zkpath '/test/$CLICKHOUSE_DATABASE/rdb/'" 2>&1| grep -Fac "There is a local database" +$CLICKHOUSE_CLIENT -q "system drop database replica 'r1' from shard 's1' from zkpath '/test/$CLICKHOUSE_DATABASE/rdb/'" 2>&1| grep -Fac "There is a local database" $CLICKHOUSE_CLIENT -q "system drop database replica 's1|r1' from zkpath '/test/$CLICKHOUSE_DATABASE/'" 2>&1| grep -Fac "does not look like a path of Replicated database" $CLICKHOUSE_CLIENT -q "system drop database replica 's2|r1' from zkpath '/test/$CLICKHOUSE_DATABASE/rdb'" 2>&1| grep -Fac "does not exist" +$CLICKHOUSE_CLIENT -q "system drop database replica 's1' from shard 'r1' from zkpath '/test/$CLICKHOUSE_DATABASE/rdb'" 2>&1| grep -Fac "does not exist" +$CLICKHOUSE_CLIENT -q "system drop database replica 's1|r1' from shard 's1' from zkpath '/test/$CLICKHOUSE_DATABASE/rdb'" 2>&1| grep -Fac "does not exist" $CLICKHOUSE_CLIENT -q "system drop database replica 's2/r1' from zkpath '/test/$CLICKHOUSE_DATABASE/rdb'" 2>&1| grep -Fac "Invalid replica name" db2="${db}_2" +db3="${db}_3" $CLICKHOUSE_CLIENT --allow_experimental_database_replicated=1 -q "create database $db2 engine=Replicated('/test/$CLICKHOUSE_DATABASE/rdb', 's1', 'r2')" +$CLICKHOUSE_CLIENT --allow_experimental_database_replicated=1 -q "create database $db3 engine=Replicated('/test/$CLICKHOUSE_DATABASE/rdb', 's2', 'r1')" $CLICKHOUSE_CLIENT -q "system sync database replica $db" -$CLICKHOUSE_CLIENT -q "select cluster, shard_num, replica_num from system.clusters where cluster='$db' order by shard_num, replica_num" +$CLICKHOUSE_CLIENT -q "select cluster, shard_num, replica_num, database_shard_name, database_replica_name, is_active from system.clusters where cluster='$db' and shard_num=1 and replica_num=1" $CLICKHOUSE_CLIENT -q "system drop database replica 's1|r1' from database $db2" 2>&1| grep -Fac "is active, cannot drop it" +$CLICKHOUSE_CLIENT -q "detach database $db3" +$CLICKHOUSE_CLIENT -q "system drop database replica 'r1' from shard 's2' from database $db" +$CLICKHOUSE_CLIENT -q "attach database $db3" 2>/dev/null +$CLICKHOUSE_CLIENT --distributed_ddl_output_mode=none -q "create table $db3.t2 as system.query_log" 2>&1| grep -Fac "Database is in readonly mode" # Suppress style check: current_database=$CLICKHOUSE_DATABASE + $CLICKHOUSE_CLIENT -q "detach database $db2" +$CLICKHOUSE_CLIENT -q "system sync database replica $db" +$CLICKHOUSE_CLIENT -q "select cluster, shard_num, replica_num, database_shard_name, database_replica_name, is_active from system.clusters where cluster='$db' order by shard_num, replica_num" $CLICKHOUSE_CLIENT -q "system drop database replica 's1|r2' from database $db" $CLICKHOUSE_CLIENT -q "attach database $db2" 2>/dev/null $CLICKHOUSE_CLIENT --distributed_ddl_output_mode=none -q "create table $db2.t2 as system.query_log" 2>&1| grep -Fac "Database is in readonly mode" # Suppress style check: current_database=$CLICKHOUSE_DATABASE $CLICKHOUSE_CLIENT -q "detach database $db" -$CLICKHOUSE_CLIENT -q "system drop database replica 's1|r1' from zkpath '/test/$CLICKHOUSE_DATABASE/rdb/'" +$CLICKHOUSE_CLIENT -q "system drop database replica 'r1' from shard 's1' from zkpath '/test/$CLICKHOUSE_DATABASE/rdb/'" $CLICKHOUSE_CLIENT -q "attach database $db" 2>/dev/null $CLICKHOUSE_CLIENT --distributed_ddl_output_mode=none -q "create table $db.t2 as system.query_log" 2>&1| grep -Fac "Database is in readonly mode" # Suppress style check: current_database=$CLICKHOUSE_DATABASE $CLICKHOUSE_CLIENT -q "show tables from $db" -db3="${db}_3" -$CLICKHOUSE_CLIENT --allow_experimental_database_replicated=1 -q "create database $db3 engine=Replicated('/test/$CLICKHOUSE_DATABASE/rdb', 's1', 'r1')" -$CLICKHOUSE_CLIENT -q "system sync database replica $db3" -$CLICKHOUSE_CLIENT -q "select cluster, shard_num, replica_num from system.clusters where cluster='$db3'" +db4="${db}_4" +$CLICKHOUSE_CLIENT --allow_experimental_database_replicated=1 -q "create database $db4 engine=Replicated('/test/$CLICKHOUSE_DATABASE/rdb', 's1', 'r1')" +$CLICKHOUSE_CLIENT -q "system sync database replica $db4" +$CLICKHOUSE_CLIENT -q "select cluster, shard_num, replica_num, database_shard_name, database_replica_name, is_active from system.clusters where cluster='$db4'" $CLICKHOUSE_CLIENT -q "drop database $db" $CLICKHOUSE_CLIENT -q "drop database $db2" $CLICKHOUSE_CLIENT -q "drop database $db3" +$CLICKHOUSE_CLIENT -q "drop database $db4" diff --git a/tests/queries/0_stateless/02456_async_inserts_logs.reference b/tests/queries/0_stateless/02456_async_inserts_logs.reference index efd8a88eca4..ba1b19fb184 100644 --- a/tests/queries/0_stateless/02456_async_inserts_logs.reference +++ b/tests/queries/0_stateless/02456_async_inserts_logs.reference @@ -1,7 +1,10 @@ 5 - Values 21 1 Ok 1 -t_async_inserts_logs JSONEachRow 39 1 Ok 1 -t_async_inserts_logs Values 8 1 Ok 1 -t_async_inserts_logs JSONEachRow 6 0 ParsingError 1 -t_async_inserts_logs Values 6 0 ParsingError 1 -t_async_inserts_logs Values 8 0 FlushError 1 + Values 21 2 1 Ok 1 +t_async_inserts_logs JSONEachRow 39 2 1 Ok 1 +t_async_inserts_logs Values 8 1 1 Ok 1 +t_async_inserts_logs JSONEachRow 6 0 0 ParsingError 1 +t_async_inserts_logs Values 6 0 0 ParsingError 1 +t_async_inserts_logs Values 8 1 0 FlushError 1 +AsyncInsertBytes 1 +AsyncInsertQuery 1 +AsyncInsertRows 1 diff --git a/tests/queries/0_stateless/02456_async_inserts_logs.sh b/tests/queries/0_stateless/02456_async_inserts_logs.sh index 006455e2d42..43cd73d7231 100755 --- a/tests/queries/0_stateless/02456_async_inserts_logs.sh +++ b/tests/queries/0_stateless/02456_async_inserts_logs.sh @@ -30,10 +30,15 @@ ${CLICKHOUSE_CLIENT} -q "SELECT count() FROM t_async_inserts_logs" ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS" ${CLICKHOUSE_CLIENT} -q " - SELECT table, format, bytes, empty(exception), status, + SELECT table, format, bytes, rows, empty(exception), status, status = 'ParsingError' ? flush_time_microseconds = 0 : flush_time_microseconds > event_time_microseconds AS time_ok FROM system.asynchronous_insert_log WHERE database = '$CLICKHOUSE_DATABASE' OR query ILIKE 'INSERT INTO FUNCTION%$CLICKHOUSE_DATABASE%' ORDER BY table, status, format" ${CLICKHOUSE_CLIENT} -q "DROP TABLE t_async_inserts_logs" + +${CLICKHOUSE_CLIENT} -q " +SELECT event, value > 0 FROM system.events +WHERE event IN ('AsyncInsertQuery', 'AsyncInsertBytes', 'AsyncInsertRows') +ORDER BY event" diff --git a/tests/queries/0_stateless/02479_nullable_primary_key_non_first_column.reference b/tests/queries/0_stateless/02479_nullable_primary_key_non_first_column.reference new file mode 100644 index 00000000000..ed6ac232d9c --- /dev/null +++ b/tests/queries/0_stateless/02479_nullable_primary_key_non_first_column.reference @@ -0,0 +1,2 @@ +a \N +1 1 \N diff --git a/tests/queries/0_stateless/02479_nullable_primary_key_non_first_column.sql b/tests/queries/0_stateless/02479_nullable_primary_key_non_first_column.sql new file mode 100644 index 00000000000..2d56e315bd1 --- /dev/null +++ b/tests/queries/0_stateless/02479_nullable_primary_key_non_first_column.sql @@ -0,0 +1,11 @@ +drop table if exists test_table; +create table test_table (A Nullable(String), B Nullable(String)) engine MergeTree order by (A,B) settings index_granularity = 1, allow_nullable_key=1; +insert into test_table values ('a', 'b'), ('a', null), (null, 'b'); +select * from test_table where B is null; +drop table test_table; + +DROP TABLE IF EXISTS dm_metric_small2; +CREATE TABLE dm_metric_small2 (`x` Nullable(Int64), `y` Nullable(Int64), `z` Nullable(Int64)) ENGINE = MergeTree() ORDER BY (x, y, z) SETTINGS index_granularity = 1, allow_nullable_key = 1; +INSERT INTO dm_metric_small2 VALUES (1,1,NULL) (1,1,1) (1,2,0) (1,2,1) (1,2,NULL) (1,2,NULL); +SELECT * FROM dm_metric_small2 WHERE (x = 1) AND (y = 1) AND z IS NULL; +DROP TABLE dm_metric_small2; \ No newline at end of file diff --git a/tests/queries/0_stateless/02479_nullable_primary_key_second_column.reference b/tests/queries/0_stateless/02479_nullable_primary_key_second_column.reference deleted file mode 100644 index f0227e1a41e..00000000000 --- a/tests/queries/0_stateless/02479_nullable_primary_key_second_column.reference +++ /dev/null @@ -1 +0,0 @@ -a \N diff --git a/tests/queries/0_stateless/02479_nullable_primary_key_second_column.sql b/tests/queries/0_stateless/02479_nullable_primary_key_second_column.sql deleted file mode 100644 index ad0c09222c2..00000000000 --- a/tests/queries/0_stateless/02479_nullable_primary_key_second_column.sql +++ /dev/null @@ -1,9 +0,0 @@ -drop table if exists test_table; - -create table test_table (A Nullable(String), B Nullable(String)) engine MergeTree order by (A,B) settings index_granularity = 1, allow_nullable_key=1; - -insert into test_table values ('a', 'b'), ('a', null), (null, 'b'); - -select * from test_table where B is null; - -drop table test_table; diff --git a/tests/queries/0_stateless/02481_async_insert_dedup.python b/tests/queries/0_stateless/02481_async_insert_dedup.python index 1be2b673b73..9fd82da1038 100644 --- a/tests/queries/0_stateless/02481_async_insert_dedup.python +++ b/tests/queries/0_stateless/02481_async_insert_dedup.python @@ -15,7 +15,7 @@ from pure_http_client import ClickHouseClient client = ClickHouseClient() # test table without partition -client.query("DROP TABLE IF EXISTS t_async_insert_dedup_no_part NO DELAY") +client.query("DROP TABLE IF EXISTS t_async_insert_dedup_no_part SYNC") client.query( """ CREATE TABLE t_async_insert_dedup_no_part ( @@ -35,7 +35,7 @@ client.query( ) result = client.query("select count(*) from t_async_insert_dedup_no_part") print(result, flush=True) -client.query("DROP TABLE IF EXISTS t_async_insert_dedup_no_part NO DELAY") +client.query("DROP TABLE IF EXISTS t_async_insert_dedup_no_part SYNC") # generate data and push to queue @@ -95,7 +95,7 @@ def fetch_and_insert_data(q, client): # main process -client.query("DROP TABLE IF EXISTS t_async_insert_dedup NO DELAY") +client.query("DROP TABLE IF EXISTS t_async_insert_dedup SYNC") client.query( """ CREATE TABLE t_async_insert_dedup ( @@ -161,6 +161,6 @@ result = int(result.split()[0]) if result <= 0: raise Exception(f"AsyncInsertCacheHits should > 0, but got {result}") -client.query("DROP TABLE IF EXISTS t_async_insert_dedup NO DELAY") +client.query("DROP TABLE IF EXISTS t_async_insert_dedup SYNC") os._exit(os.EX_OK) diff --git a/tests/queries/0_stateless/02497_if_transform_strings_to_enum.reference b/tests/queries/0_stateless/02497_if_transform_strings_to_enum.reference index f5284f38b86..a1a653361ee 100644 --- a/tests/queries/0_stateless/02497_if_transform_strings_to_enum.reference +++ b/tests/queries/0_stateless/02497_if_transform_strings_to_enum.reference @@ -405,16 +405,6 @@ QUERY id: 0 TABLE id: 7, table_name: system.numbers LIMIT CONSTANT id: 17, constant_value: UInt64_10, constant_value_type: UInt64 -\N -\N -\N -\N -\N -\N -\N -\N -\N -\N SELECT transform(number, [NULL], _CAST([\'google\', \'censor.net\', \'yahoo\'], \'Array(Enum8(\\\'censor.net\\\' = 1, \\\'google\\\' = 2, \\\'other\\\' = 3, \\\'yahoo\\\' = 4))\'), _CAST(\'other\', \'Enum8(\\\'censor.net\\\' = 1, \\\'google\\\' = 2, \\\'other\\\' = 3, \\\'yahoo\\\' = 4)\')) FROM ( @@ -424,56 +414,38 @@ FROM ) QUERY id: 0 PROJECTION COLUMNS - transform(number, [NULL], [\'google\', \'censor.net\', \'yahoo\'], \'other\') Nullable(Nothing) + transform(number, [NULL], [\'google\', \'censor.net\', \'yahoo\'], \'other\') String PROJECTION LIST id: 1, nodes: 1 - FUNCTION id: 2, function_name: transform, function_type: ordinary, result_type: Nullable(Nothing) + FUNCTION id: 2, function_name: toString, function_type: ordinary, result_type: String ARGUMENTS - LIST id: 3, nodes: 4 - COLUMN id: 4, column_name: number, result_type: Nullable(Nothing), source_id: 5 - CONSTANT id: 6, constant_value: Array_[NULL], constant_value_type: Array(Nullable(Nothing)) - CONSTANT id: 7, constant_value: Array_[\'google\', \'censor.net\', \'yahoo\'], constant_value_type: Array(String) - CONSTANT id: 8, constant_value: \'other\', constant_value_type: String + LIST id: 3, nodes: 1 + FUNCTION id: 4, function_name: transform, function_type: ordinary, result_type: Enum8(\'censor.net\' = 1, \'google\' = 2, \'other\' = 3, \'yahoo\' = 4) + ARGUMENTS + LIST id: 5, nodes: 4 + COLUMN id: 6, column_name: number, result_type: Nullable(Nothing), source_id: 7 + CONSTANT id: 8, constant_value: Array_[NULL], constant_value_type: Array(Nullable(Nothing)) + FUNCTION id: 9, function_name: _CAST, function_type: ordinary, result_type: Array(Enum8(\'censor.net\' = 1, \'google\' = 2, \'other\' = 3, \'yahoo\' = 4)) + ARGUMENTS + LIST id: 10, nodes: 2 + CONSTANT id: 11, constant_value: Array_[\'google\', \'censor.net\', \'yahoo\'], constant_value_type: Array(String) + CONSTANT id: 12, constant_value: \'Array(Enum8(\\\'censor.net\\\' = 1, \\\'google\\\' = 2, \\\'other\\\' = 3, \\\'yahoo\\\' = 4))\', constant_value_type: String + FUNCTION id: 13, function_name: _CAST, function_type: ordinary, result_type: Enum8(\'censor.net\' = 1, \'google\' = 2, \'other\' = 3, \'yahoo\' = 4) + ARGUMENTS + LIST id: 14, nodes: 2 + CONSTANT id: 15, constant_value: \'other\', constant_value_type: String + CONSTANT id: 16, constant_value: \'Enum8(\\\'censor.net\\\' = 1, \\\'google\\\' = 2, \\\'other\\\' = 3, \\\'yahoo\\\' = 4)\', constant_value_type: String JOIN TREE - QUERY id: 5, is_subquery: 1 + QUERY id: 7, is_subquery: 1 PROJECTION COLUMNS number Nullable(Nothing) PROJECTION - LIST id: 9, nodes: 1 - CONSTANT id: 10, constant_value: NULL, constant_value_type: Nullable(Nothing) + LIST id: 17, nodes: 1 + CONSTANT id: 18, constant_value: NULL, constant_value_type: Nullable(Nothing) JOIN TREE - TABLE id: 11, table_name: system.numbers + TABLE id: 19, table_name: system.numbers LIMIT - CONSTANT id: 12, constant_value: UInt64_10, constant_value_type: UInt64 -\N -\N -\N -\N -\N -\N -\N -\N -\N -\N -SELECT transform(number, NULL, _CAST([\'google\', \'censor.net\', \'yahoo\'], \'Array(Enum8(\\\'censor.net\\\' = 1, \\\'google\\\' = 2, \\\'other\\\' = 3, \\\'yahoo\\\' = 4))\'), _CAST(\'other\', \'Enum8(\\\'censor.net\\\' = 1, \\\'google\\\' = 2, \\\'other\\\' = 3, \\\'yahoo\\\' = 4)\')) -FROM system.numbers -LIMIT 10 -QUERY id: 0 - PROJECTION COLUMNS - transform(number, NULL, [\'google\', \'censor.net\', \'yahoo\'], \'other\') Nullable(Nothing) - PROJECTION - LIST id: 1, nodes: 1 - FUNCTION id: 2, function_name: transform, function_type: ordinary, result_type: Nullable(Nothing) - ARGUMENTS - LIST id: 3, nodes: 4 - COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5 - CONSTANT id: 6, constant_value: NULL, constant_value_type: Nullable(Nothing) - CONSTANT id: 7, constant_value: Array_[\'google\', \'censor.net\', \'yahoo\'], constant_value_type: Array(String) - CONSTANT id: 8, constant_value: \'other\', constant_value_type: String - JOIN TREE - TABLE id: 5, table_name: system.numbers - LIMIT - CONSTANT id: 9, constant_value: UInt64_10, constant_value_type: UInt64 + CONSTANT id: 20, constant_value: UInt64_10, constant_value_type: UInt64 other other google diff --git a/tests/queries/0_stateless/02497_if_transform_strings_to_enum.sql b/tests/queries/0_stateless/02497_if_transform_strings_to_enum.sql index c23046c7b20..492d42cb6bc 100644 --- a/tests/queries/0_stateless/02497_if_transform_strings_to_enum.sql +++ b/tests/queries/0_stateless/02497_if_transform_strings_to_enum.sql @@ -33,13 +33,13 @@ SELECT transform(number, [2, 4, 6], ['google', 'censor.net', 'yahoo'], 'other') EXPLAIN SYNTAX SELECT transform(number, [2, 4, 6], ['google', 'censor.net', 'yahoo'], 'other') as value, value FROM system.numbers LIMIT 10; EXPLAIN QUERY TREE run_passes = 1 SELECT transform(number, [2, 4, 6], ['google', 'censor.net', 'yahoo'], 'other') as value, value FROM system.numbers LIMIT 10; -SELECT transform(number, [NULL], ['google', 'censor.net', 'yahoo'], 'other') FROM (SELECT NULL as number FROM system.numbers LIMIT 10); +SELECT transform(number, [NULL], ['google', 'censor.net', 'yahoo'], 'other') FROM (SELECT NULL as number FROM system.numbers LIMIT 10); -- { serverError 36 } EXPLAIN SYNTAX SELECT transform(number, [NULL], ['google', 'censor.net', 'yahoo'], 'other') FROM (SELECT NULL as number FROM system.numbers LIMIT 10); EXPLAIN QUERY TREE run_passes = 1 SELECT transform(number, [NULL], ['google', 'censor.net', 'yahoo'], 'other') FROM (SELECT NULL as number FROM system.numbers LIMIT 10); -SELECT transform(number, NULL, ['google', 'censor.net', 'yahoo'], 'other') FROM system.numbers LIMIT 10; -EXPLAIN SYNTAX SELECT transform(number, NULL, ['google', 'censor.net', 'yahoo'], 'other') FROM system.numbers LIMIT 10; -EXPLAIN QUERY TREE run_passes = 1 SELECT transform(number, NULL, ['google', 'censor.net', 'yahoo'], 'other') FROM system.numbers LIMIT 10; +SELECT transform(number, NULL, ['google', 'censor.net', 'yahoo'], 'other') FROM system.numbers LIMIT 10; -- { serverError 43 } +EXPLAIN SYNTAX SELECT transform(number, NULL, ['google', 'censor.net', 'yahoo'], 'other') FROM system.numbers LIMIT 10; -- { serverError 43 } +EXPLAIN QUERY TREE run_passes = 1 SELECT transform(number, NULL, ['google', 'censor.net', 'yahoo'], 'other') FROM system.numbers LIMIT 10; -- { serverError 43 } SET optimize_if_transform_strings_to_enum = 0; diff --git a/tests/queries/0_stateless/02503_cache_on_write_with_small_segment_size.sh b/tests/queries/0_stateless/02503_cache_on_write_with_small_segment_size.sh index 918adc12de6..ed66c36b823 100755 --- a/tests/queries/0_stateless/02503_cache_on_write_with_small_segment_size.sh +++ b/tests/queries/0_stateless/02503_cache_on_write_with_small_segment_size.sh @@ -33,5 +33,5 @@ select count() from system.filesystem_cache_log where query_id = '$query_id' AND ${CLICKHOUSE_CLIENT} --multiline --multiquery -q " select count() from ttt; -drop table ttt no delay; +drop table ttt sync; " diff --git a/tests/queries/0_stateless/02515_cleanup_async_insert_block_ids.sh b/tests/queries/0_stateless/02515_cleanup_async_insert_block_ids.sh index 9e22089d5e1..458a5e95faa 100755 --- a/tests/queries/0_stateless/02515_cleanup_async_insert_block_ids.sh +++ b/tests/queries/0_stateless/02515_cleanup_async_insert_block_ids.sh @@ -9,7 +9,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) CLICKHOUSE_TEST_ZOOKEEPER_PREFIX="${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/${CLICKHOUSE_DATABASE}" $CLICKHOUSE_CLIENT -n --query " - DROP TABLE IF EXISTS t_async_insert_cleanup NO DELAY; + DROP TABLE IF EXISTS t_async_insert_cleanup SYNC; CREATE TABLE t_async_insert_cleanup ( KeyID UInt32 ) Engine = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/t_async_insert_cleanup', '{replica}') @@ -27,7 +27,7 @@ old_answer=$($CLICKHOUSE_CLIENT --query "SELECT count(*) FROM system.zookeeper W for i in {1..300}; do answer=$($CLICKHOUSE_CLIENT --query "SELECT count(*) FROM system.zookeeper WHERE path like '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/t_async_insert_cleanup/async_blocks%' settings allow_unrestricted_reads_from_keeper = 'true'") if [ $answer == '10' ]; then - $CLICKHOUSE_CLIENT -n --query "DROP TABLE t_async_insert_cleanup NO DELAY;" + $CLICKHOUSE_CLIENT -n --query "DROP TABLE t_async_insert_cleanup SYNC;" exit 0 fi sleep 1 @@ -36,4 +36,4 @@ done $CLICKHOUSE_CLIENT --query "SELECT count(*) FROM t_async_insert_cleanup" echo $old_answer $CLICKHOUSE_CLIENT --query "SELECT count(*) FROM system.zookeeper WHERE path like '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/t_async_insert_cleanup/async_blocks%' settings allow_unrestricted_reads_from_keeper = 'true'" -$CLICKHOUSE_CLIENT -n --query "DROP TABLE t_async_insert_cleanup NO DELAY;" +$CLICKHOUSE_CLIENT -n --query "DROP TABLE t_async_insert_cleanup SYNC;" diff --git a/tests/queries/0_stateless/02516_join_with_totals_and_subquery_bug.reference b/tests/queries/0_stateless/02516_join_with_totals_and_subquery_bug.reference index fd0b223f8e5..19da8828c30 100644 --- a/tests/queries/0_stateless/02516_join_with_totals_and_subquery_bug.reference +++ b/tests/queries/0_stateless/02516_join_with_totals_and_subquery_bug.reference @@ -2,6 +2,10 @@ 1 0 +1 +1 + +1 \N 100000000000000000000 diff --git a/tests/queries/0_stateless/02516_join_with_totals_and_subquery_bug.sql b/tests/queries/0_stateless/02516_join_with_totals_and_subquery_bug.sql index b6e60aa2e1f..6b58d737a3e 100644 --- a/tests/queries/0_stateless/02516_join_with_totals_and_subquery_bug.sql +++ b/tests/queries/0_stateless/02516_join_with_totals_and_subquery_bug.sql @@ -1,3 +1,5 @@ +SET allow_experimental_analyzer = 1; + SELECT * FROM ( @@ -12,7 +14,26 @@ INNER JOIN SELECT 1 GROUP BY 1 WITH TOTALS -) AS t2 USING (a); +) AS t2 USING (a) +SETTINGS allow_experimental_analyzer=0; + +SELECT * +FROM +( + SELECT 1 AS a +) AS t1 +INNER JOIN +( + SELECT 1 AS a + GROUP BY 1 + WITH TOTALS + UNION ALL + SELECT 1 + GROUP BY 1 + WITH TOTALS +) AS t2 USING (a) +SETTINGS allow_experimental_analyzer=1; + SELECT a FROM diff --git a/tests/queries/0_stateless/02536_replace_with_nonconst_needle_and_replacement.reference b/tests/queries/0_stateless/02536_replace_with_nonconst_needle_and_replacement.reference index c7a02045316..5e50b9e6cbf 100644 --- a/tests/queries/0_stateless/02536_replace_with_nonconst_needle_and_replacement.reference +++ b/tests/queries/0_stateless/02536_replace_with_nonconst_needle_and_replacement.reference @@ -5,18 +5,33 @@ 3 Hello World not_found x Hello World 4 Hello World [eo] x Hello World 5 Hello World . x Hello World +1 Hello World l x Hexxo Worxd +2 Hello World ll x Hexo World +3 Hello World not_found x Hello World +4 Hello World [eo] x Hello World +5 Hello World . x Hello World - const needle, non-const replacement 1 Hello World l xx Hexxxxo Worxxd 2 Hello World l x Hexxo Worxd 3 Hello World l x Hexxo Worxd 4 Hello World l x Hexxo Worxd 5 Hello World l x Hexxo Worxd +1 Hello World l xx Hexxxxo Worxxd +2 Hello World l x Hexxo Worxd +3 Hello World l x Hexxo Worxd +4 Hello World l x Hexxo Worxd +5 Hello World l x Hexxo Worxd - non-const needle, non-const replacement 1 Hello World l xx Hexxxxo Worxxd 2 Hello World ll x Hexo World 3 Hello World not_found x Hello World 4 Hello World [eo] x Hello World 5 Hello World . x Hello World +1 Hello World l xx Hexxxxo Worxxd +2 Hello World ll x Hexo World +3 Hello World not_found x Hello World +4 Hello World [eo] x Hello World +5 Hello World . x Hello World ** replaceOne() ** - non-const needle, const replacement 1 Hello World l x Hexlo World @@ -24,18 +39,33 @@ 3 Hello World not_found x Hello World 4 Hello World [eo] x Hello World 5 Hello World . x Hello World +1 Hello World l x Hexlo World +2 Hello World ll x Hexo World +3 Hello World not_found x Hello World +4 Hello World [eo] x Hello World +5 Hello World . x Hello World - const needle, non-const replacement 1 Hello World l xx Hexxlo World 2 Hello World l x Hexlo World 3 Hello World l x Hexlo World 4 Hello World l x Hexlo World 5 Hello World l x Hexlo World +1 Hello World l xx Hexxlo World +2 Hello World l x Hexlo World +3 Hello World l x Hexlo World +4 Hello World l x Hexlo World +5 Hello World l x Hexlo World - non-const needle, non-const replacement 1 Hello World l xx Hexxlo World 2 Hello World ll x Hexo World 3 Hello World not_found x Hello World 4 Hello World [eo] x Hello World 5 Hello World . x Hello World +1 Hello World l xx Hexxlo World +2 Hello World ll x Hexo World +3 Hello World not_found x Hello World +4 Hello World [eo] x Hello World +5 Hello World . x Hello World ** replaceRegexpAll() ** - non-const needle, const replacement 1 Hello World l x Hexxo Worxd @@ -43,18 +73,33 @@ 3 Hello World not_found x Hello World 4 Hello World [eo] x Hxllx Wxrld 5 Hello World . x xxxxxxxxxxx +1 Hello World l x Hexxo Worxd +2 Hello World ll x Hexo World +3 Hello World not_found x Hello World +4 Hello World [eo] x Hxllx Wxrld +5 Hello World . x xxxxxxxxxxx - const needle, non-const replacement 1 Hello World l xx Hexxxxo Worxxd 2 Hello World l x Hexxo Worxd 3 Hello World l x Hexxo Worxd 4 Hello World l x Hexxo Worxd 5 Hello World l x Hexxo Worxd +1 Hello World l xx Hexxxxo Worxxd +2 Hello World l x Hexxo Worxd +3 Hello World l x Hexxo Worxd +4 Hello World l x Hexxo Worxd +5 Hello World l x Hexxo Worxd - non-const needle, non-const replacement 1 Hello World l xx Hexxxxo Worxxd 2 Hello World ll x Hexo World 3 Hello World not_found x Hello World 4 Hello World [eo] x Hxllx Wxrld 5 Hello World . x xxxxxxxxxxx +1 Hello World l xx Hexxxxo Worxxd +2 Hello World ll x Hexo World +3 Hello World not_found x Hello World +4 Hello World [eo] x Hxllx Wxrld +5 Hello World . x xxxxxxxxxxx ** replaceRegexpOne() ** - non-const needle, const replacement 1 Hello World l x Hexlo World @@ -62,16 +107,31 @@ 3 Hello World not_found x Hello World 4 Hello World [eo] x Hxllo World 5 Hello World . x xello World +1 Hello World l x Hexlo World +2 Hello World ll x Hexo World +3 Hello World not_found x Hello World +4 Hello World [eo] x Hxllo World +5 Hello World . x xello World - const needle, non-const replacement 1 Hello World l xx Hexxlo World 2 Hello World l x Hexlo World 3 Hello World l x Hexlo World 4 Hello World l x Hexlo World 5 Hello World l x Hexlo World +1 Hello World l xx Hexxlo World +2 Hello World l x Hexlo World +3 Hello World l x Hexlo World +4 Hello World l x Hexlo World +5 Hello World l x Hexlo World - non-const needle, non-const replacement 1 Hello World l xx Hexxlo World 2 Hello World ll x Hexo World 3 Hello World not_found x Hello World 4 Hello World [eo] x Hxllo World 5 Hello World . x xello World +1 Hello World l xx Hexxlo World +2 Hello World ll x Hexo World +3 Hello World not_found x Hello World +4 Hello World [eo] x Hxllo World +5 Hello World . x xello World Check that an exception is thrown if the needle is empty diff --git a/tests/queries/0_stateless/02536_replace_with_nonconst_needle_and_replacement.sql b/tests/queries/0_stateless/02536_replace_with_nonconst_needle_and_replacement.sql index 7406f0309bb..926bde3a74b 100644 --- a/tests/queries/0_stateless/02536_replace_with_nonconst_needle_and_replacement.sql +++ b/tests/queries/0_stateless/02536_replace_with_nonconst_needle_and_replacement.sql @@ -9,53 +9,63 @@ CREATE TABLE test_tab INSERT INTO test_tab VALUES (1, 'Hello World', 'l', 'xx') (2, 'Hello World', 'll', 'x') (3, 'Hello World', 'not_found', 'x') (4, 'Hello World', '[eo]', 'x') (5, 'Hello World', '.', 'x') + SELECT '** replaceAll() **'; SELECT '- non-const needle, const replacement'; SELECT id, haystack, needle, 'x', replaceAll(haystack, needle, 'x') FROM test_tab ORDER BY id; +SELECT id, haystack, needle, 'x', replaceAll('Hello World', needle, 'x') FROM test_tab ORDER BY id; SELECT '- const needle, non-const replacement'; SELECT id, haystack, 'l', replacement, replaceAll(haystack, 'l', replacement) FROM test_tab ORDER BY id; +SELECT id, haystack, 'l', replacement, replaceAll('Hello World', 'l', replacement) FROM test_tab ORDER BY id; SELECT '- non-const needle, non-const replacement'; SELECT id, haystack, needle, replacement, replaceAll(haystack, needle, replacement) FROM test_tab ORDER BY id; +SELECT id, haystack, needle, replacement, replaceAll('Hello World', needle, replacement) FROM test_tab ORDER BY id; + SELECT '** replaceOne() **'; SELECT '- non-const needle, const replacement'; SELECT id, haystack, needle, 'x', replaceOne(haystack, needle, 'x') FROM test_tab ORDER BY id; - +SELECT id, haystack, needle, 'x', replaceOne('Hello World', needle, 'x') FROM test_tab ORDER BY id; SELECT '- const needle, non-const replacement'; SELECT id, haystack, 'l', replacement, replaceOne(haystack, 'l', replacement) FROM test_tab ORDER BY id; - +SELECT id, haystack, 'l', replacement, replaceOne('Hello World', 'l', replacement) FROM test_tab ORDER BY id; SELECT '- non-const needle, non-const replacement'; SELECT id, haystack, needle, replacement, replaceOne(haystack, needle, replacement) FROM test_tab ORDER BY id; +SELECT id, haystack, needle, replacement, replaceOne('Hello World', needle, replacement) FROM test_tab ORDER BY id; SELECT '** replaceRegexpAll() **'; SELECT '- non-const needle, const replacement'; SELECT id, haystack, needle, 'x', replaceRegexpAll(haystack, needle, 'x') FROM test_tab ORDER BY id; +SELECT id, haystack, needle, 'x', replaceRegexpAll('Hello World', needle, 'x') FROM test_tab ORDER BY id; SELECT '- const needle, non-const replacement'; SELECT id, haystack, 'l', replacement, replaceRegexpAll(haystack, 'l', replacement) FROM test_tab ORDER BY id; +SELECT id, haystack, 'l', replacement, replaceRegexpAll('Hello World', 'l', replacement) FROM test_tab ORDER BY id; SELECT '- non-const needle, non-const replacement'; SELECT id, haystack, needle, replacement, replaceRegexpAll(haystack, needle, replacement) FROM test_tab ORDER BY id; +SELECT id, haystack, needle, replacement, replaceRegexpAll('Hello World', needle, replacement) FROM test_tab ORDER BY id; SELECT '** replaceRegexpOne() **'; SELECT '- non-const needle, const replacement'; SELECT id, haystack, needle, 'x', replaceRegexpOne(haystack, needle, 'x') FROM test_tab ORDER BY id; - +SELECT id, haystack, needle, 'x', replaceRegexpOne('Hello World', needle, 'x') FROM test_tab ORDER BY id; SELECT '- const needle, non-const replacement'; SELECT id, haystack, 'l', replacement, replaceRegexpOne(haystack, 'l', replacement) FROM test_tab ORDER BY id; - +SELECT id, haystack, 'l', replacement, replaceRegexpOne('Hello World', 'l', replacement) FROM test_tab ORDER BY id; SELECT '- non-const needle, non-const replacement'; SELECT id, haystack, needle, replacement, replaceRegexpOne(haystack, needle, replacement) FROM test_tab ORDER BY id; +SELECT id, haystack, needle, replacement, replaceRegexpOne('Hello World', needle, replacement) FROM test_tab ORDER BY id; DROP TABLE IF EXISTS test_tab; diff --git a/tests/queries/0_stateless/02542_case_no_else.reference b/tests/queries/0_stateless/02542_case_no_else.reference new file mode 100644 index 00000000000..8f3fdf29168 --- /dev/null +++ b/tests/queries/0_stateless/02542_case_no_else.reference @@ -0,0 +1,3 @@ +2 +1 Z +1 Z diff --git a/tests/queries/0_stateless/02542_case_no_else.sql b/tests/queries/0_stateless/02542_case_no_else.sql new file mode 100644 index 00000000000..0c7975a750e --- /dev/null +++ b/tests/queries/0_stateless/02542_case_no_else.sql @@ -0,0 +1,14 @@ +SELECT CASE 1 WHEN 1 THEN 2 END; + +SELECT id, + CASE id + WHEN 1 THEN 'Z' + END x +FROM (SELECT 1 as id); + +SELECT id, + CASE id + WHEN 1 THEN 'Z' + ELSE 'X' + END x +FROM (SELECT 1 as id); diff --git a/tests/queries/0_stateless/02542_transform_new.reference b/tests/queries/0_stateless/02542_transform_new.reference new file mode 100644 index 00000000000..b6eaa692c41 --- /dev/null +++ b/tests/queries/0_stateless/02542_transform_new.reference @@ -0,0 +1,32 @@ +1 +1 +1 +1 +9 +9 +\N +7 +1 +9 +7 +b +b +b +b +a +a +\N +c +sep1 +80000 +80000 +sep2 +80000 +80000 +sep3 +1 +sep4 +8000 +sep5 +8000 +sep6 diff --git a/tests/queries/0_stateless/02542_transform_new.sql b/tests/queries/0_stateless/02542_transform_new.sql new file mode 100644 index 00000000000..43da0a50731 --- /dev/null +++ b/tests/queries/0_stateless/02542_transform_new.sql @@ -0,0 +1,35 @@ +select transform(2, [1,2], [9,1], materialize(null)); +select transform(2, [1,2], [9,1], materialize(7)); +select transform(2, [1,2], [9,1], null); +select transform(2, [1,2], [9,1], 7); +select transform(1, [1,2], [9,1], null); +select transform(1, [1,2], [9,1], 7); +select transform(5, [1,2], [9,1], null); +select transform(5, [1,2], [9,1], 7); +select transform(2, [1,2], [9,1]); +select transform(1, [1,2], [9,1]); +select transform(7, [1,2], [9,1]); + +select transform(2, [1,2], ['a','b'], materialize(null)); +select transform(2, [1,2], ['a','b'], materialize('c')); +select transform(2, [1,2], ['a','b'], null); +select transform(2, [1,2], ['a','b'], 'c'); +select transform(1, [1,2], ['a','b'], null); +select transform(1, [1,2], ['a','b'], 'c'); +select transform(5, [1,2], ['a','b'], null); +select transform(5, [1,2], ['a','b'], 'c'); + +select 'sep1'; +SELECT transform(number, [2], [toDecimal32(1, 1)], materialize(80000)) as x FROM numbers(2); +select 'sep2'; +SELECT transform(number, [2], [toDecimal32(1, 1)], 80000) as x FROM numbers(2); +select 'sep3'; +SELECT transform(toDecimal32(2, 1), [toDecimal32(2, 1)], [1]); +select 'sep4'; +SELECT transform(8000, [1], [toDecimal32(2, 1)]); +select 'sep5'; +SELECT transform(toDecimal32(8000,0), [1], [toDecimal32(2, 1)]); +select 'sep6'; +SELECT transform(-9223372036854775807, [-1], [toDecimal32(1024, 3)]) FROM system.numbers LIMIT 7; -- { serverError BAD_ARGUMENTS } +SELECT [NULL, NULL, NULL, NULL], transform(number, [2147483648], [toDecimal32(1, 2)]) AS x FROM numbers(257) WHERE materialize(10); -- { serverError BAD_ARGUMENTS } +SELECT transform(-2147483649, [1], [toDecimal32(1, 2)]) GROUP BY [1] WITH TOTALS; -- { serverError BAD_ARGUMENTS } diff --git a/tests/queries/0_stateless/02542_transform_old.reference b/tests/queries/0_stateless/02542_transform_old.reference new file mode 100644 index 00000000000..d03b17d40a3 --- /dev/null +++ b/tests/queries/0_stateless/02542_transform_old.reference @@ -0,0 +1,72 @@ +google +other +yahoo +yandex +#1 +20 +21 +22 +29 +#2 +0 +1 +3 +5 +7 +8 +9 +20 +21 +29 +#3 +20 +21 +22 +29 +#4 +google +other +yahoo +yandex +#5 +0 +1 +3 +5 +7 +8 +9 +google +yahoo +yandex +---- +google +other +yahoo +yandex +#1 +20 +21 +22 +29 +#3 +20 +21 +22 +29 +#4 +google +other +yahoo +yandex +---- +2000 +2100 +2200 +2900 +#1 +2000 +2100 +2200 +2900 +---- diff --git a/tests/queries/0_stateless/02542_transform_old.sql b/tests/queries/0_stateless/02542_transform_old.sql new file mode 100644 index 00000000000..01a960ec367 --- /dev/null +++ b/tests/queries/0_stateless/02542_transform_old.sql @@ -0,0 +1,25 @@ +SELECT transform(number, [2, 4, 6], ['google', 'yandex', 'yahoo'], 'other') as x FROM numbers(10) GROUP BY x ORDER BY x; +SELECT '#1'; +SELECT transform(number, [2, 4, 6], [29, 20, 21], 22) as x FROM numbers(10) GROUP BY x ORDER BY x; +SELECT '#2'; +SELECT transform(number, [2, 4, 6], [29, 20, 21]) as x FROM numbers(10) GROUP BY x ORDER BY x; +SELECT '#3'; +SELECT transform(toString(number), ['2', '4', '6'], [29, 20, 21], 22) as x FROM numbers(10) GROUP BY x ORDER BY x; +SELECT '#4'; +SELECT transform(toString(number), ['2', '4', '6'], ['google', 'yandex', 'yahoo'], 'other') as x FROM numbers(10) GROUP BY x ORDER BY x; +SELECT '#5'; +SELECT transform(toString(number), ['2', '4', '6'], ['google', 'yandex', 'yahoo']) as x FROM numbers(10) GROUP BY x ORDER BY x; +SELECT '----'; +SELECT transform(number, [2, 4, 6], ['google', 'yandex', 'yahoo'], materialize('other')) as x FROM numbers(10) GROUP BY x ORDER BY x; +SELECT '#1'; +SELECT transform(number, [2, 4, 6], [29, 20, 21], materialize(22)) as x FROM numbers(10) GROUP BY x ORDER BY x; +SELECT '#3'; +SELECT transform(toString(number), ['2', '4', '6'], [29, 20, 21], materialize(22)) as x FROM numbers(10) GROUP BY x ORDER BY x; +SELECT '#4'; +SELECT transform(toString(number), ['2', '4', '6'], ['google', 'yandex', 'yahoo'], materialize('other')) as x FROM numbers(10) GROUP BY x ORDER BY x; +SELECT '----'; +SELECT transform(number, [2, 4, 6], [2900, 2000, 2100], 2200) as x FROM numbers(10) GROUP BY x ORDER BY x; +SELECT '#1'; +SELECT transform(number, [2, 4, 6], [2900, 2000, 2100], materialize(2200)) as x FROM numbers(10) GROUP BY x ORDER BY x; +SELECT '----'; +SELECT transform(number, [1], [null]) FROM system.numbers LIMIT 1; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } diff --git a/tests/queries/0_stateless/02560_window_ntile.reference b/tests/queries/0_stateless/02560_window_ntile.reference index cae0586fa8c..1045fc1011a 100644 --- a/tests/queries/0_stateless/02560_window_ntile.reference +++ b/tests/queries/0_stateless/02560_window_ntile.reference @@ -22,7 +22,28 @@ select a, b, ntile(3) over (partition by a order by b rows between unbounded pre 1 7 3 1 8 3 1 9 3 -select a, b, ntile(2) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); +select a, b, ntile(3) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); +0 0 1 +0 1 1 +0 2 1 +0 3 1 +0 4 2 +0 5 2 +0 6 2 +0 7 3 +0 8 3 +0 9 3 +1 0 1 +1 1 1 +1 2 1 +1 3 1 +1 4 2 +1 5 2 +1 6 2 +1 7 3 +1 8 3 +1 9 3 +select a, b, ntile(2) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); 0 0 1 0 1 1 0 2 1 @@ -43,7 +64,7 @@ select a, b, ntile(2) over (partition by a order by b rows between unbounded pre 1 7 2 1 8 2 1 9 2 -select a, b, ntile(1) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); +select a, b, ntile(1) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); 0 0 1 0 1 1 0 2 1 @@ -64,7 +85,7 @@ select a, b, ntile(1) over (partition by a order by b rows between unbounded pre 1 7 1 1 8 1 1 9 1 -select a, b, ntile(100) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); +select a, b, ntile(100) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); 0 0 1 0 1 2 0 2 3 @@ -85,7 +106,7 @@ select a, b, ntile(100) over (partition by a order by b rows between unbounded p 1 7 8 1 8 9 1 9 10 -select a, b, ntile(65535) over (partition by a order by b rows between unbounded preceding and unbounded following) from (select 1 as a, number as b from numbers(65535)) limit 100; +select a, b, ntile(65535) over (partition by a order by b) from (select 1 as a, number as b from numbers(65535)) limit 100; 1 0 1 1 1 2 1 2 3 @@ -187,11 +208,11 @@ select a, b, ntile(65535) over (partition by a order by b rows between unbounded 1 98 99 1 99 100 -- Bad arguments -select a, b, ntile(3.0) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } -select a, b, ntile('2') over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } -select a, b, ntile(0) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } -select a, b, ntile(-2) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } -select a, b, ntile(b + 1) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } +select a, b, ntile(3.0) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } +select a, b, ntile('2') over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } +select a, b, ntile(0) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } +select a, b, ntile(-2) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } +select a, b, ntile(b + 1) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } -- Bad window type select a, b, ntile(2) over (partition by a) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } select a, b, ntile(2) over (partition by a order by b rows between 4 preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } diff --git a/tests/queries/0_stateless/02560_window_ntile.sql b/tests/queries/0_stateless/02560_window_ntile.sql index 4c25ecf4dd2..f2acf8fc94e 100644 --- a/tests/queries/0_stateless/02560_window_ntile.sql +++ b/tests/queries/0_stateless/02560_window_ntile.sql @@ -2,17 +2,20 @@ -- Normal cases select a, b, ntile(3) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -select a, b, ntile(2) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -select a, b, ntile(1) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -select a, b, ntile(100) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -select a, b, ntile(65535) over (partition by a order by b rows between unbounded preceding and unbounded following) from (select 1 as a, number as b from numbers(65535)) limit 100; +select a, b, ntile(3) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); +select a, b, ntile(2) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); +select a, b, ntile(1) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); +select a, b, ntile(100) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); +select a, b, ntile(65535) over (partition by a order by b) from (select 1 as a, number as b from numbers(65535)) limit 100; + + -- Bad arguments -select a, b, ntile(3.0) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } -select a, b, ntile('2') over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } -select a, b, ntile(0) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } -select a, b, ntile(-2) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } -select a, b, ntile(b + 1) over (partition by a order by b rows between unbounded preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } +select a, b, ntile(3.0) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } +select a, b, ntile('2') over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } +select a, b, ntile(0) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } +select a, b, ntile(-2) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } +select a, b, ntile(b + 1) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } -- Bad window type select a, b, ntile(2) over (partition by a) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError 36 } diff --git a/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks.reference b/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks.reference index 18e83d1244a..452e0e0801e 100644 --- a/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks.reference +++ b/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks.reference @@ -17,3 +17,42 @@ SELECT count(), _part FROM 02581_trips WHERE description = '' GROUP BY _part ORD 8000 all_2_2_0_6 8000 all_3_3_0_6 8000 all_4_4_0_6 +-- Run mutation with `id 'IN big subquery' +ALTER TABLE 02581_trips UPDATE description='a' WHERE id IN (SELECT (number*10)::UInt32 FROM numbers(10000000)) SETTINGS mutations_sync=2; +SELECT count() from 02581_trips WHERE description = ''; +28000 +ALTER TABLE 02581_trips UPDATE description='a' WHERE id IN (SELECT (number*10 + 1)::UInt32 FROM numbers(10000000)) SETTINGS mutations_sync=2, max_rows_in_set=1000; +SELECT count() from 02581_trips WHERE description = ''; +28000 +-- Run mutation with func(`id`) IN big subquery +ALTER TABLE 02581_trips UPDATE description='b' WHERE id::UInt64 IN (SELECT (number*10 + 2)::UInt32 FROM numbers(10000000)) SETTINGS mutations_sync=2; +SELECT count() from 02581_trips WHERE description = ''; +28000 +-- Run mutation with non-PK `id2` IN big subquery +ALTER TABLE 02581_trips UPDATE description='c' WHERE id2 IN (SELECT (number*10 + 3)::UInt32 FROM numbers(10000000)) SETTINGS mutations_sync=2; +SELECT count() from 02581_trips WHERE description = ''; +24000 +-- Run mutation with PK and non-PK IN big subquery +ALTER TABLE 02581_trips UPDATE description='c' +WHERE + (id IN (SELECT (number*10 + 4)::UInt32 FROM numbers(10000000))) OR + (id2 IN (SELECT (number*10 + 4)::UInt32 FROM numbers(10000000))) +SETTINGS mutations_sync=2; +SELECT count() from 02581_trips WHERE description = ''; +20000 +-- Run mutation with PK and non-PK IN big subquery +ALTER TABLE 02581_trips UPDATE description='c' +WHERE + (id::UInt64 IN (SELECT (number*10 + 5)::UInt32 FROM numbers(10000000))) OR + (id2::UInt64 IN (SELECT (number*10 + 5)::UInt32 FROM numbers(10000000))) +SETTINGS mutations_sync=2; +SELECT count() from 02581_trips WHERE description = ''; +16000 +-- Run mutation with PK and non-PK IN big subquery +ALTER TABLE 02581_trips UPDATE description='c' +WHERE + (id::UInt32 IN (SELECT (number*10 + 6)::UInt32 FROM numbers(10000000))) OR + ((id2+1)::String IN (SELECT (number*10 + 6)::UInt32 FROM numbers(10000000))) +SETTINGS mutations_sync=2; +SELECT count() from 02581_trips WHERE description = ''; +12000 diff --git a/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks.sql b/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks.sql index fc90582d20e..7b52a89b16f 100644 --- a/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks.sql +++ b/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks.sql @@ -1,12 +1,12 @@ DROP TABLE IF EXISTS 02581_trips; -CREATE TABLE 02581_trips(id UInt32, description String) ENGINE=MergeTree ORDER BY id; +CREATE TABLE 02581_trips(id UInt32, id2 UInt32, description String) ENGINE=MergeTree ORDER BY id; -- Make multiple parts -INSERT INTO 02581_trips SELECT number, '' FROM numbers(10000); -INSERT INTO 02581_trips SELECT number+10000, '' FROM numbers(10000); -INSERT INTO 02581_trips SELECT number+20000, '' FROM numbers(10000); -INSERT INTO 02581_trips SELECT number+30000, '' FROM numbers(10000); +INSERT INTO 02581_trips SELECT number, number, '' FROM numbers(10000); +INSERT INTO 02581_trips SELECT number+10000, number+10000, '' FROM numbers(10000); +INSERT INTO 02581_trips SELECT number+20000, number+20000, '' FROM numbers(10000); +INSERT INTO 02581_trips SELECT number+30000, number+30000, '' FROM numbers(10000); -- { echoOn } SELECT count(), _part FROM 02581_trips GROUP BY _part ORDER BY _part; @@ -16,6 +16,45 @@ ALTER TABLE 02581_trips UPDATE description='1' WHERE id IN (SELECT (number*10+1) SELECT count(), _part FROM 02581_trips WHERE description = '' GROUP BY _part ORDER BY _part; ALTER TABLE 02581_trips UPDATE description='2' WHERE id IN (SELECT (number*10+2)::UInt32 FROM numbers(10000)) SETTINGS mutations_sync=2; SELECT count(), _part FROM 02581_trips WHERE description = '' GROUP BY _part ORDER BY _part; + +-- Run mutation with `id 'IN big subquery' +ALTER TABLE 02581_trips UPDATE description='a' WHERE id IN (SELECT (number*10)::UInt32 FROM numbers(10000000)) SETTINGS mutations_sync=2; +SELECT count() from 02581_trips WHERE description = ''; + +ALTER TABLE 02581_trips UPDATE description='a' WHERE id IN (SELECT (number*10 + 1)::UInt32 FROM numbers(10000000)) SETTINGS mutations_sync=2, max_rows_in_set=1000; +SELECT count() from 02581_trips WHERE description = ''; + +-- Run mutation with func(`id`) IN big subquery +ALTER TABLE 02581_trips UPDATE description='b' WHERE id::UInt64 IN (SELECT (number*10 + 2)::UInt32 FROM numbers(10000000)) SETTINGS mutations_sync=2; +SELECT count() from 02581_trips WHERE description = ''; + +-- Run mutation with non-PK `id2` IN big subquery +ALTER TABLE 02581_trips UPDATE description='c' WHERE id2 IN (SELECT (number*10 + 3)::UInt32 FROM numbers(10000000)) SETTINGS mutations_sync=2; +SELECT count() from 02581_trips WHERE description = ''; + +-- Run mutation with PK and non-PK IN big subquery +ALTER TABLE 02581_trips UPDATE description='c' +WHERE + (id IN (SELECT (number*10 + 4)::UInt32 FROM numbers(10000000))) OR + (id2 IN (SELECT (number*10 + 4)::UInt32 FROM numbers(10000000))) +SETTINGS mutations_sync=2; +SELECT count() from 02581_trips WHERE description = ''; + +-- Run mutation with PK and non-PK IN big subquery +ALTER TABLE 02581_trips UPDATE description='c' +WHERE + (id::UInt64 IN (SELECT (number*10 + 5)::UInt32 FROM numbers(10000000))) OR + (id2::UInt64 IN (SELECT (number*10 + 5)::UInt32 FROM numbers(10000000))) +SETTINGS mutations_sync=2; +SELECT count() from 02581_trips WHERE description = ''; + +-- Run mutation with PK and non-PK IN big subquery +ALTER TABLE 02581_trips UPDATE description='c' +WHERE + (id::UInt32 IN (SELECT (number*10 + 6)::UInt32 FROM numbers(10000000))) OR + ((id2+1)::String IN (SELECT (number*10 + 6)::UInt32 FROM numbers(10000000))) +SETTINGS mutations_sync=2; +SELECT count() from 02581_trips WHERE description = ''; -- { echoOff } DROP TABLE 02581_trips; diff --git a/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.sql b/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.sql index 97cf979e80a..21ff453cd8e 100644 --- a/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.sql +++ b/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.sql @@ -1,4 +1,4 @@ --- Tags: long, no-tsan, no-asan, no-ubsan, no-msan +-- Tags: long, no-debug, no-tsan, no-asan, no-ubsan, no-msan DROP TABLE IF EXISTS 02581_trips; diff --git a/tests/queries/0_stateless/02596_build_set_and_remote.reference b/tests/queries/0_stateless/02596_build_set_and_remote.reference new file mode 100644 index 00000000000..8d12196ae33 --- /dev/null +++ b/tests/queries/0_stateless/02596_build_set_and_remote.reference @@ -0,0 +1,19 @@ +-- {echoOn} +SELECT arrayExists(x -> (x IN (SELECT '2')), [2]) FROM system.one; +1 +SELECT arrayExists(x -> (x IN (SELECT '2')), [2]) FROM remote('127.0.0.{2,3}', system.one); +1 +1 +SELECT arrayExists(x -> (x IN (SELECT '2')), [2]) FROM remote('127.0.0.{2,3}', system.one) GROUP BY NULL; +1 +SELECT arrayExists(x -> (x IN (SELECT '2')), [2]) FROM remote('127.0.0.{2,3}', system.one) GROUP BY 1; +1 +SELECT arrayExists(x -> (x IN (SELECT '2')), [2]) FROM remote('127.0.0.{2,3}', system.one) GROUP BY 'A'; +1 +SELECT 1 IN ( SELECT 1 ) FROM remote('127.0.0.{1,2}', system.one) GROUP BY dummy; +1 +SELECT 1000.0001, toUInt64(arrayJoin([NULL, 257, 65536, NULL])), arrayExists(x -> (x IN (SELECT '2.55')), [-9223372036854775808]) FROM remote('127.0.0.{1,2}', system.one) GROUP BY NULL, NULL, NULL, NULL; +1000.0001 \N 0 +1000.0001 257 0 +1000.0001 65536 0 +1000.0001 \N 0 diff --git a/tests/queries/0_stateless/02596_build_set_and_remote.sql b/tests/queries/0_stateless/02596_build_set_and_remote.sql new file mode 100644 index 00000000000..7a904344c91 --- /dev/null +++ b/tests/queries/0_stateless/02596_build_set_and_remote.sql @@ -0,0 +1,14 @@ +-- {echoOn} +SELECT arrayExists(x -> (x IN (SELECT '2')), [2]) FROM system.one; + +SELECT arrayExists(x -> (x IN (SELECT '2')), [2]) FROM remote('127.0.0.{2,3}', system.one); + +SELECT arrayExists(x -> (x IN (SELECT '2')), [2]) FROM remote('127.0.0.{2,3}', system.one) GROUP BY NULL; + +SELECT arrayExists(x -> (x IN (SELECT '2')), [2]) FROM remote('127.0.0.{2,3}', system.one) GROUP BY 1; + +SELECT arrayExists(x -> (x IN (SELECT '2')), [2]) FROM remote('127.0.0.{2,3}', system.one) GROUP BY 'A'; + +SELECT 1 IN ( SELECT 1 ) FROM remote('127.0.0.{1,2}', system.one) GROUP BY dummy; + +SELECT 1000.0001, toUInt64(arrayJoin([NULL, 257, 65536, NULL])), arrayExists(x -> (x IN (SELECT '2.55')), [-9223372036854775808]) FROM remote('127.0.0.{1,2}', system.one) GROUP BY NULL, NULL, NULL, NULL; diff --git a/tests/queries/0_stateless/02661_quantile_approx.reference b/tests/queries/0_stateless/02661_quantile_approx.reference index f4e66adc8d9..8369363aa9b 100644 --- a/tests/queries/0_stateless/02661_quantile_approx.reference +++ b/tests/queries/0_stateless/02661_quantile_approx.reference @@ -19,8 +19,10 @@ select quantilesGK(1000, 100/1000, 200/1000, 250/1000, 314/1000, 777/1000)(numbe [99,199,249,313,776] select quantilesGK(10000, 100/1000, 200/1000, 250/1000, 314/1000, 777/1000)(number + 1) from numbers(1000); [100,200,250,314,777] -select medianGK()(number) from numbers(10); -- { serverError BAD_ARGUMENTS } -select quantileGK()(number) from numbers(10); -- { serverError BAD_ARGUMENTS } +select medianGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 0; -- { serverError BAD_ARGUMENTS } +select medianGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 1; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +select quantileGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 0; -- { serverError BAD_ARGUMENTS } +select quantileGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 1; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } select medianGK(100)(number) from numbers(10); 4 select quantileGK(100)(number) from numbers(10); @@ -31,7 +33,8 @@ select quantileGK(100, 0.5, 0.75)(number) from numbers(10); -- { serverError NUM select quantileGK('abc', 0.5)(number) from numbers(10); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } select quantileGK(1.23, 0.5)(number) from numbers(10); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } select quantileGK(-100, 0.5)(number) from numbers(10); -- { serverError BAD_ARGUMENTS } -select quantilesGK()(number) from numbers(10); -- { serverError BAD_ARGUMENTS } +select quantilesGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 0; -- { serverError BAD_ARGUMENTS } +select quantilesGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 1; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } select quantilesGK(100)(number) from numbers(10); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } select quantilesGK(100, 0.5)(number) from numbers(10); [4] diff --git a/tests/queries/0_stateless/02661_quantile_approx.sql b/tests/queries/0_stateless/02661_quantile_approx.sql index 18c2e5de84b..52c2979ad44 100644 --- a/tests/queries/0_stateless/02661_quantile_approx.sql +++ b/tests/queries/0_stateless/02661_quantile_approx.sql @@ -1,3 +1,5 @@ +set allow_experimental_analyzer = 1; + -- { echoOn } with arrayJoin([0, 1, 2, 10]) as x select quantilesGK(100, 0.5, 0.4, 0.1)(x); with arrayJoin([0, 6, 7, 9, 10]) as x select quantileGK(100, 0.5)(x); @@ -14,8 +16,12 @@ select quantilesGK(1000, 100/1000, 200/1000, 250/1000, 314/1000, 777/1000)(numbe select quantilesGK(10000, 100/1000, 200/1000, 250/1000, 314/1000, 777/1000)(number + 1) from numbers(1000); -select medianGK()(number) from numbers(10); -- { serverError BAD_ARGUMENTS } -select quantileGK()(number) from numbers(10); -- { serverError BAD_ARGUMENTS } +select medianGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 0; -- { serverError BAD_ARGUMENTS } +select medianGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 1; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } + +select quantileGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 0; -- { serverError BAD_ARGUMENTS } +select quantileGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 1; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } + select medianGK(100)(number) from numbers(10); select quantileGK(100)(number) from numbers(10); select quantileGK(100, 0.5)(number) from numbers(10); @@ -24,7 +30,9 @@ select quantileGK('abc', 0.5)(number) from numbers(10); -- { serverError ILLEGAL select quantileGK(1.23, 0.5)(number) from numbers(10); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } select quantileGK(-100, 0.5)(number) from numbers(10); -- { serverError BAD_ARGUMENTS } -select quantilesGK()(number) from numbers(10); -- { serverError BAD_ARGUMENTS } +select quantilesGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 0; -- { serverError BAD_ARGUMENTS } +select quantilesGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 1; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } + select quantilesGK(100)(number) from numbers(10); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } select quantilesGK(100, 0.5)(number) from numbers(10); select quantilesGK('abc', 0.5, 0.75)(number) from numbers(10); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } diff --git a/tests/queries/0_stateless/02677_analyzer_bitmap_has_any.sql b/tests/queries/0_stateless/02677_analyzer_bitmap_has_any.sql index 4af06634c66..f0f9845d91d 100644 --- a/tests/queries/0_stateless/02677_analyzer_bitmap_has_any.sql +++ b/tests/queries/0_stateless/02677_analyzer_bitmap_has_any.sql @@ -18,7 +18,7 @@ FROM bitmapHasAny(bitmapBuild([toUInt64(1)]), ( SELECT groupBitmapState(toUInt64(2)) )) has2 -); -- { serverError 43 } +) SETTINGS allow_experimental_analyzer = 0; -- { serverError 43 } SELECT '--------------'; diff --git a/tests/queries/0_stateless/02680_mysql_ast_logical_err.sql b/tests/queries/0_stateless/02680_mysql_ast_logical_err.sql index 5b0530e05ae..bde91df83ca 100644 --- a/tests/queries/0_stateless/02680_mysql_ast_logical_err.sql +++ b/tests/queries/0_stateless/02680_mysql_ast_logical_err.sql @@ -1,2 +1,4 @@ +CREATE TABLE foo (key UInt32, a String, b Int64, c String) ENGINE = TinyLog; + SELECT count() FROM mysql(mysql('127.0.0.1:9004', currentDatabase(), 'foo', 'default', ''), '127.0.0.1:9004', currentDatabase(), 'foo', '', ''); -- { serverError UNKNOWN_FUNCTION } --- SELECT count() FROM mysql(mysql('127.0.0.1:9004', currentDatabase(), 'foo', 'default', '', SETTINGS connection_pool_size = 1), '127.0.0.1:9004', currentDatabase(), 'foo', '', ''); -- { serverError UNKNOWN_FUNCTION } +SELECT count() FROM mysql(mysql('127.0.0.1:9004', currentDatabase(), 'foo', 'default', '', SETTINGS connection_pool_size = 1), '127.0.0.1:9004', currentDatabase(), 'foo', '', ''); -- { serverError UNKNOWN_FUNCTION, UNSUPPORTED_METHOD } diff --git a/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.reference b/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.reference index 346025b277b..35c94347ac9 100644 --- a/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.reference +++ b/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.reference @@ -2,6 +2,8 @@ [] [[(2147483647,0),(10.0001,65535),(1,255),(1023,2147483646)]] [[[(2147483647,0),(10.0001,65535),(1023,2147483646),(2147483647,0)]]] [[(2147483647,0),(10.0001,65535),(1,255),(1023,2147483646)]] [] +[[(2147483647,0),(10.0001,65535),(1,255),(1023,2147483646)]] [[[(2147483647,0),(10.0001,65535),(1023,2147483646),(2147483647,0)]]] +[[(2147483647,0),(10.0001,65535),(1,255),(1023,2147483646)]] [[[(2147483647,0),(10.0001,65535),(1023,2147483646),(2147483647,0)]]] [[[(100.0001,1000.0001),(1000.0001,1.1920928955078125e-7),(20,-20),(20,20),(10,10),(-20,20),(100.0001,1000.0001)]]] [[[(100.0001,1000.0001),(1000.0001,1.1920928955078125e-7),(20,-20),(20,20),(10,10),(-20,20),(100.0001,1000.0001)]]] [(9223372036854775807,1.1754943508222875e-38)] [[(1,1.0001)]] \N [] diff --git a/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.sql b/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.sql index 8b9b63f7996..85307bec6e5 100644 --- a/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.sql +++ b/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.sql @@ -1,5 +1,5 @@ - SELECT polygonsSymDifferenceCartesian([[[(1., 1.)]] AS x], [x]) GROUP BY x WITH ROLLUP; -SELECT [[(2147483647, 0.), (10.0001, 65535), (1, 255), (1023, 2147483646)]], polygonsSymDifferenceCartesian([[[(2147483647, 0.), (10.0001, 65535), (1023, 2147483646)]]], [[[(1000.0001, 10.0001)]]]) GROUP BY [[(2147483647, 0.), (10.0001, 65535), (1023, 2147483646)]] WITH ROLLUP; +SELECT [[(2147483647, 0.), (10.0001, 65535), (1, 255), (1023, 2147483646)]], polygonsSymDifferenceCartesian([[[(2147483647, 0.), (10.0001, 65535), (1023, 2147483646)]]], [[[(1000.0001, 10.0001)]]]) GROUP BY [[(2147483647, 0.), (10.0001, 65535), (1023, 2147483646)]] WITH ROLLUP SETTINGS allow_experimental_analyzer=0; +SELECT [[(2147483647, 0.), (10.0001, 65535), (1, 255), (1023, 2147483646)]], polygonsSymDifferenceCartesian([[[(2147483647, 0.), (10.0001, 65535), (1023, 2147483646)]]], [[[(1000.0001, 10.0001)]]]) GROUP BY [[(2147483647, 0.), (10.0001, 65535), (1023, 2147483646)]] WITH ROLLUP SETTINGS allow_experimental_analyzer=1; SELECT polygonsSymDifferenceCartesian([[[(100.0001, 1000.0001), (-20., 20.), (10., 10.), (20., 20.), (20., -20.), (1000.0001, 1.1920928955078125e-7)]],[[(0.0001, 100000000000000000000.)]] AS x],[x]) GROUP BY x WITH ROLLUP; SELECT [(9223372036854775807, 1.1754943508222875e-38)], x, NULL, polygonsSymDifferenceCartesian([[[(1.1754943508222875e-38, 1.1920928955078125e-7), (0.5, 0.5)]], [[(1.1754943508222875e-38, 1.1920928955078125e-7), (1.1754943508222875e-38, 1.1920928955078125e-7)], [(0., 1.0001)]], [[(1., 1.0001)]] AS x], [[[(3.4028234663852886e38, 0.9999)]]]) GROUP BY GROUPING SETS ((x)) WITH TOTALS diff --git a/tests/queries/0_stateless/02703_jit_external_aggregation.reference b/tests/queries/0_stateless/02703_jit_external_aggregation.reference index cdeec60f4ef..9c558e357c4 100644 --- a/tests/queries/0_stateless/02703_jit_external_aggregation.reference +++ b/tests/queries/0_stateless/02703_jit_external_aggregation.reference @@ -1 +1 @@ -..... +. diff --git a/tests/queries/0_stateless/02703_jit_external_aggregation.sh b/tests/queries/0_stateless/02703_jit_external_aggregation.sh index 2d1dda45de0..4bc17c106fb 100755 --- a/tests/queries/0_stateless/02703_jit_external_aggregation.sh +++ b/tests/queries/0_stateless/02703_jit_external_aggregation.sh @@ -5,11 +5,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -# This query should return empty result in every of five runs: - -for _ in {1..5} -do - $CLICKHOUSE_CLIENT --compile_aggregate_expressions 0 --query " +# This query should return empty result +$CLICKHOUSE_CLIENT --compile_aggregate_expressions 1 --min_count_to_compile_aggregate_expression=0 --query " SELECT COUNT() AS c, group_key, @@ -30,6 +27,5 @@ ORDER BY group_key ASC LIMIT 10 SETTINGS max_bytes_before_external_group_by = 200000 " && echo -n '.' -done echo diff --git a/tests/queries/0_stateless/02707_complex_query_fails_analyzer.reference b/tests/queries/0_stateless/02707_complex_query_fails_analyzer.reference deleted file mode 100644 index 192f8aa904a..00000000000 --- a/tests/queries/0_stateless/02707_complex_query_fails_analyzer.reference +++ /dev/null @@ -1,10 +0,0 @@ -1 1 -59.952 -1 2 59.952 -1 3 -100 -2 1 -93.7611 -2 2 93.7611 -3 1 0 -3 2 0 ---------- -0 -0 diff --git a/tests/queries/0_stateless/02707_complex_query_fails_analyzer.sql b/tests/queries/0_stateless/02707_complex_query_fails_analyzer.sql deleted file mode 100644 index a9d83479d50..00000000000 --- a/tests/queries/0_stateless/02707_complex_query_fails_analyzer.sql +++ /dev/null @@ -1,117 +0,0 @@ -DROP TABLE IF EXISTS srv_account_parts; -DROP TABLE IF EXISTS etl_batch; - -CREATE TABLE srv_account_parts( - shard_num UInt16, - account_ids Array(Int64) -)ENGINE = ReplacingMergeTree -ORDER BY shard_num -as select * from values ((0,[]),(1,[1,2,3]),(2,[1,2,3]),(3,[1])); - -CREATE TABLE etl_batch( - batch_id UInt64, - batch_start DateTime, - batch_start_day Date DEFAULT toDate(batch_start), - batch_load DateTime, - total_num_records UInt32, - etl_server_id Int32, - account_id UInt64, - shard_num UInt16 -)ENGINE = ReplacingMergeTree -PARTITION BY toYYYYMM(batch_start_day) -ORDER BY (batch_id, etl_server_id, account_id); - -insert into etl_batch(batch_id, batch_start, batch_load, total_num_records, etl_server_id, account_id, shard_num) -select number batch_id, - toDateTime('2022-01-01') + INTERVAL 23 HOUR batch_start, - batch_start batch_load, - 333 total_num_records, - 1 etl_server_id, - number%3+1 account_id, - 1 shard_num -from numbers(1000); - -insert into etl_batch(batch_id, batch_start, batch_load, total_num_records, etl_server_id, account_id, shard_num) -select number+2000 batch_id, - toDateTime('2022-01-01') + INTERVAL 23 HOUR batch_start, - batch_start batch_load, - 333 total_num_records, - 1 etl_server_id, - number%3+1 account_id, - 2 shard_num -from numbers(1000); - -insert into etl_batch(batch_id, batch_start, batch_load, total_num_records, etl_server_id, account_id, shard_num) -select number+4000 batch_id, - toDateTime('2022-01-01') + INTERVAL 3 HOUR batch_start, - batch_start batch_load, - 3333 total_num_records, - 1 etl_server_id, - 2 account_id, - 2 shard_num -from numbers(1000); - -insert into etl_batch(batch_id, batch_start, batch_load, total_num_records, etl_server_id, account_id, shard_num) -select number+6000 batch_id, - toDateTime('2022-01-01') + INTERVAL 23 HOUR batch_start, - batch_start batch_load, - 333 total_num_records, - 1 etl_server_id, - 1 account_id, - 2 shard_num -from numbers(1000); - -insert into etl_batch(batch_id, batch_start, batch_load, total_num_records, etl_server_id, account_id, shard_num) -select number+8000 batch_id, - toDateTime('2022-01-01') + INTERVAL 23 HOUR batch_start, - batch_start batch_load, - 1000 total_num_records, - 1 etl_server_id, - 3 account_id, - 3 shard_num -from numbers(1000); - -CREATE OR REPLACE VIEW v_num_records_by_node_bias_acc as -SELECT shard_num, - arrayJoin(account_ids) AS account_id, - records_24h, - records_12h, - IF (b = '',-100,xbias) AS bias, - IF (bias > 10,0,IF (bias > 0,1,IF (bias < -10,301,300))) AS sbias -FROM srv_account_parts - LEFT JOIN (SELECT account_id, - shard_num, - records_24h, - records_12h, - xbias, - 'b' AS b - FROM (SELECT account_id, - groupArray((shard_num,records_24h,records_12h)) AS ga, - arraySum(ga.2) AS tot24, - arraySum(ga.3) AS tot12, - arrayMap(i ->(((((i.2)*LENGTH(ga))*100) / tot24) - 100),ga) AS bias24, - arrayMap(i ->(((((i.3)*LENGTH(ga))*100) / tot12) - 100),ga) AS bias12, - arrayMap((i,j,k) ->(i,IF (tot12 = 0,0,IF (ABS(j) > ABS(k),j,k))),ga,bias24,bias12) AS a_bias - FROM (SELECT shard_num, - toInt64(account_id) AS account_id, - SUM(total_num_records) AS records_24h, - sumIf(total_num_records,batch_load >(toDateTime('2022-01-02') -(3600*12))) AS records_12h - FROM etl_batch FINAL PREWHERE (batch_start_day >= (toDate('2022-01-02') - 2)) AND (batch_load > (toDateTime('2022-01-02') - (3600*24))) - where (shard_num, account_id) in (select shard_num, arrayJoin(account_ids) from srv_account_parts) - GROUP BY shard_num,account_id) - GROUP BY account_id) - ARRAY JOIN (a_bias.1).1 AS shard_num,a_bias.2 AS xbias, (a_bias.1).2 AS records_24h, (a_bias.1).3 AS records_12h - ) s USING (shard_num,account_id); - -select account_id, shard_num, round(bias,4) -from v_num_records_by_node_bias_acc -order by account_id, shard_num, bias; - -select '---------'; - -SELECT a AS b FROM (SELECT 0 a) s LEFT JOIN (SELECT 0 b) t USING (b); - -SELECT arrayJoin(a) AS b FROM (SELECT [0] a) s LEFT JOIN (SELECT 0 b) t USING (b); - -DROP TABLE srv_account_parts; -DROP TABLE etl_batch; diff --git a/tests/queries/0_stateless/02710_allow_suspicious_indices.reference b/tests/queries/0_stateless/02710_allow_suspicious_indices.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02710_allow_suspicious_indices.sql b/tests/queries/0_stateless/02710_allow_suspicious_indices.sql new file mode 100644 index 00000000000..78d52f7bc72 --- /dev/null +++ b/tests/queries/0_stateless/02710_allow_suspicious_indices.sql @@ -0,0 +1,22 @@ +-- Check CREATE TABLE + +DROP TABLE IF EXISTS tbl; +CREATE TABLE tbl (id UInt32) ENGINE = MergeTree() ORDER BY (id + 1, id + 1); -- { serverError BAD_ARGUMENTS } +CREATE TABLE tbl (id UInt32) ENGINE = MergeTree() ORDER BY (id + 1, id + 1) SETTINGS allow_suspicious_indices = 1; + +DROP TABLE IF EXISTS tbl; +CREATE TABLE tbl (id UInt32, INDEX idx (id + 1, id + 1) TYPE minmax) ENGINE = MergeTree() ORDER BY id; -- { serverError BAD_ARGUMENTS } +CREATE TABLE tbl (id UInt32, INDEX idx (id + 1, id + 1) TYPE minmax) ENGINE = MergeTree() ORDER BY id SETTINGS allow_suspicious_indices = 1; + +-- Check ALTER TABLE + +DROP TABLE IF EXISTS tbl; +CREATE TABLE tbl (id1 UInt32) ENGINE = MergeTree() ORDER BY id1; +ALTER TABLE tbl ADD COLUMN `id2` UInt32, MODIFY ORDER BY (id1, id2, id2); -- { serverError BAD_ARGUMENTS } +ALTER TABLE tbl ADD COLUMN `id2` UInt32, MODIFY ORDER BY (id1, id2, id1); -- { serverError BAD_ARGUMENTS } +ALTER TABLE tbl ADD COLUMN `id2` UInt32, MODIFY ORDER BY (id1, id2, id2) SETTINGS allow_suspicious_indices = 1; + +DROP TABLE IF EXISTS tbl; +CREATE TABLE tbl (id UInt32) ENGINE = MergeTree() ORDER BY id; +ALTER TABLE tbl ADD INDEX idx (id+1, id, id+1) TYPE minmax; -- { serverError BAD_ARGUMENTS } +ALTER TABLE tbl ADD INDEX idx (id+1, id, id+1) TYPE minmax SETTINGS allow_suspicious_indices = 1; diff --git a/tests/queries/0_stateless/02711_server_uuid_macro.sql b/tests/queries/0_stateless/02711_server_uuid_macro.sql index f10ed7f8f6f..4f562ad36bf 100644 --- a/tests/queries/0_stateless/02711_server_uuid_macro.sql +++ b/tests/queries/0_stateless/02711_server_uuid_macro.sql @@ -12,4 +12,4 @@ CREATE TABLE test2 (x UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{d -- The macro {server_uuid} is special, not a configuration-type macro. It's normal that it is inaccessible with the getMacro function. SELECT getMacro('server_uuid'); -- { serverError NO_ELEMENTS_IN_CONFIG } -DROP TABLE test NO DELAY; +DROP TABLE test SYNC; diff --git a/tests/queries/0_stateless/02713_create_user_substitutions.reference b/tests/queries/0_stateless/02713_create_user_substitutions.reference new file mode 100644 index 00000000000..f9b5cc495b5 --- /dev/null +++ b/tests/queries/0_stateless/02713_create_user_substitutions.reference @@ -0,0 +1,11 @@ +1 +2 +3 +4 +5 +6 +7 +8 +CREATE USER user9_02713 IDENTIFIED WITH ldap SERVER \'qwerty9\' +CREATE USER user10_02713 IDENTIFIED WITH kerberos REALM \'qwerty10\' +CREATE USER user11_02713 IDENTIFIED WITH ssl_certificate CN \'qwerty11\', \'qwerty12\' diff --git a/tests/queries/0_stateless/02713_create_user_substitutions.sh b/tests/queries/0_stateless/02713_create_user_substitutions.sh new file mode 100755 index 00000000000..42926335acb --- /dev/null +++ b/tests/queries/0_stateless/02713_create_user_substitutions.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "DROP USER IF EXISTS user1_02713, user2_02713, user3_02713, user4_02713, user5_02713, user6_02713, user7_02713"; + +$CLICKHOUSE_CLIENT --param_password=qwerty1 -q "CREATE USER user1_02713 IDENTIFIED BY {password:String}"; +$CLICKHOUSE_CLIENT --param_password=qwerty2 -q "CREATE USER user2_02713 IDENTIFIED WITH PLAINTEXT_PASSWORD BY {password:String}"; +$CLICKHOUSE_CLIENT --param_password=qwerty3 -q "CREATE USER user3_02713 IDENTIFIED WITH SHA256_PASSWORD BY {password:String}"; +$CLICKHOUSE_CLIENT --param_password=qwerty4 -q "CREATE USER user4_02713 IDENTIFIED WITH DOUBLE_SHA1_PASSWORD BY {password:String}"; +$CLICKHOUSE_CLIENT --param_password=qwerty5 -q "CREATE USER user5_02713 IDENTIFIED WITH BCRYPT_PASSWORD BY {password:String}"; + +# Generated online +$CLICKHOUSE_CLIENT --param_hash=310cef2caff72c0224f38ca8e2141ca6012cd4da550c692573c25a917d9a75e6 \ + -q "CREATE USER user6_02713 IDENTIFIED WITH SHA256_HASH BY {hash:String}"; +# Generated with ClickHouse +$CLICKHOUSE_CLIENT --param_hash=5886A74C452575627522F3A80D8B9E239FD8955F \ + -q "CREATE USER user7_02713 IDENTIFIED WITH DOUBLE_SHA1_HASH BY {hash:String}"; +# Generated online +$CLICKHOUSE_CLIENT --param_hash=\$2a\$12\$wuohz0HFSBBNE8huN0Yx6.kmWrefiYVKeMp4gsuNoO1rOWwF2FXXC \ + -q "CREATE USER user8_02713 IDENTIFIED WITH BCRYPT_HASH BY {hash:String}"; + +$CLICKHOUSE_CLIENT --param_server=qwerty9 -q "CREATE USER user9_02713 IDENTIFIED WITH LDAP SERVER {server:String}"; +$CLICKHOUSE_CLIENT --param_realm=qwerty10 -q "CREATE USER user10_02713 IDENTIFIED WITH KERBEROS REALM {realm:String}"; +$CLICKHOUSE_CLIENT --param_cert1=qwerty11 --param_cert2=qwerty12 -q "CREATE USER user11_02713 IDENTIFIED WITH SSL_CERTIFICATE CN {cert1:String}, {cert2:String}"; + +$CLICKHOUSE_CLIENT --user=user1_02713 --password=qwerty1 -q "SELECT 1"; +$CLICKHOUSE_CLIENT --user=user2_02713 --password=qwerty2 -q "SELECT 2"; +$CLICKHOUSE_CLIENT --user=user3_02713 --password=qwerty3 -q "SELECT 3"; +$CLICKHOUSE_CLIENT --user=user4_02713 --password=qwerty4 -q "SELECT 4"; +$CLICKHOUSE_CLIENT --user=user5_02713 --password=qwerty5 -q "SELECT 5"; +$CLICKHOUSE_CLIENT --user=user6_02713 --password=qwerty6 -q "SELECT 6"; +$CLICKHOUSE_CLIENT --user=user7_02713 --password=qwerty7 -q "SELECT 7"; +$CLICKHOUSE_CLIENT --user=user8_02713 --password=qwerty8 -q "SELECT 8"; + +$CLICKHOUSE_CLIENT -q "SHOW CREATE USER user9_02713"; +$CLICKHOUSE_CLIENT -q "SHOW CREATE USER user10_02713"; +$CLICKHOUSE_CLIENT -q "SHOW CREATE USER user11_02713"; + +$CLICKHOUSE_CLIENT -q "DROP USER user1_02713, user2_02713, user3_02713, user4_02713, user5_02713, user6_02713, user7_02713, user8_02713, user9_02713, user10_02713, user11_02713"; diff --git a/tests/queries/0_stateless/02713_sequence_match_serialization_fix.reference b/tests/queries/0_stateless/02713_sequence_match_serialization_fix.reference new file mode 100644 index 00000000000..2a1c127e635 --- /dev/null +++ b/tests/queries/0_stateless/02713_sequence_match_serialization_fix.reference @@ -0,0 +1,3 @@ +serialized state is not used 1 +serialized state is used 1 +via Distributed 1 diff --git a/tests/queries/0_stateless/02713_sequence_match_serialization_fix.sql b/tests/queries/0_stateless/02713_sequence_match_serialization_fix.sql new file mode 100644 index 00000000000..3521cb8470f --- /dev/null +++ b/tests/queries/0_stateless/02713_sequence_match_serialization_fix.sql @@ -0,0 +1,36 @@ +DROP TABLE IF EXISTS 02713_seqt; +DROP TABLE IF EXISTS 02713_seqt_distr; + +SELECT + 'serialized state is not used', sequenceMatch('(?1)(?2)')(time, number_ = 1, number_ = 0) AS seq +FROM +( + SELECT + number AS time, + number % 2 AS number_ + FROM numbers_mt(100) +); + + +CREATE TABLE 02713_seqt +ENGINE = MergeTree +ORDER BY n AS +SELECT + sequenceMatchState('(?1)(?2)')(time, number_ = 1, number_ = 0) AS seq, + 1 AS n +FROM +( + SELECT + number AS time, + number % 2 AS number_ + FROM numbers_mt(100) +); + + +SELECT 'serialized state is used', sequenceMatchMerge('(?1)(?2)')(seq) AS seq +FROM 02713_seqt; + + +CREATE TABLE 02713_seqt_distr ( seq AggregateFunction(sequenceMatch('(?1)(?2)'), UInt64, UInt8, UInt8) , n UInt8) ENGINE = Distributed(test_shard_localhost, currentDatabase(), '02713_seqt'); + +SELECT 'via Distributed', sequenceMatchMerge('(?1)(?2)')(seq) AS seq FROM 02713_seqt_distr; diff --git a/tests/queries/0_stateless/02714_date_date32_in.reference b/tests/queries/0_stateless/02714_date_date32_in.reference new file mode 100644 index 00000000000..d9ff83f1949 --- /dev/null +++ b/tests/queries/0_stateless/02714_date_date32_in.reference @@ -0,0 +1,4 @@ +1 +1 +0 +0 diff --git a/tests/queries/0_stateless/02714_date_date32_in.sql b/tests/queries/0_stateless/02714_date_date32_in.sql new file mode 100644 index 00000000000..69a087eff6f --- /dev/null +++ b/tests/queries/0_stateless/02714_date_date32_in.sql @@ -0,0 +1,4 @@ +select toDate32('2020-01-01') in (toDate('2020-01-01')); +select toDate('2020-01-01') in (toDate32('2020-01-01')); +select toDate('2020-01-01') in 1::Int64; +select toDate32('2020-01-01') in 1::UInt64; diff --git a/tests/queries/0_stateless/02714_local_object_storage.reference b/tests/queries/0_stateless/02714_local_object_storage.reference new file mode 100644 index 00000000000..b3f28057554 --- /dev/null +++ b/tests/queries/0_stateless/02714_local_object_storage.reference @@ -0,0 +1,2 @@ +1 test +1 test diff --git a/tests/queries/0_stateless/02714_local_object_storage.sql b/tests/queries/0_stateless/02714_local_object_storage.sql new file mode 100644 index 00000000000..fa9025b8b6e --- /dev/null +++ b/tests/queries/0_stateless/02714_local_object_storage.sql @@ -0,0 +1,28 @@ +SET min_bytes_to_use_direct_io='1Gi'; -- It does not work (fixme) +SET local_filesystem_read_method='pread'; -- ui_uring local_fs_method does not work here (fixme) + +DROP TABLE IF EXISTS test; + +CREATE TABLE test (a Int32, b String) +ENGINE = MergeTree() ORDER BY tuple() +SETTINGS disk = disk( + type = 'local_blob_storage', + path = '/var/lib/clickhouse/disks/${CLICKHOUSE_TEST_UNIQUE_NAME}/'); + +INSERT INTO test SELECT 1, 'test'; +SELECT * FROM test; + +DROP TABLE test SYNC; + +CREATE TABLE test (a Int32, b String) +ENGINE = MergeTree() ORDER BY tuple() +SETTINGS disk = disk( + type = 'cache', + max_size = '10Mi', + path = '/var/lib/clickhouse/caches/${CLICKHOUSE_TEST_UNIQUE_NAME}/', + disk = disk(type='local_blob_storage', path='/var/lib/clickhouse/disks/${CLICKHOUSE_TEST_UNIQUE_NAME}/')); + +INSERT INTO test SELECT 1, 'test'; +SELECT * FROM test; + +DROP TABLE test SYNC; diff --git a/tests/queries/0_stateless/02714_read_bytes_aggregateFunction.reference b/tests/queries/0_stateless/02714_read_bytes_aggregateFunction.reference new file mode 100644 index 00000000000..d315d85a11e --- /dev/null +++ b/tests/queries/0_stateless/02714_read_bytes_aggregateFunction.reference @@ -0,0 +1,6 @@ +UInt64 1 8 +UInt64 10 80 +UInt64 1000 8000 +AggregateFunction(argMax, String, DateTime) 1 80 +AggregateFunction(argMax, String, DateTime) 10 800 +AggregateFunction(argMax, String, DateTime) 1000 80000 diff --git a/tests/queries/0_stateless/02714_read_bytes_aggregateFunction.sql b/tests/queries/0_stateless/02714_read_bytes_aggregateFunction.sql new file mode 100644 index 00000000000..26bc9ebe62b --- /dev/null +++ b/tests/queries/0_stateless/02714_read_bytes_aggregateFunction.sql @@ -0,0 +1,59 @@ +CREATE TABLE test (id UInt64, `amax` AggregateFunction(argMax, String, DateTime)) +ENGINE=MergeTree() +ORDER BY id +SETTINGS ratio_of_defaults_for_sparse_serialization=1 -- Sparse columns will take more bytes for a single row +AS + SELECT number, argMaxState(number::String, '2023-04-12 16:23:01'::DateTime) + FROM numbers(1) + GROUP BY number; + +SELECT sum(id) FROM test FORMAT Null; +SELECT argMaxMerge(amax) FROM test FORMAT Null; + +INSERT INTO test + SELECT number, argMaxState(number::String, '2023-04-12 16:23:01'::DateTime) + FROM numbers(9) + GROUP BY number; + +SELECT sum(id) FROM test FORMAT Null; +SELECT argMaxMerge(amax) FROM test FORMAT Null; + +INSERT INTO test +SELECT number, argMaxState(number::String, '2023-04-12 16:23:01'::DateTime) +FROM numbers(990) +GROUP BY number; + +SELECT sum(id) FROM test FORMAT Null; +SELECT argMaxMerge(amax) FROM test FORMAT Null; + +SYSTEM FLUSH LOGS; + +SELECT 'UInt64', + read_rows, + read_bytes +FROM system.query_log +WHERE + current_database = currentDatabase() AND + query = 'SELECT sum(id) FROM test FORMAT Null;' AND + type = 2 AND event_date >= yesterday() +ORDER BY event_time_microseconds; + +-- Size of ColumnAggregateFunction: Number of pointers * pointer size + arena size +-- 1 * 8 + AggregateFunction(argMax, String, DateTime) +-- +-- Size of AggregateFunction(argMax, String, DateTime): +-- SingleValueDataString() + SingleValueDataFixed(DateTime) +-- SingleValueDataString = 64B for small strings, 64B + string size + 1 for larger +-- SingleValueDataFixed(DateTime) = 1 + 4. With padding = 8 +-- SingleValueDataString Total: 72B +-- +-- ColumnAggregateFunction total: 8 + 72 = 80 +SELECT 'AggregateFunction(argMax, String, DateTime)', + read_rows, + read_bytes +FROM system.query_log +WHERE + current_database = currentDatabase() AND + query = 'SELECT argMaxMerge(amax) FROM test FORMAT Null;' AND + type = 2 AND event_date >= yesterday() +ORDER BY event_time_microseconds; diff --git a/tests/queries/0_stateless/02720_s3_strict_upload_part_size.reference b/tests/queries/0_stateless/02720_s3_strict_upload_part_size.reference new file mode 100644 index 00000000000..360b484bf28 --- /dev/null +++ b/tests/queries/0_stateless/02720_s3_strict_upload_part_size.reference @@ -0,0 +1,4 @@ +Size: 6000001 +Size: 6000001 +Size: 6000001 +Size: 2971517 diff --git a/tests/queries/0_stateless/02720_s3_strict_upload_part_size.sh b/tests/queries/0_stateless/02720_s3_strict_upload_part_size.sh new file mode 100755 index 00000000000..69e2f734914 --- /dev/null +++ b/tests/queries/0_stateless/02720_s3_strict_upload_part_size.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, long +# Tag no-fasttest: requires S3 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +in="$CUR_DIR/$CLICKHOUSE_TEST_UNIQUE_NAME.in" +out="$CUR_DIR/$CLICKHOUSE_TEST_UNIQUE_NAME.out" +log="$CUR_DIR/$CLICKHOUSE_TEST_UNIQUE_NAME.log" + +set -e +trap 'rm -f "${out:?}" "${in:?}" "${log:?}"' EXIT + +# Generate a file of 20MiB in size, with our part size it will have 4 parts +# NOTE: 1 byte is for new line, so 1023 not 1024 +$CLICKHOUSE_LOCAL -q "SELECT randomPrintableASCII(1023) FROM numbers(20*1024) FORMAT LineAsString" > "$in" + +$CLICKHOUSE_CLIENT --send_logs_level=trace --server_logs_file="$log" -q "INSERT INTO FUNCTION s3(s3_conn, filename='$CLICKHOUSE_TEST_UNIQUE_NAME', format='LineAsString', structure='line String') FORMAT LineAsString" --s3_strict_upload_part_size=6000001 < "$in" +grep -F '' "$log" || : +grep -o 'WriteBufferFromS3: Writing part.*Size: .*' "$log" | grep -o 'Size: .*' +$CLICKHOUSE_CLIENT -q "SELECT * FROM s3(s3_conn, filename='$CLICKHOUSE_TEST_UNIQUE_NAME', format='LineAsString', structure='line String') FORMAT LineAsString" > "$out" + +diff -q "$in" "$out" diff --git a/tests/queries/0_stateless/02723_jit_aggregation_bug_48120.reference b/tests/queries/0_stateless/02723_jit_aggregation_bug_48120.reference new file mode 100644 index 00000000000..6f9b4b4fc6a --- /dev/null +++ b/tests/queries/0_stateless/02723_jit_aggregation_bug_48120.reference @@ -0,0 +1,7 @@ +-- { echoOn } +SYSTEM DROP COMPILED EXPRESSION CACHE; +SELECT minIf(num1, num1 < 5) FROM dummy GROUP BY num2; +0 +SYSTEM DROP COMPILED EXPRESSION CACHE; +SELECT minIf(num1, num1 >= 5) FROM dummy GROUP BY num2; +5 diff --git a/tests/queries/0_stateless/02723_jit_aggregation_bug_48120.sql b/tests/queries/0_stateless/02723_jit_aggregation_bug_48120.sql new file mode 100644 index 00000000000..04e0fc5e0ba --- /dev/null +++ b/tests/queries/0_stateless/02723_jit_aggregation_bug_48120.sql @@ -0,0 +1,17 @@ +-- Tags: no-fasttest, no-ubsan, no-cpu-aarch64 + +drop table if exists dummy; +CREATE TABLE dummy ( num1 Int32, num2 Enum8('foo' = 0, 'bar' = 1, 'tar' = 2) ) +ENGINE = MergeTree ORDER BY num1 as select 5, 'bar'; + +set compile_aggregate_expressions=1; +set min_count_to_compile_aggregate_expression=0; + +-- { echoOn } +SYSTEM DROP COMPILED EXPRESSION CACHE; +SELECT minIf(num1, num1 < 5) FROM dummy GROUP BY num2; +SYSTEM DROP COMPILED EXPRESSION CACHE; +SELECT minIf(num1, num1 >= 5) FROM dummy GROUP BY num2; +-- { echoOff } + +drop table dummy; diff --git a/tests/queries/0_stateless/02723_parallelize_output_setting.reference b/tests/queries/0_stateless/02723_parallelize_output_setting.reference new file mode 100644 index 00000000000..0f2a396f471 --- /dev/null +++ b/tests/queries/0_stateless/02723_parallelize_output_setting.reference @@ -0,0 +1,7 @@ +-- { echoOn } +set parallelize_output_from_storages=1; +select startsWith(trimLeft(explain),'Resize') as resize from (explain pipeline select * from file(data_02723.csv)) where resize; +1 +-- no Resize in pipeline +set parallelize_output_from_storages=0; +select startsWith(trimLeft(explain),'Resize') as resize from (explain pipeline select * from file(data_02723.csv)) where resize; diff --git a/tests/queries/0_stateless/02723_parallelize_output_setting.sql b/tests/queries/0_stateless/02723_parallelize_output_setting.sql new file mode 100644 index 00000000000..7db28ca4dec --- /dev/null +++ b/tests/queries/0_stateless/02723_parallelize_output_setting.sql @@ -0,0 +1,12 @@ +-- Tags: no-parallel + +insert into function file(data_02723.csv) select number from numbers(5) settings engine_file_truncate_on_insert=1; + +set max_threads=2; +-- { echoOn } +set parallelize_output_from_storages=1; +select startsWith(trimLeft(explain),'Resize') as resize from (explain pipeline select * from file(data_02723.csv)) where resize; +-- no Resize in pipeline +set parallelize_output_from_storages=0; +select startsWith(trimLeft(explain),'Resize') as resize from (explain pipeline select * from file(data_02723.csv)) where resize; + diff --git a/tests/queries/0_stateless/02723_zookeeper_name.reference b/tests/queries/0_stateless/02723_zookeeper_name.reference new file mode 100644 index 00000000000..074712bd8fe --- /dev/null +++ b/tests/queries/0_stateless/02723_zookeeper_name.reference @@ -0,0 +1,4 @@ +Create Tables +Insert Data +"t1","default",1 +"t2","default",1 diff --git a/tests/queries/0_stateless/02723_zookeeper_name.sql b/tests/queries/0_stateless/02723_zookeeper_name.sql new file mode 100644 index 00000000000..7ddbf4edd47 --- /dev/null +++ b/tests/queries/0_stateless/02723_zookeeper_name.sql @@ -0,0 +1,23 @@ +-- Tags: zookeeper, replica + +SELECT 'Create Tables'; +CREATE TABLE t1(k UInt32, v UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/{database}/test_02723/zookeeper_name/t1', '1') ORDER BY k; + +CREATE TABLE t2(k UInt32, v UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/{database}/test_02723/zookeeper_name/t2', '1') ORDER BY k; + +SELECT 'Insert Data'; + +INSERT INTO t1 SELECT * FROM generateRandom('k UInt32, v UInt32') LIMIT 1; +INSERT INTO t2 SELECT * FROM generateRandom('k UInt32, v UInt32') LIMIT 1; + +SELECT + table,zookeeper_name,count() +FROM system.replicas +INNER JOIN system.parts USING (database, table) +WHERE database = currentDatabase() +GROUP BY + table,zookeeper_name +FORMAT CSV; + +DROP TABLE t1; +DROP TABLE t2; diff --git a/tests/queries/0_stateless/02724_decompress_filename_exception.reference b/tests/queries/0_stateless/02724_decompress_filename_exception.reference new file mode 100644 index 00000000000..f9c5aacff7b --- /dev/null +++ b/tests/queries/0_stateless/02724_decompress_filename_exception.reference @@ -0,0 +1,8 @@ +Ok +Ok +Ok +Ok +Ok +Ok +Ok +Ok diff --git a/tests/queries/0_stateless/02724_decompress_filename_exception.sh b/tests/queries/0_stateless/02724_decompress_filename_exception.sh new file mode 100755 index 00000000000..bbc2b8d066b --- /dev/null +++ b/tests/queries/0_stateless/02724_decompress_filename_exception.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +USER_FILES_PATH=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') +FILENAME="${USER_FILES_PATH}/corrupted_file.tsv.xx" + +echo 'corrupted file' > $FILENAME; + +$CLICKHOUSE_CLIENT --query "SELECT * FROM file('${FILENAME}', 'TSV', 'c UInt32', 'gzip')" 2>&1 | grep -q "While reading from: $FILENAME" && echo 'Ok' || echo 'Fail'; +$CLICKHOUSE_CLIENT --query "SELECT * FROM file('${FILENAME}', 'TSV', 'c UInt32', 'deflate')" 2>&1 | grep -q "While reading from: $FILENAME" && echo 'Ok' || echo 'Fail'; +$CLICKHOUSE_CLIENT --query "SELECT * FROM file('${FILENAME}', 'TSV', 'c UInt32', 'br')" 2>&1 | grep -q "While reading from: $FILENAME" && echo 'Ok' || echo 'Fail'; +$CLICKHOUSE_CLIENT --query "SELECT * FROM file('${FILENAME}', 'TSV', 'c UInt32', 'xz')" 2>&1 | grep -q "While reading from: $FILENAME" && echo 'Ok' || echo 'Fail'; +$CLICKHOUSE_CLIENT --query "SELECT * FROM file('${FILENAME}', 'TSV', 'c UInt32', 'zstd')" 2>&1 | grep -q "While reading from: $FILENAME" && echo 'Ok' || echo 'Fail'; +$CLICKHOUSE_CLIENT --query "SELECT * FROM file('${FILENAME}', 'TSV', 'c UInt32', 'lz4')" 2>&1 | grep -q "While reading from: $FILENAME" && echo 'Ok' || echo 'Fail'; +$CLICKHOUSE_CLIENT --query "SELECT * FROM file('${FILENAME}', 'TSV', 'c UInt32', 'bz2')" 2>&1 | grep -q "While reading from: $FILENAME" && echo 'Ok' || echo 'Fail'; +$CLICKHOUSE_CLIENT --query "SELECT * FROM file('${FILENAME}', 'TSV', 'c UInt32', 'snappy')" 2>&1 | grep -q "While reading from: $FILENAME" && echo 'Ok' || echo 'Fail'; + +rm $FILENAME; diff --git a/tests/queries/0_stateless/02724_function_in_left_table_clause_asof_join.reference b/tests/queries/0_stateless/02724_function_in_left_table_clause_asof_join.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02724_function_in_left_table_clause_asof_join.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02724_function_in_left_table_clause_asof_join.sql b/tests/queries/0_stateless/02724_function_in_left_table_clause_asof_join.sql new file mode 100644 index 00000000000..13dfb5debe7 --- /dev/null +++ b/tests/queries/0_stateless/02724_function_in_left_table_clause_asof_join.sql @@ -0,0 +1,8 @@ +select count(*) +from ( + select 1 as id, [1, 2, 3] as arr +) as sessions +ASOF LEFT JOIN ( + select 1 as session_id, 4 as id +) as visitors +ON visitors.session_id <= sessions.id AND arrayFirst(a -> a, arrayMap((a) -> a, sessions.arr)) = visitors.id diff --git a/tests/queries/0_stateless/02724_jit_logical_functions.reference b/tests/queries/0_stateless/02724_jit_logical_functions.reference new file mode 100644 index 00000000000..673ffe02613 --- /dev/null +++ b/tests/queries/0_stateless/02724_jit_logical_functions.reference @@ -0,0 +1,18 @@ +Logical functions not null +0 0 0 0 0 +0 1 0 1 1 +1 0 0 1 1 +1 1 1 1 0 +Logical functions nullable +0 0 0 0 0 +0 1 0 1 1 +1 0 0 1 1 +1 1 1 1 0 +0 \N 0 \N \N +1 \N \N 1 \N +0 0 0 +1 1 0 +0 0 0 +1 1 0 +\N \N \N +\N \N \N diff --git a/tests/queries/0_stateless/02724_jit_logical_functions.sql b/tests/queries/0_stateless/02724_jit_logical_functions.sql new file mode 100644 index 00000000000..fe6646337d0 --- /dev/null +++ b/tests/queries/0_stateless/02724_jit_logical_functions.sql @@ -0,0 +1,21 @@ +SET compile_expressions = 1; +SET min_count_to_compile_expression = 0; + +DROP TABLE IF EXISTS test_table; +CREATE TABLE test_table (a UInt8, b UInt8) ENGINE = TinyLog; +INSERT INTO test_table VALUES (0, 0), (0, 1), (1, 0), (1, 1); + +SELECT 'Logical functions not null'; +SELECT a, b, and(a, b), or(a, b), xor(a, b) FROM test_table; + +DROP TABLE test_table; + +DROP TABLE IF EXISTS test_table_nullable; +CREATE TABLE test_table_nullable (a UInt8, b Nullable(UInt8)) ENGINE = TinyLog; +INSERT INTO test_table_nullable VALUES (0, 0), (0, 1), (1, 0), (1, 1), (0, NULL), (1, NULL); + +SELECT 'Logical functions nullable'; +SELECT a, b, and(a, b), or(a, b), xor(a, b) FROM test_table_nullable; +SELECT and(b, b), or(b, b), xor(b, b) FROM test_table_nullable; + +DROP TABLE test_table_nullable; diff --git a/tests/queries/0_stateless/02724_mutliple_storage_join.reference b/tests/queries/0_stateless/02724_mutliple_storage_join.reference new file mode 100644 index 00000000000..f7eb44d66e0 --- /dev/null +++ b/tests/queries/0_stateless/02724_mutliple_storage_join.reference @@ -0,0 +1,6 @@ +0 +0 +0 +0 +0 +0 diff --git a/tests/queries/0_stateless/02724_mutliple_storage_join.sql b/tests/queries/0_stateless/02724_mutliple_storage_join.sql new file mode 100644 index 00000000000..286e867704d --- /dev/null +++ b/tests/queries/0_stateless/02724_mutliple_storage_join.sql @@ -0,0 +1,21 @@ +CREATE TABLE user(id UInt32, name String) ENGINE = Join(ANY, LEFT, id); +INSERT INTO user VALUES (1,'U1')(2,'U2')(3,'U3'); + +CREATE TABLE product(id UInt32, name String, cate String) ENGINE = Join(ANY, LEFT, id); +INSERT INTO product VALUES (1,'P1','C1')(2,'P2','C1')(3,'P3','C2'); + +CREATE TABLE order(id UInt32, pId UInt32, uId UInt32) ENGINE = TinyLog; +INSERT INTO order VALUES (1,1,1)(2,1,2)(3,2,3); + +SELECT ignore(*) FROM ( + SELECT + uId, + user.id as `uuu` + FROM order + LEFT ANY JOIN user + ON uId = `uuu` +); + +SELECT ignore(*) FROM order +LEFT ANY JOIN user ON uId = user.id +LEFT ANY JOIN product ON pId = product.id; diff --git a/tests/queries/0_stateless/02725_agg_projection_resprect_PK.reference b/tests/queries/0_stateless/02725_agg_projection_resprect_PK.reference new file mode 100644 index 00000000000..e6b95502e1e --- /dev/null +++ b/tests/queries/0_stateless/02725_agg_projection_resprect_PK.reference @@ -0,0 +1,2 @@ + ReadFromMergeTree (p1) + Granules: 1/12 diff --git a/tests/queries/0_stateless/02725_agg_projection_resprect_PK.sql b/tests/queries/0_stateless/02725_agg_projection_resprect_PK.sql new file mode 100644 index 00000000000..a2355f78f4c --- /dev/null +++ b/tests/queries/0_stateless/02725_agg_projection_resprect_PK.sql @@ -0,0 +1,32 @@ +-- Tags: no-random-merge-tree-settings + +DROP TABLE IF EXISTS t0; + +CREATE TABLE t0 +( + c1 Int64, + c2 Int64, + c3 Int64, + PROJECTION p1 + ( + SELECT + c1, + c2, + sum(c3) + GROUP BY + c2, + c1 + ) +) +ENGINE = MergeTree ORDER BY (c1, c2) settings min_bytes_for_wide_part = 10485760, min_rows_for_wide_part = 0; + +INSERT INTO t0 SELECT + number, + -number, + number +FROM numbers_mt(1e5); + +select * from (EXPLAIN indexes = 1 SELECT c1, sum(c3) FROM t0 GROUP BY c1) where explain like '%ReadFromMergeTree%'; +select * from (EXPLAIN indexes = 1 SELECT c1, sum(c3) FROM t0 WHERE c1 = 100 GROUP BY c1) where explain like '%Granules%'; + +DROP TABLE t0; diff --git a/tests/queries/0_stateless/02725_alias_columns_should_not_allow_compression_codec.reference b/tests/queries/0_stateless/02725_alias_columns_should_not_allow_compression_codec.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02725_alias_columns_should_not_allow_compression_codec.sql b/tests/queries/0_stateless/02725_alias_columns_should_not_allow_compression_codec.sql new file mode 100644 index 00000000000..083a3aefdaf --- /dev/null +++ b/tests/queries/0_stateless/02725_alias_columns_should_not_allow_compression_codec.sql @@ -0,0 +1,7 @@ +drop table if exists alias_column_should_not_allow_compression; +create table if not exists alias_column_should_not_allow_compression ( user_id UUID, user_id_hashed ALIAS (cityHash64(user_id))) engine=MergeTree() order by tuple(); +create table if not exists alias_column_should_not_allow_compression_fail ( user_id UUID, user_id_hashed ALIAS (cityHash64(user_id)) codec(LZ4HC(1))) engine=MergeTree() order by tuple(); -- { serverError BAD_ARGUMENTS } +alter table alias_column_should_not_allow_compression modify column user_id codec(LZ4HC(1)); +alter table alias_column_should_not_allow_compression modify column user_id_hashed codec(LZ4HC(1)); -- { serverError BAD_ARGUMENTS } +alter table alias_column_should_not_allow_compression add column user_id_hashed_1 UInt64 ALIAS (cityHash64(user_id)) codec(LZ4HC(1)); -- { serverError BAD_ARGUMENTS } +drop table if exists alias_column_should_not_allow_compression; diff --git a/tests/queries/0_stateless/02725_alias_with_restricted_keywords.reference b/tests/queries/0_stateless/02725_alias_with_restricted_keywords.reference new file mode 100644 index 00000000000..9874d6464ab --- /dev/null +++ b/tests/queries/0_stateless/02725_alias_with_restricted_keywords.reference @@ -0,0 +1 @@ +1 2 diff --git a/tests/queries/0_stateless/02725_alias_with_restricted_keywords.sql b/tests/queries/0_stateless/02725_alias_with_restricted_keywords.sql new file mode 100644 index 00000000000..6df0e856061 --- /dev/null +++ b/tests/queries/0_stateless/02725_alias_with_restricted_keywords.sql @@ -0,0 +1 @@ +SELECT 1 `array`, 2 "union"; diff --git a/tests/queries/0_stateless/02725_async_insert_table_setting.reference b/tests/queries/0_stateless/02725_async_insert_table_setting.reference new file mode 100644 index 00000000000..5f5235c569f --- /dev/null +++ b/tests/queries/0_stateless/02725_async_insert_table_setting.reference @@ -0,0 +1,4 @@ +2 +2 +default.t_mt_async_insert 1 +default.t_mt_sync_insert 0 diff --git a/tests/queries/0_stateless/02725_async_insert_table_setting.sh b/tests/queries/0_stateless/02725_async_insert_table_setting.sh new file mode 100755 index 00000000000..13911e8d677 --- /dev/null +++ b/tests/queries/0_stateless/02725_async_insert_table_setting.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -n --query " +DROP TABLE IF EXISTS t_mt_async_insert; +DROP TABLE IF EXISTS t_mt_sync_insert; + +CREATE TABLE t_mt_async_insert (id UInt64, s String) +ENGINE = MergeTree ORDER BY id SETTINGS async_insert = 1; + +CREATE TABLE t_mt_sync_insert (id UInt64, s String) +ENGINE = MergeTree ORDER BY id SETTINGS async_insert = 0;" + +url="${CLICKHOUSE_URL}&async_insert=0&wait_for_async_insert=1" + +${CLICKHOUSE_CURL} -sS "$url" -d "INSERT INTO t_mt_async_insert VALUES (1, 'aa'), (2, 'bb')" +${CLICKHOUSE_CURL} -sS "$url" -d "INSERT INTO t_mt_sync_insert VALUES (1, 'aa'), (2, 'bb')" + +${CLICKHOUSE_CLIENT} -n --query " +SELECT count() FROM t_mt_async_insert; +SELECT count() FROM t_mt_sync_insert; + +SYSTEM FLUSH LOGS; +SELECT tables[1], ProfileEvents['AsyncInsertQuery'] FROM system.query_log +WHERE + type = 'QueryFinish' AND + current_database = currentDatabase() AND + query ILIKE 'INSERT INTO t_mt_%sync_insert%' +ORDER BY tables[1]; + +DROP TABLE IF EXISTS t_mt_async_insert; +DROP TABLE IF EXISTS t_mt_sync_insert;" diff --git a/tests/queries/0_stateless/02725_keeper_fault_inject_sequential_cleanup.reference b/tests/queries/0_stateless/02725_keeper_fault_inject_sequential_cleanup.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02725_keeper_fault_inject_sequential_cleanup.sql b/tests/queries/0_stateless/02725_keeper_fault_inject_sequential_cleanup.sql new file mode 100644 index 00000000000..e1db4ba2fa6 --- /dev/null +++ b/tests/queries/0_stateless/02725_keeper_fault_inject_sequential_cleanup.sql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS keeper_fault_inject_sequential_cleanup; + +CREATE TABLE keeper_fault_inject_sequential_cleanup (d Int8) ENGINE = ReplicatedMergeTree('/clickhouse/{database}/test_02725/tables/keeper_fault_inject_sequential_cleanup', '1') ORDER BY d; + +INSERT INTO keeper_fault_inject_sequential_cleanup VALUES (1); +INSERT INTO keeper_fault_inject_sequential_cleanup SETTINGS insert_deduplicate = 0 VALUES (1); +INSERT INTO keeper_fault_inject_sequential_cleanup SETTINGS insert_deduplicate = 0, insert_keeper_fault_injection_probability = 0.4, insert_keeper_fault_injection_seed = 5619964844601345291 VALUES (1); + +-- with database ordinary it produced a warning +DROP TABLE keeper_fault_inject_sequential_cleanup; diff --git a/tests/queries/0_stateless/02725_memory-for-merges.reference b/tests/queries/0_stateless/02725_memory-for-merges.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02725_memory-for-merges.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02725_memory-for-merges.sql b/tests/queries/0_stateless/02725_memory-for-merges.sql new file mode 100644 index 00000000000..b6ae7af7f1a --- /dev/null +++ b/tests/queries/0_stateless/02725_memory-for-merges.sql @@ -0,0 +1,27 @@ +-- Tags: no-s3-storage +-- We allocate a lot of memory for buffers when reading or writing to S3 + +DROP TABLE IF EXISTS 02725_memory_for_merges SYNC; + +CREATE TABLE 02725_memory_for_merges +( n UInt64, + s String +) +ENGINE = MergeTree +ORDER BY n +SETTINGS merge_max_block_size_bytes=1024, index_granularity_bytes=1024; + +INSERT INTO 02725_memory_for_merges SELECT number, randomPrintableASCII(1000000) FROM numbers(100); +INSERT INTO 02725_memory_for_merges SELECT number, randomPrintableASCII(1000000) FROM numbers(100); +INSERT INTO 02725_memory_for_merges SELECT number, randomPrintableASCII(1000000) FROM numbers(100); +INSERT INTO 02725_memory_for_merges SELECT number, randomPrintableASCII(1000000) FROM numbers(100); +INSERT INTO 02725_memory_for_merges SELECT number, randomPrintableASCII(1000000) FROM numbers(100); + +OPTIMIZE TABLE 02725_memory_for_merges FINAL; + +SYSTEM FLUSH LOGS; + +WITH (SELECT uuid FROM system.tables WHERE table='02725_memory_for_merges' and database=currentDatabase()) as uuid +SELECT sum(peak_memory_usage) < 1024 * 1024 * 200 from system.part_log where table_uuid=uuid and event_type='MergeParts'; + +DROP TABLE IF EXISTS 02725_memory_for_merges SYNC; diff --git a/tests/queries/0_stateless/02725_start_stop_fetches.reference b/tests/queries/0_stateless/02725_start_stop_fetches.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02725_start_stop_fetches.sh b/tests/queries/0_stateless/02725_start_stop_fetches.sh new file mode 100755 index 00000000000..0ca687ae951 --- /dev/null +++ b/tests/queries/0_stateless/02725_start_stop_fetches.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Tags: race, zookeeper, no-parallel, no-upgrade-check, no-replicated-database + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +set -e + +NUM_REPLICAS=5 + +for i in $(seq 1 $NUM_REPLICAS); do + $CLICKHOUSE_CLIENT -n -q " + DROP TABLE IF EXISTS r$i SYNC; + CREATE TABLE r$i (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', 'r$i') ORDER BY x SETTINGS replicated_deduplication_window = 1, allow_remote_fs_zero_copy_replication = 1; + " +done + +function thread { + while true; do + REPLICA=$(($RANDOM % 5 + 1)) + $CLICKHOUSE_CLIENT --query "INSERT INTO r$REPLICA SELECT rand()" + done +} + +function nemesis_thread1 { + while true; do + REPLICA=$(($RANDOM % 5 + 1)) + $CLICKHOUSE_CLIENT --query "SYSTEM STOP REPLICATED SENDS r$REPLICA" + sleep 0.5 + $CLICKHOUSE_CLIENT --query "SYSTEM START REPLICATED SENDS r$REPLICA" + done +} + +function nemesis_thread2 { + while true; do + REPLICA=$(($RANDOM % 5 + 1)) + $CLICKHOUSE_CLIENT --query "SYSTEM STOP FETCHES r$REPLICA" + sleep 0.5 + $CLICKHOUSE_CLIENT --query "SYSTEM START FETCHES r$REPLICA" + done +} + + + +export -f thread +export -f nemesis_thread1 +export -f nemesis_thread2 + +TIMEOUT=20 + +timeout $TIMEOUT bash -c thread 2>/dev/null & +timeout $TIMEOUT bash -c thread 2>/dev/null & +timeout $TIMEOUT bash -c thread 2>/dev/null & +timeout $TIMEOUT bash -c nemesis_thread1 2>/dev/null & +timeout $TIMEOUT bash -c nemesis_thread1 2>/dev/null & +timeout $TIMEOUT bash -c nemesis_thread1 2>/dev/null & +timeout $TIMEOUT bash -c nemesis_thread2 2>/dev/null & +timeout $TIMEOUT bash -c nemesis_thread2 2>/dev/null & +timeout $TIMEOUT bash -c nemesis_thread2 2>/dev/null & + +wait + + +for i in $(seq 1 $NUM_REPLICAS); do + $CLICKHOUSE_CLIENT -q "SYSTEM START FETCHES r$REPLICA" + $CLICKHOUSE_CLIENT -q "SYSTEM START REPLICATED SENDS r$REPLICA" +done + +for i in $(seq 1 $NUM_REPLICAS); do + $CLICKHOUSE_CLIENT --max_execution_time 60 -q "SYSTEM SYNC REPLICA r$i PULL" +done + +for i in $(seq 1 $NUM_REPLICAS); do + $CLICKHOUSE_CLIENT -q "DROP TABLE r$i" 2>/dev/null & +done + +wait diff --git a/tests/queries/0_stateless/02730_dictionary_hashed_load_factor_element_count.reference b/tests/queries/0_stateless/02730_dictionary_hashed_load_factor_element_count.reference new file mode 100644 index 00000000000..09d337562b5 --- /dev/null +++ b/tests/queries/0_stateless/02730_dictionary_hashed_load_factor_element_count.reference @@ -0,0 +1,2 @@ +dict_sharded 1 1000000 0.4768 +dict_sharded_multi 5 1000000 0.4768 diff --git a/tests/queries/0_stateless/02730_dictionary_hashed_load_factor_element_count.sql b/tests/queries/0_stateless/02730_dictionary_hashed_load_factor_element_count.sql new file mode 100644 index 00000000000..1e42f56889d --- /dev/null +++ b/tests/queries/0_stateless/02730_dictionary_hashed_load_factor_element_count.sql @@ -0,0 +1,17 @@ +DROP DICTIONARY IF EXISTS dict_sharded; +DROP DICTIONARY IF EXISTS dict_sharded_multi; +DROP TABLE IF EXISTS dict_data; + +CREATE TABLE dict_data (key UInt64, v0 UInt16, v1 UInt16, v2 UInt16, v3 UInt16, v4 UInt16) engine=Memory() AS SELECT number, number%65535, number%65535, number%6553, number%655355, number%65535 FROM numbers(1e6); + +CREATE DICTIONARY dict_sharded (key UInt64, v0 UInt16) PRIMARY KEY key SOURCE(CLICKHOUSE(TABLE 'dict_data')) LIFETIME(MIN 0 MAX 0) LAYOUT(HASHED(SHARDS 32)); +SYSTEM RELOAD DICTIONARY dict_sharded; +SELECT name, length(attribute.names), element_count, round(load_factor, 4) FROM system.dictionaries WHERE database = currentDatabase() AND name = 'dict_sharded'; +DROP DICTIONARY dict_sharded; + +CREATE DICTIONARY dict_sharded_multi (key UInt64, v0 UInt16, v1 UInt16, v2 UInt16, v3 UInt16, v4 UInt16) PRIMARY KEY key SOURCE(CLICKHOUSE(TABLE 'dict_data')) LIFETIME(MIN 0 MAX 0) LAYOUT(HASHED(SHARDS 32)); +SYSTEM RELOAD DICTIONARY dict_sharded_multi; +SELECT name, length(attribute.names), element_count, round(load_factor, 4) FROM system.dictionaries WHERE database = currentDatabase() AND name = 'dict_sharded_multi'; +DROP DICTIONARY dict_sharded_multi; + +DROP TABLE dict_data; diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt index 0df7691bd64..8c5d877755f 100644 --- a/utils/check-style/aspell-ignore/en/aspell-dict.txt +++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt @@ -91,6 +91,7 @@ LOCALTIMESTAMP LibFuzzer LineAsString LinksDeployment +LLVM's LowCardinality MEMTABLE MVCC diff --git a/utils/check-style/check-style b/utils/check-style/check-style index 7dbd7d7a816..afaf2ee6d48 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -13,7 +13,7 @@ # and then to run formatter only for the specified files. ROOT_PATH=$(git rev-parse --show-toplevel) -EXCLUDE_DIRS='build/|integration/|widechar_width/|glibc-compatibility/|poco/|memcpy/|consistent-hashing|benchmark|tests/' +EXCLUDE_DIRS='build/|integration/|widechar_width/|glibc-compatibility/|poco/|memcpy/|consistent-hashing|benchmark|tests/|utils/keeper-bench/example.yaml' # From [1]: # But since array_to_string_internal() in array.c still loops over array diff --git a/utils/keeper-bench/CMakeLists.txt b/utils/keeper-bench/CMakeLists.txt index 2596be4addd..87fa64b1761 100644 --- a/utils/keeper-bench/CMakeLists.txt +++ b/utils/keeper-bench/CMakeLists.txt @@ -1,2 +1,7 @@ +if (NOT TARGET ch_contrib::rapidjson) + message (${RECONFIGURE_MESSAGE_LEVEL} "Not building keeper-bench due to rapidjson is disabled") + return() +endif() + clickhouse_add_executable(keeper-bench Generator.cpp Runner.cpp Stats.cpp main.cpp) -target_link_libraries(keeper-bench PRIVATE clickhouse_common_zookeeper_no_log) +target_link_libraries(keeper-bench PRIVATE clickhouse_common_config_no_zookeeper_log ch_contrib::rapidjson) diff --git a/utils/keeper-bench/Generator.cpp b/utils/keeper-bench/Generator.cpp index b6d8223862c..2212f7158ae 100644 --- a/utils/keeper-bench/Generator.cpp +++ b/utils/keeper-bench/Generator.cpp @@ -1,16 +1,18 @@ #include "Generator.h" +#include "Common/Exception.h" +#include "Common/ZooKeeper/ZooKeeperCommon.h" +#include #include #include +#include using namespace Coordination; using namespace zkutil; -namespace DB -{ -namespace ErrorCodes +namespace DB::ErrorCodes { extern const int LOGICAL_ERROR; -} + extern const int BAD_ARGUMENTS; } namespace @@ -38,16 +40,6 @@ std::string generateRandomString(size_t length) } } -std::string generateRandomPath(const std::string & prefix, size_t length) -{ - return std::filesystem::path(prefix) / generateRandomString(length); -} - -std::string generateRandomData(size_t size) -{ - return generateRandomString(size); -} - void removeRecursive(Coordination::ZooKeeper & zookeeper, const std::string & path) { namespace fs = std::filesystem; @@ -96,139 +88,629 @@ void removeRecursive(Coordination::ZooKeeper & zookeeper, const std::string & pa remove_future.get(); } - -void CreateRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) +NumberGetter +NumberGetter::fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config, std::optional default_value) { - removeRecursive(zookeeper, path_prefix); + NumberGetter number_getter; - auto promise = std::make_shared>(); - auto future = promise->get_future(); - auto create_callback = [promise] (const CreateResponse & response) + if (!config.has(key) && default_value.has_value()) { - if (response.error != Coordination::Error::ZOK) - promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); - else - promise->set_value(); - }; - zookeeper.create(path_prefix, "", false, false, default_acls, create_callback); - future.get(); + number_getter.value = *default_value; + } + else if (config.has(key + ".min_value") && config.has(key + ".max_value")) + { + NumberRange range{.min_value = config.getUInt64(key + ".min_value"), .max_value = config.getUInt64(key + ".max_value")}; + if (range.max_value <= range.min_value) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Range is invalid for key {}: [{}, {}]", key, range.min_value, range.max_value); + number_getter.value = range; + } + else + { + number_getter.value = config.getUInt64(key); + } + + return number_getter; } -ZooKeeperRequestPtr CreateRequestGenerator::generate() +std::string NumberGetter::description() const { + if (const auto * number = std::get_if(&value)) + return std::to_string(*number); + + const auto & range = std::get(value); + return fmt::format("random value from range [{}, {}]", range.min_value, range.max_value); +} + +uint64_t NumberGetter::getNumber() const +{ + if (const auto * number = std::get_if(&value)) + return *number; + + const auto & range = std::get(value); + static pcg64 rng(randomSeed()); + return std::uniform_int_distribution(range.min_value, range.max_value)(rng); +} + +StringGetter StringGetter::fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config) +{ + StringGetter string_getter; + if (config.has(key + ".random_string")) + string_getter.value + = NumberGetter::fromConfig(key + ".random_string.size", config); + else + string_getter.value = config.getString(key); + + return string_getter; +} + +void StringGetter::setString(std::string name) +{ + value = std::move(name); +} + +std::string StringGetter::getString() const +{ + if (const auto * string = std::get_if(&value)) + return *string; + + const auto number_getter = std::get(value); + return generateRandomString(number_getter.getNumber()); +} + +std::string StringGetter::description() const +{ + if (const auto * string = std::get_if(&value)) + return *string; + + const auto number_getter = std::get(value); + return fmt::format("random string with size of {}", number_getter.description()); +} + +bool StringGetter::isRandom() const +{ + return std::holds_alternative(value); +} + +PathGetter PathGetter::fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config) +{ + static constexpr std::string_view path_key_string = "path"; + + PathGetter path_getter; + Poco::Util::AbstractConfiguration::Keys path_keys; + config.keys(key, path_keys); + + for (const auto & path_key : path_keys) + { + if (!path_key.starts_with(path_key_string)) + continue; + + const auto current_path_key_string = key + "." + path_key; + const auto children_of_key = current_path_key_string + ".children_of"; + if (config.has(children_of_key)) + { + auto parent_node = config.getString(children_of_key); + if (parent_node.empty() || parent_node[0] != '/') + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Invalid path for request generator: '{}'", parent_node); + path_getter.parent_paths.push_back(std::move(parent_node)); + } + else + { + auto path = config.getString(key + "." + path_key); + + if (path.empty() || path[0] != '/') + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Invalid path for request generator: '{}'", path); + + path_getter.paths.push_back(std::move(path)); + } + } + + path_getter.path_picker = std::uniform_int_distribution(0, path_getter.paths.size() - 1); + return path_getter; +} + +void PathGetter::initialize(Coordination::ZooKeeper & zookeeper) +{ + for (const auto & parent_path : parent_paths) + { + auto list_promise = std::make_shared>(); + auto list_future = list_promise->get_future(); + auto callback = [list_promise] (const ListResponse & response) + { + if (response.error != Coordination::Error::ZOK) + list_promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); + else + list_promise->set_value(response); + }; + zookeeper.list(parent_path, ListRequestType::ALL, std::move(callback), {}); + auto list_response = list_future.get(); + + for (const auto & child : list_response.names) + paths.push_back(std::filesystem::path(parent_path) / child); + } + + path_picker = std::uniform_int_distribution(0, paths.size() - 1); + initialized = true; +} + +std::string PathGetter::getPath() const +{ + if (!initialized) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "PathGetter is not initialized"); + + if (paths.size() == 1) + return paths[0]; + + static pcg64 rng(randomSeed()); + return paths[path_picker(rng)]; +} + +std::string PathGetter::description() const +{ + std::string description; + for (const auto & path : parent_paths) + { + if (!description.empty()) + description += ", "; + description += fmt::format("children of {}", path); + } + + for (const auto & path : paths) + { + if (!description.empty()) + description += ", "; + description += path; + } + + return description; +} + +RequestGetter::RequestGetter(std::vector request_generators_) + : request_generators(std::move(request_generators_)) +{} + +RequestGetter RequestGetter::fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config, bool for_multi) +{ + RequestGetter request_getter; + + Poco::Util::AbstractConfiguration::Keys generator_keys; + config.keys(key, generator_keys); + + bool use_weights = false; + size_t weight_sum = 0; + auto & generators = request_getter.request_generators; + for (const auto & generator_key : generator_keys) + { + RequestGeneratorPtr request_generator; + + if (generator_key.starts_with("create")) + request_generator = std::make_unique(); + else if (generator_key.starts_with("set")) + request_generator = std::make_unique(); + else if (generator_key.starts_with("get")) + request_generator = std::make_unique(); + else if (generator_key.starts_with("list")) + request_generator = std::make_unique(); + else if (generator_key.starts_with("multi")) + { + if (for_multi) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Nested multi requests are not allowed"); + request_generator = std::make_unique(); + } + else + { + if (for_multi) + continue; + + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Unknown generator {}", key + "." + generator_key); + } + + request_generator->getFromConfig(key + "." + generator_key, config); + + auto weight = request_generator->getWeight(); + use_weights |= weight != 1; + weight_sum += weight; + + generators.push_back(std::move(request_generator)); + } + + if (generators.empty()) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "No request generators found in config for key '{}'", key); + + + size_t max_value = use_weights ? weight_sum - 1 : generators.size() - 1; + request_getter.request_generator_picker = std::uniform_int_distribution(0, max_value); + + /// construct weight vector + if (use_weights) + { + auto & weights = request_getter.weights; + weights.reserve(generators.size()); + weights.push_back(generators[0]->getWeight() - 1); + + for (size_t i = 1; i < generators.size(); ++i) + weights.push_back(weights.back() + generators[i]->getWeight()); + } + + return request_getter; +} + +RequestGeneratorPtr RequestGetter::getRequestGenerator() const +{ + static pcg64 rng(randomSeed()); + + auto random_number = request_generator_picker(rng); + + if (weights.empty()) + return request_generators[random_number]; + + for (size_t i = 0; i < request_generators.size(); ++i) + { + if (random_number <= weights[i]) + return request_generators[i]; + } + + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Invalid number generated: {}", random_number); +} + +std::string RequestGetter::description() const +{ + std::string guard(30, '-'); + std::string description = guard; + + for (const auto & request_generator : request_generators) + description += fmt::format("\n{}\n", request_generator->description()); + return description + guard; +} + +void RequestGetter::startup(Coordination::ZooKeeper & zookeeper) +{ + for (const auto & request_generator : request_generators) + request_generator->startup(zookeeper); +} + +const std::vector & RequestGetter::requestGenerators() const +{ + return request_generators; +} + +void RequestGenerator::getFromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config) +{ + if (config.has(key + ".weight")) + weight = config.getUInt64(key + ".weight"); + getFromConfigImpl(key, config); +} + +std::string RequestGenerator::description() +{ + std::string weight_string = weight == 1 ? "" : fmt::format("\n- weight: {}", weight); + return fmt::format("{}{}", descriptionImpl(), weight_string); +} + +Coordination::ZooKeeperRequestPtr RequestGenerator::generate(const Coordination::ACLs & acls) +{ + return generateImpl(acls); +} + +void RequestGenerator::startup(Coordination::ZooKeeper & zookeeper) +{ + startupImpl(zookeeper); +} + +size_t RequestGenerator::getWeight() const +{ + return weight; +} + +CreateRequestGenerator::CreateRequestGenerator() + : rng(randomSeed()) + , remove_picker(0, 1.0) +{} + +void CreateRequestGenerator::getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) +{ + parent_path = PathGetter::fromConfig(key, config); + + name = StringGetter(NumberGetter::fromConfig(key + ".name_length", config, 5)); + + if (config.has(key + ".data")) + data = StringGetter::fromConfig(key + ".data", config); + + if (config.has(key + ".remove_factor")) + remove_factor = config.getDouble(key + ".remove_factor"); +} + +std::string CreateRequestGenerator::descriptionImpl() +{ + std::string data_string + = data.has_value() ? fmt::format("data for created nodes: {}", data->description()) : "no data for created nodes"; + std::string remove_factor_string + = remove_factor.has_value() ? fmt::format("- remove factor: {}", *remove_factor) : "- without removes"; + return fmt::format( + "Create Request Generator\n" + "- parent path(s) for created nodes: {}\n" + "- name for created nodes: {}\n" + "- {}\n" + "{}", + parent_path.description(), + name.description(), + data_string, + remove_factor_string); +} + +void CreateRequestGenerator::startupImpl(Coordination::ZooKeeper & zookeeper) +{ + parent_path.initialize(zookeeper); +} + +Coordination::ZooKeeperRequestPtr CreateRequestGenerator::generateImpl(const Coordination::ACLs & acls) +{ + if (remove_factor.has_value() && !paths_created.empty() && remove_picker(rng) < *remove_factor) + { + auto request = std::make_shared(); + auto it = paths_created.begin(); + request->path = *it; + paths_created.erase(it); + return request; + } + auto request = std::make_shared(); - request->acls = default_acls; - size_t plength = 5; - if (path_length) - plength = *path_length; - auto path_candidate = generateRandomPath(path_prefix, plength); + request->acls = acls; + + std::string path_candidate = std::filesystem::path(parent_path.getPath()) / name.getString(); while (paths_created.contains(path_candidate)) - path_candidate = generateRandomPath(path_prefix, plength); + path_candidate = std::filesystem::path(parent_path.getPath()) / name.getString(); paths_created.insert(path_candidate); - request->path = path_candidate; - if (data_size) - request->data = generateRandomData(*data_size); + request->path = std::move(path_candidate); + + if (data) + request->data = data->getString(); return request; } - -void SetRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) +void SetRequestGenerator::getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) { - removeRecursive(zookeeper, path_prefix); + path = PathGetter::fromConfig(key, config); - auto promise = std::make_shared>(); - auto future = promise->get_future(); - auto create_callback = [promise] (const CreateResponse & response) - { - if (response.error != Coordination::Error::ZOK) - promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); - else - promise->set_value(); - }; - zookeeper.create(path_prefix, "", false, false, default_acls, create_callback); - future.get(); + data = StringGetter::fromConfig(key + ".data", config); } -ZooKeeperRequestPtr SetRequestGenerator::generate() +std::string SetRequestGenerator::descriptionImpl() +{ + return fmt::format( + "Set Request Generator\n" + "- path(s) to set: {}\n" + "- data to set: {}", + path.description(), + data.description()); +} + +Coordination::ZooKeeperRequestPtr SetRequestGenerator::generateImpl(const Coordination::ACLs & /*acls*/) { auto request = std::make_shared(); - request->path = path_prefix; - request->data = generateRandomData(data_size); - + request->path = path.getPath(); + request->data = data.getString(); return request; } -void MixedRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) +void SetRequestGenerator::startupImpl(Coordination::ZooKeeper & zookeeper) { - for (auto & generator : generators) - generator->startup(zookeeper); + path.initialize(zookeeper); } -ZooKeeperRequestPtr MixedRequestGenerator::generate() +void GetRequestGenerator::getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) { - pcg64 rng(randomSeed()); - std::uniform_int_distribution distribution(0, generators.size() - 1); - - return generators[distribution(rng)]->generate(); + path = PathGetter::fromConfig(key, config); } -void GetRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) +std::string GetRequestGenerator::descriptionImpl() { - auto promise = std::make_shared>(); - auto future = promise->get_future(); - auto create_callback = [promise] (const CreateResponse & response) - { - if (response.error != Coordination::Error::ZOK) - promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); - else - promise->set_value(); - }; - zookeeper.create(path_prefix, "", false, false, default_acls, create_callback); - future.get(); - size_t total_nodes = 1; - if (num_nodes) - total_nodes = *num_nodes; - - for (size_t i = 0; i < total_nodes; ++i) - { - auto path = generateRandomPath(path_prefix, 5); - while (std::find(paths_to_get.begin(), paths_to_get.end(), path) != paths_to_get.end()) - path = generateRandomPath(path_prefix, 5); - - auto create_promise = std::make_shared>(); - auto create_future = create_promise->get_future(); - auto callback = [create_promise] (const CreateResponse & response) - { - if (response.error != Coordination::Error::ZOK) - create_promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); - else - create_promise->set_value(); - }; - std::string data; - if (nodes_data_size) - data = generateRandomString(*nodes_data_size); - - zookeeper.create(path, data, false, false, default_acls, callback); - create_future.get(); - paths_to_get.push_back(path); - } + return fmt::format( + "Get Request Generator\n" + "- path(s) to get: {}", + path.description()); } -Coordination::ZooKeeperRequestPtr GetRequestGenerator::generate() +Coordination::ZooKeeperRequestPtr GetRequestGenerator::generateImpl(const Coordination::ACLs & /*acls*/) { auto request = std::make_shared(); - - size_t path_index = distribution(rng); - request->path = paths_to_get[path_index]; + request->path = path.getPath(); return request; } -void ListRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) +void GetRequestGenerator::startupImpl(Coordination::ZooKeeper & zookeeper) { + path.initialize(zookeeper); +} + +void ListRequestGenerator::getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) +{ + path = PathGetter::fromConfig(key, config); +} + +std::string ListRequestGenerator::descriptionImpl() +{ + return fmt::format( + "List Request Generator\n" + "- path(s) to get: {}", + path.description()); +} + +Coordination::ZooKeeperRequestPtr ListRequestGenerator::generateImpl(const Coordination::ACLs & /*acls*/) +{ + auto request = std::make_shared(); + request->path = path.getPath(); + return request; +} + +void ListRequestGenerator::startupImpl(Coordination::ZooKeeper & zookeeper) +{ + path.initialize(zookeeper); +} + +void MultiRequestGenerator::getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) +{ + if (config.has(key + ".size")) + size = NumberGetter::fromConfig(key + ".size", config); + + request_getter = RequestGetter::fromConfig(key, config, /*for_multi*/ true); +}; + +std::string MultiRequestGenerator::descriptionImpl() +{ + std::string size_string = size.has_value() ? fmt::format("- number of requests: {}\n", size->description()) : ""; + return fmt::format( + "Multi Request Generator\n" + "{}" + "- requests:\n{}", + size_string, + request_getter.description()); +} + +Coordination::ZooKeeperRequestPtr MultiRequestGenerator::generateImpl(const Coordination::ACLs & acls) +{ + Coordination::Requests ops; + + if (size) + { + auto request_count = size->getNumber(); + + for (size_t i = 0; i < request_count; ++i) + ops.push_back(request_getter.getRequestGenerator()->generate(acls)); + } + else + { + for (const auto & request_generator : request_getter.requestGenerators()) + ops.push_back(request_generator->generate(acls)); + } + + return std::make_shared(ops, acls); +} + +void MultiRequestGenerator::startupImpl(Coordination::ZooKeeper & zookeeper) +{ + request_getter.startup(zookeeper); +} + +Generator::Generator(const Poco::Util::AbstractConfiguration & config) +{ + Coordination::ACL acl; + acl.permissions = Coordination::ACL::All; + acl.scheme = "world"; + acl.id = "anyone"; + default_acls.emplace_back(std::move(acl)); + + static const std::string generator_key = "generator"; + + std::cerr << "---- Parsing setup ---- " << std::endl; + static const std::string setup_key = generator_key + ".setup"; + Poco::Util::AbstractConfiguration::Keys keys; + config.keys(setup_key, keys); + for (const auto & key : keys) + { + if (key.starts_with("node")) + { + auto node_key = setup_key + "." + key; + auto parsed_root_node = parseNode(node_key, config); + const auto node = root_nodes.emplace_back(parsed_root_node); + + if (config.has(node_key + ".repeat")) + { + if (!node->name.isRandom()) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Repeating node creation for key {}, but name is not randomly generated", node_key); + + auto repeat_count = config.getUInt64(node_key + ".repeat"); + node->repeat_count = repeat_count; + for (size_t i = 1; i < repeat_count; ++i) + root_nodes.emplace_back(node->clone()); + } + + std::cerr << "Tree to create:" << std::endl; + + node->dumpTree(); + std::cerr << std::endl; + } + } + std::cerr << "---- Done parsing data setup ----\n" << std::endl; + + std::cerr << "---- Collecting request generators ----" << std::endl; + static const std::string requests_key = generator_key + ".requests"; + request_getter = RequestGetter::fromConfig(requests_key, config); + std::cerr << request_getter.description() << std::endl; + std::cerr << "---- Done collecting request generators ----\n" << std::endl; +} + +std::shared_ptr Generator::parseNode(const std::string & key, const Poco::Util::AbstractConfiguration & config) +{ + auto node = std::make_shared(); + node->name = StringGetter::fromConfig(key + ".name", config); + + if (config.has(key + ".data")) + node->data = StringGetter::fromConfig(key + ".data", config); + + Poco::Util::AbstractConfiguration::Keys node_keys; + config.keys(key, node_keys); + + for (const auto & node_key : node_keys) + { + if (!node_key.starts_with("node")) + continue; + + const auto node_key_string = key + "." + node_key; + auto child_node = parseNode(node_key_string, config); + node->children.push_back(child_node); + + if (config.has(node_key_string + ".repeat")) + { + if (!child_node->name.isRandom()) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "Repeating node creation for key {}, but name is not randomly generated", node_key_string); + + auto repeat_count = config.getUInt64(node_key_string + ".repeat"); + child_node->repeat_count = repeat_count; + for (size_t i = 1; i < repeat_count; ++i) + node->children.push_back(child_node); + } + } + + return node; +} + +void Generator::Node::dumpTree(int level) const +{ + std::string data_string + = data.has_value() ? fmt::format("{}", data->description()) : "no data"; + + std::string repeat_count_string = repeat_count != 0 ? fmt::format(", repeated {} times", repeat_count) : ""; + + std::cerr << fmt::format("{}name: {}, data: {}{}", std::string(level, '\t'), name.description(), data_string, repeat_count_string) << std::endl; + + for (auto it = children.begin(); it != children.end();) + { + const auto & child = *it; + child->dumpTree(level + 1); + std::advance(it, child->repeat_count != 0 ? child->repeat_count : 1); + } +} + +std::shared_ptr Generator::Node::clone() const +{ + auto new_node = std::make_shared(); + new_node->name = name; + new_node->data = data; + new_node->repeat_count = repeat_count; + + // don't do deep copy of children because we will do clone only for root nodes + new_node->children = children; + + return new_node; +} + +void Generator::Node::createNode(Coordination::ZooKeeper & zookeeper, const std::string & parent_path, const Coordination::ACLs & acls) const +{ + auto path = std::filesystem::path(parent_path) / name.getString(); auto promise = std::make_shared>(); auto future = promise->get_future(); auto create_callback = [promise] (const CreateResponse & response) @@ -238,103 +720,47 @@ void ListRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) else promise->set_value(); }; - zookeeper.create(path_prefix, "", false, false, default_acls, create_callback); + zookeeper.create(path, data ? data->getString() : "", false, false, acls, create_callback); future.get(); - size_t total_nodes = 1; - if (num_nodes) - total_nodes = *num_nodes; - - size_t path_length = 5; - if (paths_length) - path_length = *paths_length; - - for (size_t i = 0; i < total_nodes; ++i) - { - auto path = generateRandomPath(path_prefix, path_length); - - auto create_promise = std::make_shared>(); - auto create_future = create_promise->get_future(); - auto callback = [create_promise] (const CreateResponse & response) - { - if (response.error != Coordination::Error::ZOK) - create_promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); - else - create_promise->set_value(); - }; - zookeeper.create(path, "", false, false, default_acls, callback); - create_future.get(); - } + for (const auto & child : children) + child->createNode(zookeeper, path, acls); } -Coordination::ZooKeeperRequestPtr ListRequestGenerator::generate() +void Generator::startup(Coordination::ZooKeeper & zookeeper) { - auto request = std::make_shared(); - request->path = path_prefix; - return request; + std::cerr << "---- Creating test data ----" << std::endl; + for (const auto & node : root_nodes) + { + auto node_name = node->name.getString(); + node->name.setString(node_name); + + std::string root_path = std::filesystem::path("/") / node_name; + std::cerr << "Cleaning up " << root_path << std::endl; + removeRecursive(zookeeper, root_path); + + node->createNode(zookeeper, "/", default_acls); + } + std::cerr << "---- Created test data ----\n" << std::endl; + + std::cerr << "---- Initializing generators ----" << std::endl; + + request_getter.startup(zookeeper); } -std::unique_ptr getGenerator(const std::string & name) +Coordination::ZooKeeperRequestPtr Generator::generate() { - if (name == "create_no_data") - { - return std::make_unique(); - } - else if (name == "create_small_data") - { - return std::make_unique("/create_generator", 5, 32); - } - else if (name == "create_medium_data") - { - return std::make_unique("/create_generator", 5, 1024); - } - else if (name == "create_big_data") - { - return std::make_unique("/create_generator", 5, 512 * 1024); - } - else if (name == "get_no_data") - { - return std::make_unique("/get_generator", 10, 0); - } - else if (name == "get_small_data") - { - return std::make_unique("/get_generator", 10, 32); - } - else if (name == "get_medium_data") - { - return std::make_unique("/get_generator", 10, 1024); - } - else if (name == "get_big_data") - { - return std::make_unique("/get_generator", 10, 512 * 1024); - } - else if (name == "list_no_nodes") - { - return std::make_unique("/list_generator", 0, 1); - } - else if (name == "list_few_nodes") - { - return std::make_unique("/list_generator", 10, 5); - } - else if (name == "list_medium_nodes") - { - return std::make_unique("/list_generator", 1000, 5); - } - else if (name == "list_a_lot_nodes") - { - return std::make_unique("/list_generator", 100000, 5); - } - else if (name == "set_small_data") - { - return std::make_unique("/set_generator", 5); - } - else if (name == "mixed_small_data") - { - std::vector> generators; - generators.push_back(std::make_unique("/set_generator", 5)); - generators.push_back(std::make_unique("/get_generator", 10, 32)); - return std::make_unique(std::move(generators)); - } - - throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unknown generator {}", name); + return request_getter.getRequestGenerator()->generate(default_acls); +} + +void Generator::cleanup(Coordination::ZooKeeper & zookeeper) +{ + std::cerr << "---- Cleaning up test data ----" << std::endl; + for (const auto & node : root_nodes) + { + auto node_name = node->name.getString(); + std::string root_path = std::filesystem::path("/") / node_name; + std::cerr << "Cleaning up " << root_path << std::endl; + removeRecursive(zookeeper, root_path); + } } diff --git a/utils/keeper-bench/Generator.h b/utils/keeper-bench/Generator.h index e2c546e4bce..5b4c05b2d8b 100644 --- a/utils/keeper-bench/Generator.h +++ b/utils/keeper-bench/Generator.h @@ -6,135 +6,194 @@ #include #include #include +#include #include - -std::string generateRandomPath(const std::string & prefix, size_t length = 5); - -std::string generateRandomData(size_t size); - -class IGenerator +struct NumberGetter { -public: - IGenerator() + static NumberGetter fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config, std::optional default_value = std::nullopt); + uint64_t getNumber() const; + std::string description() const; +private: + struct NumberRange { - Coordination::ACL acl; - acl.permissions = Coordination::ACL::All; - acl.scheme = "world"; - acl.id = "anyone"; - default_acls.emplace_back(std::move(acl)); - } - virtual void startup(Coordination::ZooKeeper & /*zookeeper*/) {} - virtual Coordination::ZooKeeperRequestPtr generate() = 0; - - virtual ~IGenerator() = default; - - Coordination::ACLs default_acls; + uint64_t min_value; + uint64_t max_value; + }; + std::variant value; }; -class CreateRequestGenerator final : public IGenerator +struct StringGetter { -public: - explicit CreateRequestGenerator( - std::string path_prefix_ = "/create_generator", - std::optional path_length_ = std::nullopt, - std::optional data_size_ = std::nullopt) - : path_prefix(path_prefix_) - , path_length(path_length_) - , data_size(data_size_) + explicit StringGetter(NumberGetter number_getter) + : value(std::move(number_getter)) {} - void startup(Coordination::ZooKeeper & zookeeper) override; - Coordination::ZooKeeperRequestPtr generate() override; + StringGetter() = default; + static StringGetter fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config); + void setString(std::string name); + std::string getString() const; + std::string description() const; + bool isRandom() const; private: - std::string path_prefix; - std::optional path_length; - std::optional data_size; + std::variant value; +}; + +struct PathGetter +{ + static PathGetter fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config); + + std::string getPath() const; + std::string description() const; + + void initialize(Coordination::ZooKeeper & zookeeper); +private: + std::vector parent_paths; + + bool initialized = false; + + std::vector paths; + mutable std::uniform_int_distribution path_picker; +}; + +struct RequestGenerator +{ + virtual ~RequestGenerator() = default; + + void getFromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config); + + Coordination::ZooKeeperRequestPtr generate(const Coordination::ACLs & acls); + + std::string description(); + + void startup(Coordination::ZooKeeper & zookeeper); + + size_t getWeight() const; +private: + virtual void getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) = 0; + virtual std::string descriptionImpl() = 0; + virtual Coordination::ZooKeeperRequestPtr generateImpl(const Coordination::ACLs & acls) = 0; + virtual void startupImpl(Coordination::ZooKeeper &) {} + + size_t weight = 1; +}; + +using RequestGeneratorPtr = std::shared_ptr; + +struct CreateRequestGenerator final : public RequestGenerator +{ + CreateRequestGenerator(); +private: + void getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) override; + std::string descriptionImpl() override; + Coordination::ZooKeeperRequestPtr generateImpl(const Coordination::ACLs & acls) override; + void startupImpl(Coordination::ZooKeeper & zookeeper) override; + + PathGetter parent_path; + StringGetter name; + std::optional data; + + std::optional remove_factor; + pcg64 rng; + std::uniform_real_distribution remove_picker; + std::unordered_set paths_created; }; - -class GetRequestGenerator final : public IGenerator +struct SetRequestGenerator final : public RequestGenerator { -public: - explicit GetRequestGenerator( - std::string path_prefix_ = "/get_generator", - std::optional num_nodes_ = std::nullopt, - std::optional nodes_data_size_ = std::nullopt) - : path_prefix(path_prefix_) - , num_nodes(num_nodes_) - , nodes_data_size(nodes_data_size_) - , rng(randomSeed()) - , distribution(0, num_nodes ? *num_nodes - 1 : 0) - {} - - void startup(Coordination::ZooKeeper & zookeeper) override; - Coordination::ZooKeeperRequestPtr generate() override; - private: - std::string path_prefix; - std::optional num_nodes; - std::optional nodes_data_size; - std::vector paths_to_get; + void getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) override; + std::string descriptionImpl() override; + Coordination::ZooKeeperRequestPtr generateImpl(const Coordination::ACLs & acls) override; + void startupImpl(Coordination::ZooKeeper & zookeeper) override; - pcg64 rng; - std::uniform_int_distribution distribution; + PathGetter path; + StringGetter data; }; -class ListRequestGenerator final : public IGenerator +struct GetRequestGenerator final : public RequestGenerator { -public: - explicit ListRequestGenerator( - std::string path_prefix_ = "/list_generator", - std::optional num_nodes_ = std::nullopt, - std::optional paths_length_ = std::nullopt) - : path_prefix(path_prefix_) - , num_nodes(num_nodes_) - , paths_length(paths_length_) - {} - - void startup(Coordination::ZooKeeper & zookeeper) override; - Coordination::ZooKeeperRequestPtr generate() override; - private: - std::string path_prefix; - std::optional num_nodes; - std::optional paths_length; + void getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) override; + std::string descriptionImpl() override; + Coordination::ZooKeeperRequestPtr generateImpl(const Coordination::ACLs & acls) override; + void startupImpl(Coordination::ZooKeeper & zookeeper) override; + + PathGetter path; }; -class SetRequestGenerator final : public IGenerator +struct ListRequestGenerator final : public RequestGenerator { -public: - explicit SetRequestGenerator( - std::string path_prefix_ = "/set_generator", - uint64_t data_size_ = 5) - : path_prefix(path_prefix_) - , data_size(data_size_) - {} - - void startup(Coordination::ZooKeeper & zookeeper) override; - Coordination::ZooKeeperRequestPtr generate() override; - private: - std::string path_prefix; - uint64_t data_size; + void getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) override; + std::string descriptionImpl() override; + Coordination::ZooKeeperRequestPtr generateImpl(const Coordination::ACLs & acls) override; + void startupImpl(Coordination::ZooKeeper & zookeeper) override; + + PathGetter path; }; -class MixedRequestGenerator final : public IGenerator +struct RequestGetter { -public: - explicit MixedRequestGenerator(std::vector> generators_) - : generators(std::move(generators_)) - {} + explicit RequestGetter(std::vector request_generators_); - void startup(Coordination::ZooKeeper & zookeeper) override; - Coordination::ZooKeeperRequestPtr generate() override; + RequestGetter() = default; + static RequestGetter fromConfig(const std::string & key, const Poco::Util::AbstractConfiguration & config, bool for_multi = false); + + RequestGeneratorPtr getRequestGenerator() const; + std::string description() const; + void startup(Coordination::ZooKeeper & zookeeper); + const std::vector & requestGenerators() const; private: - std::vector> generators; + std::vector request_generators; + std::vector weights; + mutable std::uniform_int_distribution request_generator_picker; }; +struct MultiRequestGenerator final : public RequestGenerator +{ +private: + void getFromConfigImpl(const std::string & key, const Poco::Util::AbstractConfiguration & config) override; + std::string descriptionImpl() override; + Coordination::ZooKeeperRequestPtr generateImpl(const Coordination::ACLs & acls) override; + void startupImpl(Coordination::ZooKeeper & zookeeper) override; -std::unique_ptr getGenerator(const std::string & name); + std::optional size; + RequestGetter request_getter; +}; + +class Generator +{ +public: + explicit Generator(const Poco::Util::AbstractConfiguration & config); + + void startup(Coordination::ZooKeeper & zookeeper); + Coordination::ZooKeeperRequestPtr generate(); + void cleanup(Coordination::ZooKeeper & zookeeper); +private: + struct Node + { + StringGetter name; + std::optional data; + std::vector> children; + size_t repeat_count = 0; + + std::shared_ptr clone() const; + + void createNode(Coordination::ZooKeeper & zookeeper, const std::string & parent_path, const Coordination::ACLs & acls) const; + void dumpTree(int level = 0) const; + }; + + static std::shared_ptr parseNode(const std::string & key, const Poco::Util::AbstractConfiguration & config); + + std::uniform_int_distribution request_picker; + std::vector> root_nodes; + RequestGetter request_getter; + Coordination::ACLs default_acls; +}; + +std::optional getGenerator(const std::string & name); diff --git a/utils/keeper-bench/README.md b/utils/keeper-bench/README.md new file mode 100644 index 00000000000..8b498228799 --- /dev/null +++ b/utils/keeper-bench/README.md @@ -0,0 +1,317 @@ +# Keeper Bench + +Keeper Bench is a tool for benchmarking Keeper or any ZooKeeper compatible systems. + +To run it call following command from the build folder: + +``` +./utils/keeper-bench --config benchmark_config_file.yaml +``` + +## Configuration file + +Keeper Bench runs need to be configured inside a yaml or XML file. +An example of a configuration file can be found in `./utils/keeper-bench/example.yaml` + +### Table of contents +- [Special Types](#special-types) +- [General settings](#general-settings) +- [Connections](#connections) +- [Generator](#generator) +- [Output](#output) + + +## Special types + +### IntegerGetter + +Can be defined with constant integer or as a random value from a range. + +```yaml +key: integer +key: + min_value: integer + max_value: integer +``` + +Example for a constant value: + +```yaml +some_key: 2 +``` + +Example for random value from [10, 20]: + +```yaml +some_key: + min_value: 10 + max_value: 20 +``` + +### StringGetter + +Can be defined with constant string or as a random string of some size. + +```yaml +key: string +key: + random_string: + size: IntegerGetter +``` + +Example for a constant value: +```yaml +some_key: "string" +``` + +Example for a random string with a random size from [10, 20]: +```yaml +some_key: + random_string: + size: + min_value: 10 + max_value: 20 +``` + + +### PathGetter + +If a section contains one or more `path` keys, all `path` keys are collected into a list. \ +Additionally, paths can be defined with key `children_of` which will add all children of some path to the list. + +```yaml +path: string +path: + children_of: string +``` + +Example for defining list of paths (`/path1`, `/path2` and children of `/path3`): + +```yaml +main: + path: + - "/path1" + - "/path2" + path: + children_of: "/path3" +``` + + +## General settings + +```yaml +# number of parallel queries (default: 1) +concurrency: integer + +# amount of queries to be executed, set 0 to disable limit (default: 0) +iterations: integer + +# delay between intermediate reports in seconds, set 0 to disable reports (default: 1.0) +report_delay: double + +# stop launch of queries after specified time limit, set 0 to disable limit (default: 0) +timelimit: double + +# continue testing even if a query fails (default: false) +continue_on_errors: boolean +``` + + +## Connections + +Connection definitions that will be used throughout tests defined under `connections` key. + +Following configurations can be defined under `connections` key or for each specific connection. \ +If it's defined under `connections` key, it will be used by default unless a specific connection overrides it. + +```yaml +secure: boolean +operation_timeout_ms: integer +session_timeout_ms: integer +connection_timeout_ms: integer +``` + +Specific configuration can be defined with a string or with a detailed description. + +```yaml +host: string +connection: + host: string + + # number of sessions to create for host + sessions: integer + # any connection configuration defined above +``` + +Example definition of 3 connections in total, 1 to `localhost:9181` and 2 to `localhost:9182` both will use secure connections: + +```yaml +connections: + secure: true + + host: "localhost:9181" + connection: + host: "localhost:9182" + sessions: 2 +``` + + +## Generator + +Main part of the benchmark is the generator itself which creates necessary nodes and defines how the requests will be generated. \ +It is defined under `generator` key. + +### Setup + +Setup defines nodes that are needed for test, defined under `setup` key. + +Each node is defined with a `node` key in the following format: + +```yaml +node: StringGetter + +node: + name: StringGetter + data: StringGetter + repeat: integer + node: Node +``` + +If only string is defined, a node with that name will be created. \ +Otherwise more detailed definition could be included to set data or the children of the node. \ +If `repeat` key is set, the node definition will be used multiple times. For a `repeat` key to be valid, the name of the node needs to be a random string. + +Example for a setup: + +```yaml +generator: + setup: + node: "node1" + node: + name: + random_string: + size: 20 + data: "somedata" + repeat: 4 + node: + name: + random_string: + size: 10 + repeat: 2 +``` + +We will create node `/node1` with no data and 4 children of random name of size 20 and data set to `somedata`. \ +We will also create 2 nodes with no data and random name of size 10 under `/` node. + +### Requests + +While benchmark is running, we are generating requests. + +Request generator is defined under `requests` key. \ +For each request `weight` (default: 1) can be defined which defines preference for a certain request. + +#### `create` + +```yaml +create: + # parent path for created nodes + path: string + + # length of the name for the create node (default: 5) + name_length: IntegerGetter + + # data for create nodes (default: "") + data: StringGetter + + # value in range [0.0, 1.0> denoting how often a remove request should be generated compared to create request (default: 0) + remove_factor: double +``` + +#### `set` + +```yaml +set: + # paths on which we randomly set data + path: PathGetter + + # data to set + data: StringGetter +``` + +#### `get` + +```yaml +get: + # paths for which we randomly get data + path: PathGetter +``` + +#### `list` + +```yaml +list: + # paths for which we randomly do list request + path: PathGetter +``` + +#### `multi` + +```yaml +multi: + # any request definition defined above can be added + + # optional size for the multi request + size: IntegerGetter +``` + +Multi request definition can contain any other request generator definitions described above. \ +If `size` key is defined, we will randomly pick `size` amount of requests from defined request generators. \ +All request generators can have a higher pick probability by using `weight` key. \ +If `size` is not defined, multi request with same request generators will always be generated. \ +Both write and read multi requests are supported. + +#### Example + +```yaml +generator: + requests: + create: + path: "/test_create" + name_length: + min_value: 10 + max_value: 20 + multi: + weight: 20 + size: 10 + get: + path: + children_of: "/test_get1" + get: + weight: 2 + path: + children_of: "/test_get2" +``` + +We defined a request geneator that will generate either a `create` or a `multi` request. \ +Each `create` request will create a node under `/test_create` with a randomly generated name with size from range `[10, 20]`. \ +`multi` request will be generated 20 times more than `create` request. \ +`multi` request will contain 10 requests and approximately twice as much get requests to children of "/test_get2". + + +## Output + +```yaml +output: + # if defined, JSON output of results will be stored at the defined path + file: string + # or + file: + # if defined, JSON output of results will be stored at the defined path + path: string + + # if set to true, timestamp will be appended to the output file name (default: false) + with_timestamp: boolean + + # if set to true, output will be printed to stdout also (default: false) + stdout: boolean +``` diff --git a/utils/keeper-bench/Runner.cpp b/utils/keeper-bench/Runner.cpp index c858b476483..f86d2b44dd7 100644 --- a/utils/keeper-bench/Runner.cpp +++ b/utils/keeper-bench/Runner.cpp @@ -1,15 +1,160 @@ #include "Runner.h" +#include -namespace DB +#include "Common/ZooKeeper/ZooKeeperCommon.h" +#include "Common/ZooKeeper/ZooKeeperConstants.h" +#include +#include +#include "IO/ReadBufferFromString.h" +#include +#include +#include + +namespace CurrentMetrics { + extern const Metric LocalThread; + extern const Metric LocalThreadActive; +} -namespace ErrorCodes +namespace DB::ErrorCodes { extern const int CANNOT_BLOCK_SIGNAL; + extern const int BAD_ARGUMENTS; } +Runner::Runner( + std::optional concurrency_, + const std::string & config_path, + const Strings & hosts_strings_, + std::optional max_time_, + std::optional delay_, + std::optional continue_on_error_, + std::optional max_iterations_) + : info(std::make_shared()) +{ + + DB::ConfigProcessor config_processor(config_path, true, false); + auto config = config_processor.loadConfig().configuration; + + generator.emplace(*config); + + if (!hosts_strings_.empty()) + { + for (const auto & host : hosts_strings_) + connection_infos.push_back({.host = host}); + } + else + { + if (!config) + throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "No config file or hosts defined"); + + parseHostsFromConfig(*config); + } + + std::cerr << "---- Run options ---- " << std::endl; + static constexpr uint64_t DEFAULT_CONCURRENCY = 1; + if (concurrency_) + concurrency = *concurrency_; + else + concurrency = config->getUInt64("concurrency", DEFAULT_CONCURRENCY); + std::cerr << "Concurrency: " << concurrency << std::endl; + + static constexpr uint64_t DEFAULT_ITERATIONS = 0; + if (max_iterations_) + max_iterations = *max_iterations_; + else + max_iterations = config->getUInt64("iterations", DEFAULT_ITERATIONS); + std::cerr << "Iterations: " << max_iterations << std::endl; + + static constexpr double DEFAULT_DELAY = 1.0; + if (delay_) + delay = *delay_; + else + delay = config->getDouble("report_delay", DEFAULT_DELAY); + std::cerr << "Report delay: " << delay << std::endl; + + static constexpr double DEFAULT_TIME_LIMIT = 0.0; + if (max_time_) + max_time = *max_time_; + else + max_time = config->getDouble("timelimit", DEFAULT_TIME_LIMIT); + std::cerr << "Time limit: " << max_time << std::endl; + + if (continue_on_error_) + continue_on_error = *continue_on_error_; + else + continue_on_error = config->getBool("continue_on_error", false); + std::cerr << "Continue on error: " << continue_on_error << std::endl; + + static const std::string output_key = "output"; + print_to_stdout = config->getBool(output_key + ".stdout", false); + std::cerr << "Printing output to stdout: " << print_to_stdout << std::endl; + + static const std::string output_file_key = output_key + ".file"; + if (config->has(output_file_key)) + { + if (config->has(output_file_key + ".path")) + { + file_output = config->getString(output_file_key + ".path"); + output_file_with_timestamp = config->getBool(output_file_key + ".with_timestamp"); + } + else + file_output = config->getString(output_file_key); + + std::cerr << "Result file path: " << file_output->string() << std::endl; + } + + std::cerr << "---- Run options ----\n" << std::endl; + + pool.emplace(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, concurrency); + queue.emplace(concurrency); } +void Runner::parseHostsFromConfig(const Poco::Util::AbstractConfiguration & config) +{ + ConnectionInfo default_connection_info; + + const auto fill_connection_details = [&](const std::string & key, auto & connection_info) + { + if (config.has(key + ".secure")) + connection_info.secure = config.getBool(key + ".secure"); + + if (config.has(key + ".session_timeout_ms")) + connection_info.session_timeout_ms = config.getInt(key + ".session_timeout_ms"); + + if (config.has(key + ".operation_timeout_ms")) + connection_info.operation_timeout_ms = config.getInt(key + ".operation_timeout_ms"); + + if (config.has(key + ".connection_timeout_ms")) + connection_info.connection_timeout_ms = config.getInt(key + ".connection_timeout_ms"); + }; + + fill_connection_details("connections", default_connection_info); + + Poco::Util::AbstractConfiguration::Keys connections_keys; + config.keys("connections", connections_keys); + + for (const auto & key : connections_keys) + { + std::string connection_key = "connections." + key; + auto connection_info = default_connection_info; + if (key.starts_with("host")) + { + connection_info.host = config.getString(connection_key); + connection_infos.push_back(std::move(connection_info)); + } + else if (key.starts_with("connection") && key != "connection_timeout_ms") + { + connection_info.host = config.getString(connection_key + ".host"); + if (config.has(connection_key + ".sessions")) + connection_info.sessions = config.getUInt64(connection_key + ".sessions"); + + fill_connection_details(connection_key, connection_info); + + connection_infos.push_back(std::move(connection_info)); + } + } +} void Runner::thread(std::vector> zookeepers) { @@ -33,7 +178,7 @@ void Runner::thread(std::vector> zookee while (!extracted) { - extracted = queue.tryPop(request, 100); + extracted = queue->tryPop(request, 100); if (shutdown || (max_iterations && requests_executed >= max_iterations)) @@ -47,9 +192,35 @@ void Runner::thread(std::vector> zookee auto promise = std::make_shared>(); auto future = promise->get_future(); - Coordination::ResponseCallback callback = [promise](const Coordination::Response & response) + Coordination::ResponseCallback callback = [&request, promise](const Coordination::Response & response) { - if (response.error != Coordination::Error::ZOK) + bool set_exception = true; + + if (response.error == Coordination::Error::ZOK) + { + set_exception = false; + } + else if (response.error == Coordination::Error::ZNONODE) + { + /// remove can fail with ZNONODE because of different order of execution + /// of generated create and remove requests + /// this is okay for concurrent runs + if (dynamic_cast(&response)) + set_exception = false; + else if (const auto * multi_response = dynamic_cast(&response)) + { + const auto & responses = multi_response->responses; + size_t i = 0; + while (responses[i]->error != Coordination::Error::ZNONODE) + ++i; + + const auto & multi_request = dynamic_cast(*request); + if (dynamic_cast(&*multi_request.requests[i])) + set_exception = false; + } + } + + if (set_exception) promise->set_exception(std::make_exception_ptr(zkutil::KeeperException(response.error))); else promise->set_value(response.bytesSize()); @@ -62,14 +233,14 @@ void Runner::thread(std::vector> zookee try { auto response_size = future.get(); - double seconds = watch.elapsedSeconds(); + auto microseconds = watch.elapsedMicroseconds(); std::lock_guard lock(mutex); if (request->isReadRequest()) - info->addRead(seconds, 1, request->bytesSize() + response_size); + info->addRead(microseconds, 1, request->bytesSize() + response_size); else - info->addWrite(seconds, 1, request->bytesSize() + response_size); + info->addWrite(microseconds, 1, request->bytesSize() + response_size); } catch (...) { @@ -95,7 +266,7 @@ void Runner::thread(std::vector> zookee { try { - zookeepers = getConnections(); + zookeepers = refreshConnections(); break; } catch (...) @@ -110,13 +281,13 @@ void Runner::thread(std::vector> zookee } } -bool Runner::tryPushRequestInteractively(const Coordination::ZooKeeperRequestPtr & request, DB::InterruptListener & interrupt_listener) +bool Runner::tryPushRequestInteractively(Coordination::ZooKeeperRequestPtr && request, DB::InterruptListener & interrupt_listener) { bool inserted = false; while (!inserted) { - inserted = queue.tryPush(request, 100); + inserted = queue->tryPush(std::move(request), 100); if (shutdown) { @@ -126,13 +297,13 @@ bool Runner::tryPushRequestInteractively(const Coordination::ZooKeeperRequestPtr if (max_time > 0 && total_watch.elapsedSeconds() >= max_time) { - std::cout << "Stopping launch of queries. Requested time limit is exhausted.\n"; + std::cerr << "Stopping launch of queries. Requested time limit is exhausted.\n"; return false; } if (interrupt_listener.check()) { - std::cout << "Stopping launch of queries. SIGINT received." << std::endl; + std::cerr << "Stopping launch of queries. SIGINT received." << std::endl; return false; } @@ -141,7 +312,7 @@ bool Runner::tryPushRequestInteractively(const Coordination::ZooKeeperRequestPtr printNumberOfRequestsExecuted(requests_executed); std::lock_guard lock(mutex); - report(info, concurrency); + info->report(concurrency); delay_watch.restart(); } } @@ -152,23 +323,26 @@ bool Runner::tryPushRequestInteractively(const Coordination::ZooKeeperRequestPtr void Runner::runBenchmark() { - auto aux_connections = getConnections(); + createConnections(); std::cerr << "Preparing to run\n"; - generator->startup(*aux_connections[0]); + generator->startup(*connections[0]); std::cerr << "Prepared\n"; + + auto start_timestamp_ms = Poco::Timestamp().epochMicroseconds() / 1000; + try { - auto connections = getConnections(); for (size_t i = 0; i < concurrency; ++i) { - pool.scheduleOrThrowOnError([this, connections]() mutable { thread(connections); }); + auto thread_connections = connections; + pool->scheduleOrThrowOnError([this, connections = std::move(thread_connections)]() mutable { thread(connections); }); } } catch (...) { shutdown = true; - pool.wait(); + pool->wait(); throw; } @@ -185,31 +359,102 @@ void Runner::runBenchmark() } } - pool.wait(); + pool->wait(); total_watch.stop(); printNumberOfRequestsExecuted(requests_executed); std::lock_guard lock(mutex); - report(info, concurrency); -} + info->report(concurrency); + DB::WriteBufferFromOwnString out; + info->writeJSON(out, concurrency, start_timestamp_ms); + auto output_string = std::move(out.str()); -std::vector> Runner::getConnections() -{ - std::vector> zookeepers; - for (const auto & host_string : hosts_strings) + if (print_to_stdout) + std::cout << output_string << std::endl; + + if (file_output) { - Coordination::ZooKeeper::Node node{Poco::Net::SocketAddress{host_string}, false}; - std::vector nodes; - nodes.push_back(node); - zkutil::ZooKeeperArgs args; - args.session_timeout_ms = 30000; - args.connection_timeout_ms = 1000; - args.operation_timeout_ms = 10000; - zookeepers.emplace_back(std::make_shared(nodes, args, nullptr)); + auto path = *file_output; + + if (output_file_with_timestamp) + { + auto filename = file_output->filename(); + filename = fmt::format("{}_{}{}", filename.stem().generic_string(), start_timestamp_ms, filename.extension().generic_string()); + path = file_output->parent_path() / filename; + } + + std::cerr << "Storing output to " << path << std::endl; + + DB::WriteBufferFromFile file_output_buffer(path); + DB::ReadBufferFromString read_buffer(output_string); + DB::copyData(read_buffer, file_output_buffer); } - - - return zookeepers; } + + +void Runner::createConnections() +{ + DB::EventNotifier::init(); + std::cerr << "---- Creating connections ---- " << std::endl; + for (size_t connection_info_idx = 0; connection_info_idx < connection_infos.size(); ++connection_info_idx) + { + const auto & connection_info = connection_infos[connection_info_idx]; + std::cerr << fmt::format("Creating {} session(s) for:\n" + "- host: {}\n" + "- secure: {}\n" + "- session timeout: {}ms\n" + "- operation timeout: {}ms\n" + "- connection timeout: {}ms", + connection_info.sessions, + connection_info.host, + connection_info.secure, + connection_info.session_timeout_ms, + connection_info.operation_timeout_ms, + connection_info.connection_timeout_ms) << std::endl; + + for (size_t session = 0; session < connection_info.sessions; ++session) + { + connections.emplace_back(getConnection(connection_info)); + connections_to_info_map[connections.size() - 1] = connection_info_idx; + } + } + std::cerr << "---- Done creating connections ----\n" << std::endl; +} + +std::shared_ptr Runner::getConnection(const ConnectionInfo & connection_info) +{ + Coordination::ZooKeeper::Node node{Poco::Net::SocketAddress{connection_info.host}, connection_info.secure}; + std::vector nodes; + nodes.push_back(node); + zkutil::ZooKeeperArgs args; + args.session_timeout_ms = connection_info.session_timeout_ms; + args.connection_timeout_ms = connection_info.operation_timeout_ms; + args.operation_timeout_ms = connection_info.connection_timeout_ms; + return std::make_shared(nodes, args, nullptr); +} + +std::vector> Runner::refreshConnections() +{ + std::lock_guard lock(connection_mutex); + for (size_t connection_idx = 0; connection_idx < connections.size(); ++connection_idx) + { + auto & connection = connections[connection_idx]; + if (connection->isExpired()) + { + const auto & connection_info = connection_infos[connections_to_info_map[connection_idx]]; + connection = getConnection(connection_info); + } + } + return connections; +} + +Runner::~Runner() +{ + queue->clearAndFinish(); + shutdown = true; + pool->wait(); + generator->cleanup(*connections[0]); +} + diff --git a/utils/keeper-bench/Runner.h b/utils/keeper-bench/Runner.h index a00b7b43eff..f899f1d538d 100644 --- a/utils/keeper-bench/Runner.h +++ b/utils/keeper-bench/Runner.h @@ -1,50 +1,35 @@ #pragma once +#include "Common/ZooKeeper/ZooKeeperConstants.h" #include #include "Generator.h" #include +#include #include #include #include -#include -#include #include #include #include +#include #include "Stats.h" +#include + using Ports = std::vector; using Strings = std::vector; -namespace CurrentMetrics -{ - extern const Metric LocalThread; - extern const Metric LocalThreadActive; -} - class Runner { public: Runner( - size_t concurrency_, - const std::string & generator_name, + std::optional concurrency_, + const std::string & config_path, const Strings & hosts_strings_, - double max_time_, - double delay_, - bool continue_on_error_, - size_t max_iterations_) - : concurrency(concurrency_) - , pool(CurrentMetrics::LocalThread, CurrentMetrics::LocalThreadActive, concurrency) - , hosts_strings(hosts_strings_) - , generator(getGenerator(generator_name)) - , max_time(max_time_) - , delay(delay_) - , continue_on_error(continue_on_error_) - , max_iterations(max_iterations_) - , info(std::make_shared()) - , queue(concurrency) - { - } + std::optional max_time_, + std::optional delay_, + std::optional continue_on_error_, + std::optional max_iterations_); void thread(std::vector> zookeepers); @@ -53,18 +38,19 @@ public: std::cerr << "Requests executed: " << num << ".\n"; } - bool tryPushRequestInteractively(const Coordination::ZooKeeperRequestPtr & request, DB::InterruptListener & interrupt_listener); + bool tryPushRequestInteractively(Coordination::ZooKeeperRequestPtr && request, DB::InterruptListener & interrupt_listener); void runBenchmark(); - + ~Runner(); private: + void parseHostsFromConfig(const Poco::Util::AbstractConfiguration & config); size_t concurrency = 1; - ThreadPool pool; - Strings hosts_strings; - std::unique_ptr generator; + std::optional pool; + + std::optional generator; double max_time = 0; double delay = 1; bool continue_on_error = false; @@ -73,6 +59,9 @@ private: std::atomic shutdown = false; std::shared_ptr info; + bool print_to_stdout; + std::optional file_output; + bool output_file_with_timestamp; Stopwatch total_watch; Stopwatch delay_watch; @@ -80,7 +69,26 @@ private: std::mutex mutex; using Queue = ConcurrentBoundedQueue; - Queue queue; + std::optional queue; - std::vector> getConnections(); + struct ConnectionInfo + { + std::string host; + + bool secure = false; + int32_t session_timeout_ms = Coordination::DEFAULT_SESSION_TIMEOUT_MS; + int32_t connection_timeout_ms = Coordination::DEFAULT_CONNECTION_TIMEOUT_MS; + int32_t operation_timeout_ms = Coordination::DEFAULT_OPERATION_TIMEOUT_MS; + + size_t sessions = 1; + }; + + std::mutex connection_mutex; + std::vector connection_infos; + std::vector> connections; + std::unordered_map connections_to_info_map; + + void createConnections(); + std::shared_ptr getConnection(const ConnectionInfo & connection_info); + std::vector> refreshConnections(); }; diff --git a/utils/keeper-bench/Stats.cpp b/utils/keeper-bench/Stats.cpp index 1f8b02ed09d..f5e5f84ba14 100644 --- a/utils/keeper-bench/Stats.cpp +++ b/utils/keeper-bench/Stats.cpp @@ -1,67 +1,177 @@ #include "Stats.h" #include -void report(std::shared_ptr & info, size_t concurrency) +#include +#include +#include +#include + +void Stats::StatsCollector::add(uint64_t microseconds, size_t requests_inc, size_t bytes_inc) +{ + work_time += microseconds; + requests += requests_inc; + requests_bytes += bytes_inc; + sampler.insert(microseconds); +} + +void Stats::addRead(uint64_t microseconds, size_t requests_inc, size_t bytes_inc) +{ + read_collector.add(microseconds, requests_inc, bytes_inc); +} + +void Stats::addWrite(uint64_t microseconds, size_t requests_inc, size_t bytes_inc) +{ + write_collector.add(microseconds, requests_inc, bytes_inc); +} + +void Stats::StatsCollector::clear() +{ + requests = 0; + work_time = 0; + requests_bytes = 0; + sampler.clear(); +} + +void Stats::clear() +{ + read_collector.clear(); + write_collector.clear(); +} + +std::pair Stats::StatsCollector::getThroughput(size_t concurrency) +{ + assert(requests != 0); + double seconds = work_time / 1'000'000.0 / concurrency; + + return {requests / seconds, requests_bytes / seconds}; +} + +double Stats::StatsCollector::getPercentile(double percent) +{ + return sampler.quantileNearest(percent / 100.0) / 1000.0; +} + +void Stats::report(size_t concurrency) { std::cerr << "\n"; + const auto & read_requests = read_collector.requests; + const auto & write_requests = write_collector.requests; + /// Avoid zeros, nans or exceptions - if (0 == info->read_requests && 0 == info->write_requests) + if (0 == read_requests && 0 == write_requests) return; - double read_seconds = info->read_work_time / concurrency; - double write_seconds = info->write_work_time / concurrency; + auto [read_rps, read_bps] = read_collector.getThroughput(concurrency); + auto [write_rps, write_bps] = write_collector.getThroughput(concurrency); - std::cerr << "read requests " << info->read_requests << ", write requests " << info->write_requests << ", "; - if (info->errors) - { - std::cerr << "errors " << info->errors << ", "; - } - if (0 != info->read_requests) + std::cerr << "read requests " << read_requests << ", write requests " << write_requests << ", "; + if (errors) + std::cerr << "errors " << errors << ", "; + + if (0 != read_requests) { std::cerr - << "Read RPS: " << (info->read_requests / read_seconds) << ", " - << "Read MiB/s: " << (info->requests_read_bytes / read_seconds / 1048576); - if (0 != info->write_requests) + << "Read RPS: " << read_rps << ", " + << "Read MiB/s: " << read_bps / 1048576; + + if (0 != write_requests) std::cerr << ", "; } - if (0 != info->write_requests) + + if (0 != write_requests) { std::cerr - << "Write RPS: " << (info->write_requests / write_seconds) << ", " - << "Write MiB/s: " << (info->requests_write_bytes / write_seconds / 1048576) << ". " + << "Write RPS: " << write_rps << ", " + << "Write MiB/s: " << write_bps / 1048576 << ". " << "\n"; } std::cerr << "\n"; - auto print_percentile = [&](double percent, Stats::Sampler & sampler) + auto print_percentile = [&](double percent, Stats::StatsCollector & collector) { std::cerr << percent << "%\t\t"; - std::cerr << sampler.quantileNearest(percent / 100.0) << " sec.\t"; + std::cerr << collector.getPercentile(percent) << " msec.\t"; std::cerr << "\n"; }; - if (0 != info->read_requests) + const auto print_all_percentiles = [&](auto & collector) + { + for (int percent = 0; percent <= 90; percent += 10) + print_percentile(percent, collector); + + print_percentile(95, collector); + print_percentile(99, collector); + print_percentile(99.9, collector); + print_percentile(99.99, collector); + }; + + if (0 != read_requests) { std::cerr << "Read sampler:\n"; - for (int percent = 0; percent <= 90; percent += 10) - print_percentile(percent, info->read_sampler); - - print_percentile(95, info->read_sampler); - print_percentile(99, info->read_sampler); - print_percentile(99.9, info->read_sampler); - print_percentile(99.99, info->read_sampler); + print_all_percentiles(read_collector); } - if (0 != info->write_requests) + if (0 != write_requests) { std::cerr << "Write sampler:\n"; - for (int percent = 0; percent <= 90; percent += 10) - print_percentile(percent, info->write_sampler); - - print_percentile(95, info->write_sampler); - print_percentile(99, info->write_sampler); - print_percentile(99.9, info->write_sampler); - print_percentile(99.99, info->write_sampler); + print_all_percentiles(write_collector); } } + +void Stats::writeJSON(DB::WriteBuffer & out, size_t concurrency, int64_t start_timestamp) +{ + using namespace rapidjson; + Document results; + auto & allocator = results.GetAllocator(); + results.SetObject(); + + results.AddMember("timestamp", Value(start_timestamp), allocator); + + const auto get_results = [&](auto & collector) + { + Value specific_results(kObjectType); + + specific_results.AddMember("total_requests", Value(collector.requests), allocator); + + auto [rps, bps] = collector.getThroughput(concurrency); + specific_results.AddMember("requests_per_second", Value(rps), allocator); + specific_results.AddMember("bytes_per_second", Value(bps), allocator); + + Value percentiles(kArrayType); + + const auto add_percentile = [&](double percent) + { + Value percentile(kObjectType); + Value percent_key(fmt::format("{:.2f}", percent).c_str(), allocator); + percentile.AddMember(percent_key, Value(collector.getPercentile(percent)), allocator); + percentiles.PushBack(percentile, allocator); + }; + + for (int percent = 0; percent <= 90; percent += 10) + add_percentile(percent); + + add_percentile(95); + add_percentile(99); + add_percentile(99.9); + add_percentile(99.99); + + specific_results.AddMember("percentiles", percentiles, allocator); + + return specific_results; + }; + + if (read_collector.requests != 0) + results.AddMember("read_results", get_results(read_collector), results.GetAllocator()); + + if (write_collector.requests != 0) + results.AddMember("write_results", get_results(write_collector), results.GetAllocator()); + + StringBuffer strbuf; + strbuf.Clear(); + Writer writer(strbuf); + results.Accept(writer); + + const char * output_string = strbuf.GetString(); + out.write(output_string, strlen(output_string)); +} diff --git a/utils/keeper-bench/Stats.h b/utils/keeper-bench/Stats.h index 1b9a31bb734..bc50588e837 100644 --- a/utils/keeper-bench/Stats.h +++ b/utils/keeper-bench/Stats.h @@ -5,48 +5,38 @@ #include +#include + struct Stats { - std::atomic read_requests{0}; - std::atomic write_requests{0}; size_t errors = 0; - size_t requests_write_bytes = 0; - size_t requests_read_bytes = 0; - double read_work_time = 0; - double write_work_time = 0; using Sampler = ReservoirSampler; - Sampler read_sampler {1 << 16}; - Sampler write_sampler {1 << 16}; - - void addRead(double seconds, size_t requests_inc, size_t bytes_inc) + struct StatsCollector { - read_work_time += seconds; - read_requests += requests_inc; - requests_read_bytes += bytes_inc; - read_sampler.insert(seconds); - } + std::atomic requests{0}; + uint64_t requests_bytes = 0; + uint64_t work_time = 0; + Sampler sampler; - void addWrite(double seconds, size_t requests_inc, size_t bytes_inc) - { - write_work_time += seconds; - write_requests += requests_inc; - requests_write_bytes += bytes_inc; - write_sampler.insert(seconds); - } + /// requests/second, bytes/second + std::pair getThroughput(size_t concurrency); + double getPercentile(double percent); - void clear() - { - read_requests = 0; - write_requests = 0; - read_work_time = 0; - write_work_time = 0; - requests_read_bytes = 0; - requests_write_bytes = 0; - read_sampler.clear(); - write_sampler.clear(); - } + void add(uint64_t microseconds, size_t requests_inc, size_t bytes_inc); + void clear(); + }; + + StatsCollector read_collector; + StatsCollector write_collector; + + void addRead(uint64_t microseconds, size_t requests_inc, size_t bytes_inc); + void addWrite(uint64_t microseconds, size_t requests_inc, size_t bytes_inc); + + void clear(); + + void report(size_t concurrency); + void writeJSON(DB::WriteBuffer & out, size_t concurrency, int64_t start_timestamp); }; -void report(std::shared_ptr & info, size_t concurrency); diff --git a/utils/keeper-bench/example.yaml b/utils/keeper-bench/example.yaml new file mode 100644 index 00000000000..e800e923482 --- /dev/null +++ b/utils/keeper-bench/example.yaml @@ -0,0 +1,117 @@ +concurrency: 20 +iterations: 10000 +delay: 4 +timelimit: 300 +continue_on_errors: true + +connections: + operation_timeout_ms: 3000 + connection_timeout_ms: 40000 + + connection: + secure: false + operation_timeout_ms: 2000 + session_timeout_ms: 2000 + connection_timeout_ms: 50000 + host: "localhost:9181" + sessions: 1 + + host: "localhost:9181" + +generator: + setup: + node: + name: "test3" + node: + name: "test_create" + node: + name: "test4" + node: + name: "test" + data: "somedata" + node: + repeat: 4 + name: + random_string: + size: 15 + data: + random_string: + size: + min_value: 10 + max_value: 20 + node: + repeat: 2 + node: + repeat: 2 + name: + random_string: + size: 12 + name: + random_string: + size: 15 + data: + random_string: + size: + min_value: 10 + max_value: 20 + node: + name: "test2" + data: "somedata" + requests: + create: + path: "/test_create" + name_length: 10 + remove_factor: 0.5 + multi: + size: 20 + create: + path: "/test" + data: + random_string: + size: + min_value: 10 + max_value: 20 + remove_factor: 0.8 + set: + weight: 2 + path: + - "/test3" + - "/test4" + path: + children_of: "/test" + data: + random_string: + size: 10 + get: + path: + - "/test3" + - "/test4" + path: + children_of: "/test" + + multi: + weight: 10 + get: + path: + - "/test3" + - "/test4" + path: + children_of: "/test" + list: + path: + - "/test3" + path: + children_of: "/test" + + list: + path: + - "/test3" + - "/test4" + path: + children_of: "/test" + +output: + file: + path: "output.json" + with_timestamp: true + stdout: true diff --git a/utils/keeper-bench/main.cpp b/utils/keeper-bench/main.cpp index 39af28e7580..0753d66850f 100644 --- a/utils/keeper-bench/main.cpp +++ b/utils/keeper-bench/main.cpp @@ -3,10 +3,24 @@ #include "Runner.h" #include "Stats.h" #include "Generator.h" +#include "Common/Exception.h" #include #include +#include -using namespace std; +namespace +{ + +template +std::optional valueToOptional(const boost::program_options::variable_value & value) +{ + if (value.empty()) + return std::nullopt; + + return value.as(); +} + +} int main(int argc, char *argv[]) { @@ -19,15 +33,14 @@ int main(int argc, char *argv[]) boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth()); desc.add_options() - ("help", "produce help message") - ("generator", value()->default_value("set_small_data"), "query to execute") - ("concurrency,c", value()->default_value(1), "number of parallel queries") - ("delay,d", value()->default_value(1), "delay between intermediate reports in seconds (set 0 to disable reports)") - ("iterations,i", value()->default_value(0), "amount of queries to be executed") - ("timelimit,t", value()->default_value(0.), "stop launch of queries after specified time limit") - ("hosts,h", value()->multitoken(), "") + ("help", "produce help message") + ("config", value()->default_value(""), "yaml/xml file containing configuration") + ("concurrency,c", value(), "number of parallel queries") + ("report-delay,d", value(), "delay between intermediate reports in seconds (set 0 to disable reports)") + ("iterations,i", value(), "amount of queries to be executed") + ("time-limit,t", value(), "stop launch of queries after specified time limit") + ("hosts,h", value()->multitoken()->default_value(Strings{}, ""), "") ("continue_on_errors", "continue testing even if a query fails") - ("reconnect", "establish new connection for every query") ; boost::program_options::variables_map options; @@ -41,15 +54,22 @@ int main(int argc, char *argv[]) return 1; } - Runner runner(options["concurrency"].as(), - options["generator"].as(), - options["hosts"].as(), - options["timelimit"].as(), - options["delay"].as(), - options.count("continue_on_errors"), - options["iterations"].as()); + Runner runner(valueToOptional(options["concurrency"]), + options["config"].as(), + options["hosts"].as(), + valueToOptional(options["time-limit"]), + valueToOptional(options["report-delay"]), + options.count("continue_on_errors") ? std::optional(true) : std::nullopt, + valueToOptional(options["iterations"])); - runner.runBenchmark(); + try + { + runner.runBenchmark(); + } + catch (const DB::Exception & e) + { + std::cout << "Got exception while trying to run benchmark: " << e.message() << std::endl; + } return 0; } diff --git a/utils/list-licenses/list-licenses.sh b/utils/list-licenses/list-licenses.sh index db3eb5e59e8..dd23e6321c8 100755 --- a/utils/list-licenses/list-licenses.sh +++ b/utils/list-licenses/list-licenses.sh @@ -40,14 +40,21 @@ ls -1 -d ${LIBS_PATH}/*/ | ${GREP_CMD} -F -v -- '-cmake' | LC_ALL=C sort | while ${GREP_CMD} -q -i -F 'Altered source versions must be plainly marked as such' "$LIB_LICENSE" && ${GREP_CMD} -q -i -F 'This notice may not be removed or altered' "$LIB_LICENSE" && echo "zLib") || + (${GREP_CMD} -q -i -F 'This program, "bzip2", the associated library "libbzip2"' "$LIB_LICENSE" && + echo "bzip2") || (${GREP_CMD} -q -i -F 'Permission is hereby granted, free of charge, to any person' "$LIB_LICENSE" && - ${GREP_CMD} -q -i -F 'The above copyright notice and this permission notice shall be included' "$LIB_LICENSE" && + ${GREP_CMD} -q -i -F 'The above copyright notice and this permission notice shall be' "$LIB_LICENSE" && ${GREP_CMD} -q -i -F 'THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND' "$LIB_LICENSE" && echo "MIT") || + (${GREP_CMD} -q -F 'PostgreSQL' "$LIB_LICENSE" && + echo "PostgreSQL") || (${GREP_CMD} -q -i -F 'Permission to use, copy, modify, and distribute this software for any purpose' "$LIB_LICENSE" && ${GREP_CMD} -q -i -F 'the name of a copyright holder shall not' "$LIB_LICENSE" && ${GREP_CMD} -q -i -F 'THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND' "$LIB_LICENSE" && echo "MIT/curl") || + (${GREP_CMD} -q -i -F 'OpenLDAP Public License' "$LIB_LICENSE" && + ${GREP_CMD} -q -i -F 'Version 2.8' "$LIB_LICENSE" && + echo "OpenLDAP Version 2.8") || (${GREP_CMD} -q -i -F 'Redistributions of source code must retain the above copyright' "$LIB_LICENSE" && ${GREP_CMD} -q -i -F 'Redistributions in binary form must reproduce' "$LIB_LICENSE" && ${GREP_CMD} -q -i -F 'Neither the name' "$LIB_LICENSE" && @@ -55,6 +62,14 @@ ls -1 -d ${LIBS_PATH}/*/ | ${GREP_CMD} -F -v -- '-cmake' | LC_ALL=C sort | while (${GREP_CMD} -q -i -F 'Redistributions of source code must retain the above copyright' "$LIB_LICENSE" && ${GREP_CMD} -q -i -F 'Redistributions in binary form must reproduce' "$LIB_LICENSE" && echo "BSD 2-clause") || + (${GREP_CMD} -q -i -F 'Permission to use, copy, modify, and distribute this software' "$LIB_LICENSE" && + ${GREP_CMD} -q -i -F 'documentation for any purpose and without fee is hereby granted' "$LIB_LICENSE" && + ${GREP_CMD} -q -i -F 'the above copyright notice appear in all copies and that both that copyright' "$LIB_LICENSE" && + ${GREP_CMD} -q -i -F 'notice and this permission notice appear in supporting documentation' "$LIB_LICENSE" && + ${GREP_CMD} -q -i -F 'not be used in advertising or publicity pertaining' "$LIB_LICENSE" && + ${GREP_CMD} -q -i -F 'distribution of the software without specific, written prior permission' "$LIB_LICENSE" && + ${GREP_CMD} -q -i -F 'makes no representations about the suitability of this software' "$LIB_LICENSE" && + echo "HPND") || echo "Unknown") RELATIVE_PATH=$(echo "$LIB_LICENSE" | sed -r -e 's!^.+/contrib/!/contrib/!') diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index f3aabeec87e..653a0cd5388 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,5 @@ +v23.4.2.11-stable 2023-05-02 +v23.4.1.1943-stable 2023-04-27 v23.3.2.37-lts 2023-04-22 v23.3.1.2823-lts 2023-03-31 v23.2.6.34-stable 2023-04-23 diff --git a/utils/security-generator/generate_security.py b/utils/security-generator/generate_security.py index d25612e8bc6..83180ccce1c 100755 --- a/utils/security-generator/generate_security.py +++ b/utils/security-generator/generate_security.py @@ -48,17 +48,20 @@ A public disclosure date is negotiated by the ClickHouse maintainers and the bug """ -def generate_supported_versions(): +def generate_supported_versions() -> str: with open(VERSIONS_FILE, "r", encoding="utf-8") as fd: versions = [line.split(maxsplit=1)[0][1:] for line in fd.readlines()] # The versions in VERSIONS_FILE are ordered ascending, so the first one is # the greatest one. We may have supported versions in the previous year - unsupported_year = int(versions[0].split(".", maxsplit=1)[0]) - 2 - # 3 supported versions - supported = [] # type: List[str] - # 2 LTS versions, one of them could be in supported + greatest_year = int(versions[0].split(".", maxsplit=1)[0]) + unsupported_year = greatest_year - 2 + # 3 regular versions + regular = [] # type: List[str] + max_regular = 3 + # 2 LTS versions, one of them could be in regular lts = [] # type: List[str] + max_lts = 2 # The rest are unsupported unsupported = [] # type: List[str] table = [ @@ -69,18 +72,21 @@ def generate_supported_versions(): year = int(version.split(".")[0]) month = int(version.split(".")[1]) version = f"{year}.{month}" - if version in supported or version in lts: + to_append = "" + if version in regular or version in lts: continue - if len(supported) < 3: - supported.append(version) - if len(lts) < 2 and month in [3, 8]: - # The version can be LTS as well - lts.append(version) - table.append(f"| {version} | ✔️ |") - continue - if len(lts) < 2 and month in [3, 8]: + if len(regular) < max_regular: + regular.append(version) + to_append = f"| {version} | ✔️ |" + if len(lts) < max_lts and month in [3, 8]: lts.append(version) - table.append(f"| {version} | ✔️ |") + to_append = f"| {version} | ✔️ |" + if to_append: + if len(regular) == max_regular and len(lts) == max_lts: + # if we reached the max number of supported versions, the rest + # are unsopported, so year.* will be used + unsupported_year = min(greatest_year - 1, year) + table.append(to_append) continue if year <= unsupported_year: # The whole year is unsopported @@ -92,7 +98,7 @@ def generate_supported_versions(): return "\n".join(table) + "\n" -def main(): +def main() -> None: print(HEADER) print(generate_supported_versions()) print(FOOTER) diff --git a/utils/tests-visualizer/index.html b/utils/tests-visualizer/index.html index 11b2d6504e4..b2db5dbed33 100644 --- a/utils/tests-visualizer/index.html +++ b/utils/tests-visualizer/index.html @@ -20,9 +20,7 @@ width: 130px; display: block; margin: 30px auto; - -webkit-animation: spin 2s ease-in-out infinite; - -moz-animation: spin 2s ease-in-out infinite; - animation: spin 2s ease-in-out infinite; + animation: spin 10s ease-in-out infinite; } h1 { @@ -45,16 +43,9 @@ cursor: pointer; } - @-moz-keyframes spin { - 100% { -moz-transform: rotate(360deg); } - } - - @-webkit-keyframes spin { - 100% { -webkit-transform: rotate(360deg); } - } - @keyframes spin { - 100% { transform:rotate(360deg); } + 50% { transform:scale(150%); } + 100% { transform:scale(100%); } } @@ -67,33 +58,26 @@