mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-10 09:32:06 +00:00
Merge branch 'master' into threads-in-overcommit-tracker-metric
This commit is contained in:
Alexander Tokmakov
GitHub
commit
ec36ce7bb3
5
.gitignore
vendored
5
.gitignore
vendored
@ -159,8 +159,3 @@ website/package-lock.json
|
||||
tests/queries/0_stateless/test_*
|
||||
tests/queries/0_stateless/*.binary
|
||||
tests/queries/0_stateless/*.generated-expect
|
||||
|
||||
# rust
|
||||
/rust/**/target
|
||||
# It is autogenerated from *.in
|
||||
/rust/**/.cargo/config.toml
|
||||
|
||||
91
CHANGELOG.md
91
CHANGELOG.md
@ -18,13 +18,12 @@
|
||||
### <a id="2212"></a> ClickHouse release 22.12, 2022-12-15
|
||||
|
||||
#### Upgrade Notes
|
||||
* Fixed backward incompatibility in (de)serialization of states of `min`, `max`, `any*`, `argMin`, `argMax` aggregate functions with `String` argument. The incompatibility affects 22.9, 22.10 and 22.11 branches (fixed since 22.9.6, 22.10.4 and 22.11.2 correspondingly). Some minor releases of 22.3, 22.7 and 22.8 branches are also affected: 22.3.13...22.3.14 (fixed since 22.3.15), 22.8.6...22.8.9 (fixed since 22.8.10), 22.7.6 and newer (will not be fixed in 22.7, we recommend to upgrade from 22.7.* to 22.8.10 or newer). This release note does not concern users that have never used affected versions. Incompatible versions append extra `'\0'` to strings when reading states of the aggregate functions mentioned above. For example, if an older version saved state of `anyState('foobar')` to `state_column` then incompatible version will print `'foobar\0'` on `anyMerge(state_column)`. Also incompatible versions write states of the aggregate functions without trailing `'\0'`. Newer versions (that have the fix) can correctly read data written by all versions including incompatible versions, except one corner case. If an incompatible version saved a state with a string that actually ends with null character, then newer version will trim trailing `'\0'` when reading state of affected aggregate function. For example, if an incompatible version saved state of `anyState('abrac\0dabra\0')` to `state_column` then newer versions will print `'abrac\0dabra'` on `anyMerge(state_column)`. The issue also affects distributed queries when an incompatible version works in a cluster together with older or newer versions. [#43038](https://github.com/ClickHouse/ClickHouse/pull/43038) ([Alexander Tokmakov](https://github.com/tavplubix), [Raúl Marín](https://github.com/Algunenano)). Note: all the official ClickHouse builds already include the patches. This is not necessarily true for unofficial third-party builds that should be avoided.
|
||||
* Fixed backward incompatibility in (de)serialization of states of `min`, `max`, `any*`, `argMin`, `argMax` aggregate functions with `String` argument. The incompatibility affects 22.9, 22.10 and 22.11 branches (fixed since 22.9.6, 22.10.4 and 22.11.2 correspondingly). Some minor releases of 22.3, 22.7 and 22.8 branches are also affected: 22.3.13...22.3.14 (fixed since 22.3.15), 22.8.6...22.8.9 (fixed since 22.8.10), 22.7.6 and newer (will not be fixed in 22.7, we recommend upgrading from 22.7.* to 22.8.10 or newer). This release note does not concern users that have never used affected versions. Incompatible versions append an extra `'\0'` to strings when reading states of the aggregate functions mentioned above. For example, if an older version saved state of `anyState('foobar')` to `state_column` then the incompatible version will print `'foobar\0'` on `anyMerge(state_column)`. Also incompatible versions write states of the aggregate functions without trailing `'\0'`. Newer versions (that have the fix) can correctly read data written by all versions including incompatible versions, except one corner case. If an incompatible version saved a state with a string that actually ends with null character, then newer version will trim trailing `'\0'` when reading state of affected aggregate function. For example, if an incompatible version saved state of `anyState('abrac\0dabra\0')` to `state_column` then newer versions will print `'abrac\0dabra'` on `anyMerge(state_column)`. The issue also affects distributed queries when an incompatible version works in a cluster together with older or newer versions. [#43038](https://github.com/ClickHouse/ClickHouse/pull/43038) ([Alexander Tokmakov](https://github.com/tavplubix), [Raúl Marín](https://github.com/Algunenano)). Note: all the official ClickHouse builds already include the patches. This is not necessarily true for unofficial third-party builds that should be avoided.
|
||||
|
||||
#### New Feature
|
||||
* Add `BSONEachRow` input/output format. In this format, ClickHouse formats/parses each row as a separate BSON document and each column is formatted/parsed as a single BSON field with column name as a key. [#42033](https://github.com/ClickHouse/ClickHouse/pull/42033) ([mark-polokhov](https://github.com/mark-polokhov)).
|
||||
* Add `BSONEachRow` input/output format. In this format, ClickHouse formats/parses each row as a separate BSON document and each column is formatted/parsed as a single BSON field with the column name as the key. [#42033](https://github.com/ClickHouse/ClickHouse/pull/42033) ([mark-polokhov](https://github.com/mark-polokhov)).
|
||||
* Add `grace_hash` JOIN algorithm, it can be enabled with `SET join_algorithm = 'grace_hash'`. [#38191](https://github.com/ClickHouse/ClickHouse/pull/38191) ([BigRedEye](https://github.com/BigRedEye), [Vladimir C](https://github.com/vdimir)).
|
||||
* Allow configuring password complexity rules and checks for creating and changing users. [#43719](https://github.com/ClickHouse/ClickHouse/pull/43719) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Add `CREATE / ALTER / DROP NAMED COLLECTION` queries. [#43252](https://github.com/ClickHouse/ClickHouse/pull/43252) ([Kseniia Sumarokova](https://github.com/kssenii)). Restrict default access to named collections for user defined in config. It must have explicit `show_named_collections = 1` to be able to see them. [#43325](https://github.com/ClickHouse/ClickHouse/pull/43325) ([Kseniia Sumarokova](https://github.com/kssenii)). The `system.named_collections` table is introduced [#43147](https://github.com/ClickHouse/ClickHouse/pull/43147) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Mask sensitive information in logs; mask secret parts in the output of queries `SHOW CREATE TABLE` and `SELECT FROM system.tables`. Also resolves [#41418](https://github.com/ClickHouse/ClickHouse/issues/41418). [#43227](https://github.com/ClickHouse/ClickHouse/pull/43227) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Add `GROUP BY ALL` syntax: [#37631](https://github.com/ClickHouse/ClickHouse/issues/37631). [#42265](https://github.com/ClickHouse/ClickHouse/pull/42265) ([刘陶峰](https://github.com/taofengliu)).
|
||||
* Add `FROM table SELECT column` syntax. [#41095](https://github.com/ClickHouse/ClickHouse/pull/41095) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
@ -32,23 +31,24 @@
|
||||
* Added `multiplyDecimal` and `divideDecimal` functions for decimal operations with fixed precision. [#42438](https://github.com/ClickHouse/ClickHouse/pull/42438) ([Andrey Zvonov](https://github.com/zvonand)).
|
||||
* Added `system.moves` table with list of currently moving parts. [#42660](https://github.com/ClickHouse/ClickHouse/pull/42660) ([Sergei Trifonov](https://github.com/serxa)).
|
||||
* Add support for embedded Prometheus endpoint for ClickHouse Keeper. [#43087](https://github.com/ClickHouse/ClickHouse/pull/43087) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Support numeric literals with `_` as separator as, for example, `1_000_000`. [#43925](https://github.com/ClickHouse/ClickHouse/pull/43925) ([jh0x](https://github.com/jh0x)).
|
||||
* Added possibility to use array as a second parameter for `cutURLParameter` function. It will cut multiple parameters. Close [#6827](https://github.com/ClickHouse/ClickHouse/issues/6827). [#43788](https://github.com/ClickHouse/ClickHouse/pull/43788) ([Roman Vasin](https://github.com/rvasin)).
|
||||
* Support numeric literals with `_` as the separator, for example, `1_000_000`. [#43925](https://github.com/ClickHouse/ClickHouse/pull/43925) ([jh0x](https://github.com/jh0x)).
|
||||
* Added possibility to use an array as a second parameter for `cutURLParameter` function. It will cut multiple parameters. Close [#6827](https://github.com/ClickHouse/ClickHouse/issues/6827). [#43788](https://github.com/ClickHouse/ClickHouse/pull/43788) ([Roman Vasin](https://github.com/rvasin)).
|
||||
* Add a column with the expression of the index in the `system.data_skipping_indices` table. [#43308](https://github.com/ClickHouse/ClickHouse/pull/43308) ([Guillaume Tassery](https://github.com/YiuRULE)).
|
||||
* Add column `engine_full` to system table `databases` so that users can access whole engine definition of database via system tables. [#43468](https://github.com/ClickHouse/ClickHouse/pull/43468) ([凌涛](https://github.com/lingtaolf)).
|
||||
* New hash function [xxh3](https://github.com/Cyan4973/xxHash) added. Also performance of `xxHash32` and `xxHash64` improved on arm thanks to library update. [#43411](https://github.com/ClickHouse/ClickHouse/pull/43411) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Add column `engine_full` to system table `databases` so that users can access the entire engine definition of a database via system tables. [#43468](https://github.com/ClickHouse/ClickHouse/pull/43468) ([凌涛](https://github.com/lingtaolf)).
|
||||
* New hash function [xxh3](https://github.com/Cyan4973/xxHash) added. Also, the performance of `xxHash32` and `xxHash64` are improved on ARM thanks to a library update. [#43411](https://github.com/ClickHouse/ClickHouse/pull/43411) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Added support to define constraints for merge tree settings. For example you can forbid overriding the `storage_policy` by users. [#43903](https://github.com/ClickHouse/ClickHouse/pull/43903) ([Sergei Trifonov](https://github.com/serxa)).
|
||||
* Add a new setting `input_format_json_read_objects_as_strings` that allows to parse nested JSON objects into Strings in all JSON input formats. This setting is disabled by default. [#44052](https://github.com/ClickHouse/ClickHouse/pull/44052) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Add a new setting `input_format_json_read_objects_as_strings` that allows the parsing of nested JSON objects into Strings in all JSON input formats. This setting is disabled by default. [#44052](https://github.com/ClickHouse/ClickHouse/pull/44052) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
|
||||
#### Experimental Feature
|
||||
* Support deduplication for asynchronous inserts. Before this change async inserts don't support deduplication, because multiple small inserts will coexist in one inserted batch. Closes [#38075](https://github.com/ClickHouse/ClickHouse/issues/38075). [#43304](https://github.com/ClickHouse/ClickHouse/pull/43304) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* Support deduplication for asynchronous inserts. Before this change, async inserts did not support deduplication, because multiple small inserts coexisted in one inserted batch. Closes [#38075](https://github.com/ClickHouse/ClickHouse/issues/38075). [#43304](https://github.com/ClickHouse/ClickHouse/pull/43304) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* Add support for cosine distance for the experimental Annoy (vector similarity search) index. [#42778](https://github.com/ClickHouse/ClickHouse/pull/42778) ([Filatenkov Artur](https://github.com/FArthur-cmd)).
|
||||
* Add `CREATE / ALTER / DROP NAMED COLLECTION` queries. [#43252](https://github.com/ClickHouse/ClickHouse/pull/43252) ([Kseniia Sumarokova](https://github.com/kssenii)). This feature is under development and the queries are not effective as of version 22.12. This changelog entry is added only to avoid confusion. Restrict default access to named collections to the user defined in config. This requires that `show_named_collections = 1` is set to be able to see them. [#43325](https://github.com/ClickHouse/ClickHouse/pull/43325) ([Kseniia Sumarokova](https://github.com/kssenii)). The `system.named_collections` table is introduced [#43147](https://github.com/ClickHouse/ClickHouse/pull/43147) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
|
||||
#### Performance Improvement
|
||||
* Add settings `max_streams_for_merge_tree_reading` and `allow_asynchronous_read_from_io_pool_for_merge_tree`. Setting `max_streams_for_merge_tree_reading` limits the number of reading streams for MergeTree tables. Setting `allow_asynchronous_read_from_io_pool_for_merge_tree` enables background I/O pool to read from `MergeTree` tables. This may increase performance for I/O bound queries if used together with `max_streams_to_max_threads_ratio` or `max_streams_for_merge_tree_reading`. [#43260](https://github.com/ClickHouse/ClickHouse/pull/43260) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). This improves performance up to 100 times in case of high latency storage, low number of CPU and high number of data parts.
|
||||
* Settings `merge_tree_min_rows_for_concurrent_read_for_remote_filesystem/merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem` did not respect adaptive granularity. Fat rows did not decrease the number of read rows (as it is was done for `merge_tree_min_rows_for_concurrent_read/merge_tree_min_bytes_for_concurrent_read`, which could lead to high memory usage when using remote filesystems. [#43965](https://github.com/ClickHouse/ClickHouse/pull/43965) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Optimized number of list requests to ZooKeeper or Keeper when selecting a part to merge. Previously it could produce thousands of requests in some cases. Fixes [#43647](https://github.com/ClickHouse/ClickHouse/issues/43647). [#43675](https://github.com/ClickHouse/ClickHouse/pull/43675) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Optimisation is getting skipped now if `max_size_to_preallocate_for_aggregation` has too small value. Default value of this setting increased to `10^8`. [#43945](https://github.com/ClickHouse/ClickHouse/pull/43945) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Add settings `max_streams_for_merge_tree_reading` and `allow_asynchronous_read_from_io_pool_for_merge_tree`. Setting `max_streams_for_merge_tree_reading` limits the number of reading streams for MergeTree tables. Setting `allow_asynchronous_read_from_io_pool_for_merge_tree` enables a background I/O pool to read from `MergeTree` tables. This may increase performance for I/O bound queries if used together with `max_streams_to_max_threads_ratio` or `max_streams_for_merge_tree_reading`. [#43260](https://github.com/ClickHouse/ClickHouse/pull/43260) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). This improves performance up to 100 times in case of high latency storage, low number of CPU and high number of data parts.
|
||||
* Settings `merge_tree_min_rows_for_concurrent_read_for_remote_filesystem/merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem` did not respect adaptive granularity. Fat rows did not decrease the number of read rows (as it was done for `merge_tree_min_rows_for_concurrent_read/merge_tree_min_bytes_for_concurrent_read`, which could lead to high memory usage when using remote filesystems. [#43965](https://github.com/ClickHouse/ClickHouse/pull/43965) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Optimized the number of list requests to ZooKeeper or ClickHouse Keeper when selecting a part to merge. Previously it could produce thousands of requests in some cases. Fixes [#43647](https://github.com/ClickHouse/ClickHouse/issues/43647). [#43675](https://github.com/ClickHouse/ClickHouse/pull/43675) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Optimization is getting skipped now if `max_size_to_preallocate_for_aggregation` has too small a value. The default value of this setting increased to `10^8`. [#43945](https://github.com/ClickHouse/ClickHouse/pull/43945) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Speed-up server shutdown by avoiding cleaning up of old data parts. Because it is unnecessary after https://github.com/ClickHouse/ClickHouse/pull/41145. [#43760](https://github.com/ClickHouse/ClickHouse/pull/43760) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
* Merging on initiator now uses the same memory bound approach as merging of local aggregation results if `enable_memory_bound_merging_of_aggregation_results` is set. [#40879](https://github.com/ClickHouse/ClickHouse/pull/40879) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Keeper improvement: try syncing logs to disk in parallel with replication. [#43450](https://github.com/ClickHouse/ClickHouse/pull/43450) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
@ -56,25 +56,24 @@
|
||||
|
||||
#### Improvement
|
||||
* Implement referential dependencies and use them to create tables in the correct order while restoring from a backup. [#43834](https://github.com/ClickHouse/ClickHouse/pull/43834) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Substitute UDFs in `CREATE` query to avoid failures during loading at the startup. Additionally, UDFs can now be used as `DEFAULT` expressions for columns. [#43539](https://github.com/ClickHouse/ClickHouse/pull/43539) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Change how the followed queries delete parts: TRUNCATE TABLE, ALTER TABLE DROP PART, ALTER TABLE DROP PARTITION. Now these queries make empty parts which cover old parts. This makes TRUNCATE query works without exclusive lock which means concurrent reads aren't locked. Also achieved durability in all those queries. If request is succeeded then no resurrected pars appear later. Note that atomicity is achieved only with transaction scope. [#41145](https://github.com/ClickHouse/ClickHouse/pull/41145) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
* Substitute UDFs in `CREATE` query to avoid failures during loading at startup. Additionally, UDFs can now be used as `DEFAULT` expressions for columns. [#43539](https://github.com/ClickHouse/ClickHouse/pull/43539) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Change how the following queries delete parts: TRUNCATE TABLE, ALTER TABLE DROP PART, ALTER TABLE DROP PARTITION. Now, these queries make empty parts which cover the old parts. This makes the TRUNCATE query work without a followedexclusive lock which means concurrent reads aren't locked. Also achieved durability in all those queries. If the request succeeds, then no resurrected parts appear later. Note that atomicity is achieved only with transaction scope. [#41145](https://github.com/ClickHouse/ClickHouse/pull/41145) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
* `SET param_x` query no longer requires manual string serialization for the value of the parameter. For example, query `SET param_a = '[\'a\', \'b\']'` can now be written like `SET param_a = ['a', 'b']`. [#41874](https://github.com/ClickHouse/ClickHouse/pull/41874) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Show read rows in the progress indication while reading from stdin from client. Closes [#43423](https://github.com/ClickHouse/ClickHouse/issues/43423). [#43442](https://github.com/ClickHouse/ClickHouse/pull/43442) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Show read rows in the progress indication while reading from STDIN from client. Closes [#43423](https://github.com/ClickHouse/ClickHouse/issues/43423). [#43442](https://github.com/ClickHouse/ClickHouse/pull/43442) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Show progress bar while reading from s3 table function / engine. [#43454](https://github.com/ClickHouse/ClickHouse/pull/43454) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Progress bar will show both read and written rows. [#43496](https://github.com/ClickHouse/ClickHouse/pull/43496) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
* `filesystemAvailable` and related functions support one optional argument with disk name, and change `filesystemFree` to `filesystemUnreserved`. Closes [#35076](https://github.com/ClickHouse/ClickHouse/issues/35076). [#42064](https://github.com/ClickHouse/ClickHouse/pull/42064) ([flynn](https://github.com/ucasfl)).
|
||||
* Integration with LDAP: increased the default value of search_limit to 256, and added LDAP server config option to change that to an arbitrary value. Closes: [#42276](https://github.com/ClickHouse/ClickHouse/issues/42276). [#42461](https://github.com/ClickHouse/ClickHouse/pull/42461) ([Vasily Nemkov](https://github.com/Enmk)).
|
||||
* Allow to remove sensitive information (see the `query_masking_rules` in the configuration file) from the exception messages as well. Resolves [#41418](https://github.com/ClickHouse/ClickHouse/issues/41418). [#42940](https://github.com/ClickHouse/ClickHouse/pull/42940) ([filimonov](https://github.com/filimonov)).
|
||||
* Support query like `SHOW FULL TABLES ...` for MySQL compatibility. [#43910](https://github.com/ClickHouse/ClickHouse/pull/43910) ([Filatenkov Artur](https://github.com/FArthur-cmd)).
|
||||
* Allow the removal of sensitive information (see the `query_masking_rules` in the configuration file) from the exception messages as well. Resolves [#41418](https://github.com/ClickHouse/ClickHouse/issues/41418). [#42940](https://github.com/ClickHouse/ClickHouse/pull/42940) ([filimonov](https://github.com/filimonov)).
|
||||
* Support queries like `SHOW FULL TABLES ...` for MySQL compatibility. [#43910](https://github.com/ClickHouse/ClickHouse/pull/43910) ([Filatenkov Artur](https://github.com/FArthur-cmd)).
|
||||
* Keeper improvement: Add 4lw command `rqld` which can manually assign a node as leader. [#43026](https://github.com/ClickHouse/ClickHouse/pull/43026) ([JackyWoo](https://github.com/JackyWoo)).
|
||||
* Apply connection timeouts settings for Distributed async INSERT from the query. [#43156](https://github.com/ClickHouse/ClickHouse/pull/43156) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Apply connection timeout settings for Distributed async INSERT from the query. [#43156](https://github.com/ClickHouse/ClickHouse/pull/43156) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* The `unhex` function now supports `FixedString` arguments. [issue42369](https://github.com/ClickHouse/ClickHouse/issues/42369). [#43207](https://github.com/ClickHouse/ClickHouse/pull/43207) ([DR](https://github.com/freedomDR)).
|
||||
* Priority is given to deleting completely expired parts according to the TTL rules, see [#42869](https://github.com/ClickHouse/ClickHouse/issues/42869). [#43222](https://github.com/ClickHouse/ClickHouse/pull/43222) ([zhongyuankai](https://github.com/zhongyuankai)).
|
||||
* More precise and reactive CPU load indication in clickhouse-client. [#43307](https://github.com/ClickHouse/ClickHouse/pull/43307) ([Sergei Trifonov](https://github.com/serxa)).
|
||||
* Support reading of subcolumns of nested types from storage `S3` and table function `s3` with formats `Parquet`, `Arrow` and `ORC`. [#43329](https://github.com/ClickHouse/ClickHouse/pull/43329) ([chen](https://github.com/xiedeyantu)).
|
||||
* Add `table_uuid` column to the `system.parts` table. [#43404](https://github.com/ClickHouse/ClickHouse/pull/43404) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Added client option to display the number of locally processed rows in non-interactive mode (`--print-num-processed-rows`). [#43407](https://github.com/ClickHouse/ClickHouse/pull/43407) ([jh0x](https://github.com/jh0x)).
|
||||
* Implement `aggregation-in-order` optimization on top of query plan. It is enabled by default (but works only together with `optimize_aggregation_in_order`, which is disabled by default). Set `query_plan_aggregation_in_order = 0` to use previous AST-based version. [#43592](https://github.com/ClickHouse/ClickHouse/pull/43592) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Implement `aggregation-in-order` optimization on top of a query plan. It is enabled by default (but works only together with `optimize_aggregation_in_order`, which is disabled by default). Set `query_plan_aggregation_in_order = 0` to use the previous AST-based version. [#43592](https://github.com/ClickHouse/ClickHouse/pull/43592) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Allow to collect profile events with `trace_type = 'ProfileEvent'` to `system.trace_log` on each increment with current stack, profile event name and value of the increment. It can be enabled by the setting `trace_profile_events` and used to investigate performance of queries. [#43639](https://github.com/ClickHouse/ClickHouse/pull/43639) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Add a new setting `input_format_max_binary_string_size` to limit string size in RowBinary format. [#43842](https://github.com/ClickHouse/ClickHouse/pull/43842) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* When ClickHouse requests a remote HTTP server, and it returns an error, the numeric HTTP code was not displayed correctly in the exception message. Closes [#43919](https://github.com/ClickHouse/ClickHouse/issues/43919). [#43920](https://github.com/ClickHouse/ClickHouse/pull/43920) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
@ -82,50 +81,50 @@
|
||||
|
||||
#### Build/Testing/Packaging Improvement
|
||||
|
||||
* Systemd integration now correctly notifies systemd that service is really started and is ready to server requests. [#43400](https://github.com/ClickHouse/ClickHouse/pull/43400) ([Коренберг Марк](https://github.com/socketpair)).
|
||||
* If someone wants, they can build ClickHouse with OpenSSL instead of BoringSSL, and even use dynamic library. This type of build is unsupported and not recommended anyhow. It is not tested and therefore not secure. The use-case is to supply the FIPS 140-2 certified build of OpenSSL. [#43991](https://github.com/ClickHouse/ClickHouse/pull/43991) ([Boris Kuschel](https://github.com/bkuschel)).
|
||||
* This is to upgrade the new `DeflateQpl` compression codec which has been implemented on previous PR (details: https://github.com/ClickHouse/ClickHouse/pull/39494). This patch improves codec on below aspects: 1. QPL v0.2.0 to QPL v0.3.0 [Intel® Query Processing Library (QPL)](https://github.com/intel/qpl) 2. Improve CMake file for fixing QPL build issues for QPL v0.3.0. 3. Link the QPL library with libaccel-config at build time instead of runtime loading on QPL v0.2.0 (dlopen) 4. Fixed log print issue in CompressionCodecDeflateQpl.cpp. [#44024](https://github.com/ClickHouse/ClickHouse/pull/44024) ([jasperzhu](https://github.com/jinjunzh)).
|
||||
* Systemd integration now correctly notifies systemd that the service is really started and is ready to serve requests. [#43400](https://github.com/ClickHouse/ClickHouse/pull/43400) ([Коренберг Марк](https://github.com/socketpair)).
|
||||
* Added the option to build ClickHouse with OpenSSL using the [OpenSSL FIPS Module](https://www.openssl.org/docs/man3.0/man7/fips_module.html). This build type has not been tested to validate security and is not supported. [#43991](https://github.com/ClickHouse/ClickHouse/pull/43991) ([Boris Kuschel](https://github.com/bkuschel)).
|
||||
* Upgrade to the new `DeflateQpl` compression codec which has been implemented in a previous PR (details: https://github.com/ClickHouse/ClickHouse/pull/39494). This patch improves codec on below aspects: 1. QPL v0.2.0 to QPL v0.3.0 [Intel® Query Processing Library (QPL)](https://github.com/intel/qpl) 2. Improve CMake file for fixing QPL build issues for QPL v0.3.0. 3. Link the QPL library with libaccel-config at build time instead of runtime loading on QPL v0.2.0 (dlopen) 4. Fixed log print issue in CompressionCodecDeflateQpl.cpp. [#44024](https://github.com/ClickHouse/ClickHouse/pull/44024) ([jasperzhu](https://github.com/jinjunzh)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in official stable or prestable release)
|
||||
|
||||
* Fixed bug which could lead to deadlock while using asynchronous inserts. [#43233](https://github.com/ClickHouse/ClickHouse/pull/43233) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Fix some incorrect logic in AST level optimization `optimize_normalize_count_variants`. [#43873](https://github.com/ClickHouse/ClickHouse/pull/43873) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
* Fix a case when mutations not making progress when checksums do not match between replicas (e.g. caused by a change in data format on an upgrade). [#36877](https://github.com/ClickHouse/ClickHouse/pull/36877) ([nvartolomei](https://github.com/nvartolomei)).
|
||||
* Fix a case when mutations are not making progress when checksums do not match between replicas (e.g. caused by a change in data format on an upgrade). [#36877](https://github.com/ClickHouse/ClickHouse/pull/36877) ([nvartolomei](https://github.com/nvartolomei)).
|
||||
* Fix the `skip_unavailable_shards` optimization which did not work with the `hdfsCluster` table function. [#43236](https://github.com/ClickHouse/ClickHouse/pull/43236) ([chen](https://github.com/xiedeyantu)).
|
||||
* Fix `s3` support for the `?` wildcard. Closes [#42731](https://github.com/ClickHouse/ClickHouse/issues/42731). [#43253](https://github.com/ClickHouse/ClickHouse/pull/43253) ([chen](https://github.com/xiedeyantu)).
|
||||
* Fix functions `arrayFirstOrNull` and `arrayLastOrNull` or null when array contains `Nullable` elements. [#43274](https://github.com/ClickHouse/ClickHouse/pull/43274) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
* Fix functions `arrayFirstOrNull` and `arrayLastOrNull` or null when the array contains `Nullable` elements. [#43274](https://github.com/ClickHouse/ClickHouse/pull/43274) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
* Fix incorrect `UserTimeMicroseconds`/`SystemTimeMicroseconds` accounting related to Kafka tables. [#42791](https://github.com/ClickHouse/ClickHouse/pull/42791) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Do not suppress exceptions in `web` disks. Fix retries for the `web` disk. [#42800](https://github.com/ClickHouse/ClickHouse/pull/42800) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fixed (logical) race condition between inserts and dropping materialized views. A race condition happened when a Materialized View was dropped at the same time as an INSERT, where the MVs was present as a dependency of the insert at the beggining of the execution, but the table has been dropped by the time the insert chain tries to access to it, producing either an `UNKNOWN_TABLE` or `TABLE_IS_DROPPED` exception, and stopping the insertion. After this change we avoid these exceptions and just continue with the insert if the dependency is gone. [#43161](https://github.com/ClickHouse/ClickHouse/pull/43161) ([AlfVII](https://github.com/AlfVII)).
|
||||
* Fixed (logical) race condition between inserts and dropping materialized views. A race condition happened when a Materialized View was dropped at the same time as an INSERT, where the MVs were present as a dependency of the insert at the begining of the execution, but the table has been dropped by the time the insert chain tries to access it, producing either an `UNKNOWN_TABLE` or `TABLE_IS_DROPPED` exception, and stopping the insertion. After this change, we avoid these exceptions and just continue with the insert if the dependency is gone. [#43161](https://github.com/ClickHouse/ClickHouse/pull/43161) ([AlfVII](https://github.com/AlfVII)).
|
||||
* Fix undefined behavior in the `quantiles` function, which might lead to uninitialized memory. Found by fuzzer. This closes [#44066](https://github.com/ClickHouse/ClickHouse/issues/44066). [#44067](https://github.com/ClickHouse/ClickHouse/pull/44067) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Additional check on zero uncompressed size is added to `CompressionCodecDelta`. [#43255](https://github.com/ClickHouse/ClickHouse/pull/43255) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Flatten arrays from Parquet to avoid an issue with inconsistent data in arrays. These incorrect files can be generated by Apache Iceberg. [#43297](https://github.com/ClickHouse/ClickHouse/pull/43297) ([Arthur Passos](https://github.com/arthurpassos)).
|
||||
* Fix bad cast from `LowCardinality` column when using short circuit function execution. [#43311](https://github.com/ClickHouse/ClickHouse/pull/43311) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Fixed queries with `SAMPLE BY` with prewhere optimization on tables using `Merge` engine. [#43315](https://github.com/ClickHouse/ClickHouse/pull/43315) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Check and compare the content of the `format_version` file in `MergeTreeData` so tables can be loaded even if the storage policy was changed. [#43328](https://github.com/ClickHouse/ClickHouse/pull/43328) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Check and compare the content of the `format_version` file in `MergeTreeData` so that tables can be loaded even if the storage policy was changed. [#43328](https://github.com/ClickHouse/ClickHouse/pull/43328) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Fix possible (very unlikely) "No column to rollback" logical error during INSERT into `Buffer` tables. [#43336](https://github.com/ClickHouse/ClickHouse/pull/43336) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix a bug that allowed the parser to parse an unlimited amount of round brackets into one function if `allow_function_parameters` is set. [#43350](https://github.com/ClickHouse/ClickHouse/pull/43350) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* `MaterializeMySQL` (experimental feature) support DDL： `drop table t1, t2` and compatible with most of MySQL DROP DDL. [#43366](https://github.com/ClickHouse/ClickHouse/pull/43366) ([zzsmdfj](https://github.com/zzsmdfj)).
|
||||
* `session_log` (experimental feature): Fixed the unability to log in (because of failure to create the session_log entry) in a very rare case of messed up setting profiles. [#42641](https://github.com/ClickHouse/ClickHouse/pull/42641) ([Vasily Nemkov](https://github.com/Enmk)).
|
||||
* `session_log` (experimental feature): Fixed the inability to log in (because of failure to create the session_log entry) in a very rare case of messed up setting profiles. [#42641](https://github.com/ClickHouse/ClickHouse/pull/42641) ([Vasily Nemkov](https://github.com/Enmk)).
|
||||
* Fix possible `Cannot create non-empty column with type Nothing` in functions `if`/`multiIf`. Closes [#43356](https://github.com/ClickHouse/ClickHouse/issues/43356). [#43368](https://github.com/ClickHouse/ClickHouse/pull/43368) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Fix a bug when a row level filter uses default value of column. [#43387](https://github.com/ClickHouse/ClickHouse/pull/43387) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Fix a bug when a row level filter uses the default value of a column. [#43387](https://github.com/ClickHouse/ClickHouse/pull/43387) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Query with `DISTINCT` + `LIMIT BY` + `LIMIT` can return fewer rows than expected. Fixes [#43377](https://github.com/ClickHouse/ClickHouse/issues/43377). [#43410](https://github.com/ClickHouse/ClickHouse/pull/43410) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Fix `sumMap` for `Nullable(Decimal(...))`. [#43414](https://github.com/ClickHouse/ClickHouse/pull/43414) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix `date_diff` for hour/minute on macOS. Close [#42742](https://github.com/ClickHouse/ClickHouse/issues/42742). [#43466](https://github.com/ClickHouse/ClickHouse/pull/43466) ([zzsmdfj](https://github.com/zzsmdfj)).
|
||||
* Fix incorrect memory accounting because of merges/mutations. [#43516](https://github.com/ClickHouse/ClickHouse/pull/43516) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fixed primary key analysis with conditions involving `toString(enum)`. [#43596](https://github.com/ClickHouse/ClickHouse/pull/43596) ([Nikita Taranov](https://github.com/nickitat)). This error has been found by @tisonkun.
|
||||
* Ensure consistency when `clickhouse-copier` update status and `attach_is_done` in keeper after partition attach is done. [#43602](https://github.com/ClickHouse/ClickHouse/pull/43602) ([lzydmxy](https://github.com/lzydmxy)).
|
||||
* During recovering of the lost replica of a `Replicated` database (experimental feature) there could a situation where we need to atomically swap two table names (use EXCHANGE), but instead previously we tried to use two RENAME queries. Which was obviously failed and moreover failed the whole recovery process of the database replica. [#43628](https://github.com/ClickHouse/ClickHouse/pull/43628) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Fix the case when `s3Cluster` function throws `NOT_FOUND_COLUMN_IN_BLOCK` error. Closes [#43534](https://github.com/ClickHouse/ClickHouse/issues/43534). [#43629](https://github.com/ClickHouse/ClickHouse/pull/43629) ([chen](https://github.com/xiedeyantu)).
|
||||
* Fix posssible logical error `Array sizes mismatched` while parsing JSON object with arrays with same key names but with different nesting level. Closes [#43569](https://github.com/ClickHouse/ClickHouse/issues/43569). [#43693](https://github.com/ClickHouse/ClickHouse/pull/43693) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Fixed possible exception in case of distributed `GROUP BY` with an `ALIAS` column among aggregation keys. [#43709](https://github.com/ClickHouse/ClickHouse/pull/43709) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Ensure consistency when `clickhouse-copier` updates status and `attach_is_done` in Keeper after partition attach is done. [#43602](https://github.com/ClickHouse/ClickHouse/pull/43602) ([lzydmxy](https://github.com/lzydmxy)).
|
||||
* During the recovery of a lost replica of a `Replicated` database (experimental feature), there could a situation where we need to atomically swap two table names (use EXCHANGE). Previously we tried to use two RENAME queries, which was obviously failing and moreover, failed the whole recovery process of the database replica. [#43628](https://github.com/ClickHouse/ClickHouse/pull/43628) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Fix the case when the `s3Cluster` function throws `NOT_FOUND_COLUMN_IN_BLOCK` error. Closes [#43534](https://github.com/ClickHouse/ClickHouse/issues/43534). [#43629](https://github.com/ClickHouse/ClickHouse/pull/43629) ([chen](https://github.com/xiedeyantu)).
|
||||
* Fix possible logical error `Array sizes mismatched` while parsing JSON object with arrays with same key names but with different nesting level. Closes [#43569](https://github.com/ClickHouse/ClickHouse/issues/43569). [#43693](https://github.com/ClickHouse/ClickHouse/pull/43693) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Fixed possible exception in the case of distributed `GROUP BY` with an `ALIAS` column among aggregation keys. [#43709](https://github.com/ClickHouse/ClickHouse/pull/43709) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Fix bug which can lead to broken projections if zero-copy replication (experimental feature) is enabled and used. [#43764](https://github.com/ClickHouse/ClickHouse/pull/43764) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix using multipart upload for very large S3 objects in AWS S3. [#43824](https://github.com/ClickHouse/ClickHouse/pull/43824) ([ianton-ru](https://github.com/ianton-ru)).
|
||||
* Fixed `ALTER ... RESET SETTING` with `ON CLUSTER`. It could be applied to one replica only. Fixes [#43843](https://github.com/ClickHouse/ClickHouse/issues/43843). [#43848](https://github.com/ClickHouse/ClickHouse/pull/43848) ([Elena Torró](https://github.com/elenatorro)).
|
||||
* Fixed `ALTER ... RESET SETTING` with `ON CLUSTER`. It could have been applied to one replica only. Fixes [#43843](https://github.com/ClickHouse/ClickHouse/issues/43843). [#43848](https://github.com/ClickHouse/ClickHouse/pull/43848) ([Elena Torró](https://github.com/elenatorro)).
|
||||
* Fix a logical error in JOIN with `Join` table engine at right hand side, if `USING` is being used. [#43963](https://github.com/ClickHouse/ClickHouse/pull/43963) ([Vladimir C](https://github.com/vdimir)). Fix a bug with wrong order of keys in `Join` table engine. [#44012](https://github.com/ClickHouse/ClickHouse/pull/44012) ([Vladimir C](https://github.com/vdimir)).
|
||||
* Keeper fix: throw if interserver port for Raft is already in use. [#43984](https://github.com/ClickHouse/ClickHouse/pull/43984) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Keeper fix: throw if the interserver port for Raft is already in use. [#43984](https://github.com/ClickHouse/ClickHouse/pull/43984) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Fix ORDER BY positional argument (example: `ORDER BY 1, 2`) in case of unneeded columns pruning from subqueries. Closes [#43964](https://github.com/ClickHouse/ClickHouse/issues/43964). [#43987](https://github.com/ClickHouse/ClickHouse/pull/43987) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fixed exception when a subquery contains HAVING but doesn't contain actual aggregation. [#44051](https://github.com/ClickHouse/ClickHouse/pull/44051) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Fixed exception when a subquery contains HAVING but doesn't contain an actual aggregation. [#44051](https://github.com/ClickHouse/ClickHouse/pull/44051) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Fix race in s3 multipart upload. This race could cause the error `Part number must be an integer between 1 and 10000, inclusive. (S3_ERROR)` while restoring from a backup. [#44065](https://github.com/ClickHouse/ClickHouse/pull/44065) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
|
||||
|
||||
@ -651,30 +650,30 @@
|
||||
* Add counters (ProfileEvents) for cases when query complexity limitation has been set and has reached (a separate counter for `overflow_mode` = `break` and `throw`). For example, if you have set up `max_rows_to_read` with `read_overflow_mode = 'break'`, looking at the value of `OverflowBreak` counter will allow distinguishing incomplete results. [#40205](https://github.com/ClickHouse/ClickHouse/pull/40205) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix memory accounting in case of "Memory limit exceeded" errors (previously [peak] memory usage was takes failed allocations into account). [#40249](https://github.com/ClickHouse/ClickHouse/pull/40249) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Add metrics for filesystem cache: `FilesystemCacheSize` and `FilesystemCacheElements`. [#40260](https://github.com/ClickHouse/ClickHouse/pull/40260) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Support hadoop secure RPC transfer (hadoop.rpc.protection=privacy and hadoop.rpc.protection=integrity). [#39411](https://github.com/ClickHouse/ClickHouse/pull/39411) ([michael1589](https://github.com/michael1589)).
|
||||
* Support Hadoop secure RPC transfer (hadoop.rpc.protection=privacy and hadoop.rpc.protection=integrity). [#39411](https://github.com/ClickHouse/ClickHouse/pull/39411) ([michael1589](https://github.com/michael1589)).
|
||||
* Avoid continuously growing memory consumption of pattern cache when using functions multi(Fuzzy)Match(Any|AllIndices|AnyIndex)(). [#40264](https://github.com/ClickHouse/ClickHouse/pull/40264) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Add cache for schema inference for file/s3/hdfs/url table functions. Now, schema inference will be performed only on the first query to the file, all subsequent queries to the same file will use the schema from cache if data wasn't changed. Add system table system.schema_inference_cache with all current schemas in cache and system queries SYSTEM DROP SCHEMA CACHE [FOR FILE/S3/HDFS/URL] to drop schemas from cache. [#38286](https://github.com/ClickHouse/ClickHouse/pull/38286) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Add cache for schema inference for file/s3/hdfs/url table functions. Now, schema inference will be performed only on the first query to the file, all subsequent queries to the same file will use the schema from the cache if data has not changed. Add system table system.schema_inference_cache with all current schemas in cache and system queries SYSTEM DROP SCHEMA CACHE [FOR FILE/S3/HDFS/URL] to drop schemas from cache. [#38286](https://github.com/ClickHouse/ClickHouse/pull/38286) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Add support for LARGE_BINARY/LARGE_STRING with Arrow (Closes [#32401](https://github.com/ClickHouse/ClickHouse/issues/32401)). [#40293](https://github.com/ClickHouse/ClickHouse/pull/40293) ([Josh Taylor](https://github.com/joshuataylor)).
|
||||
|
||||
#### Build/Testing/Packaging Improvement
|
||||
|
||||
* [ClickFiddle](https://fiddle.clickhouse.com/): A new tool for testing ClickHouse versions in read/write mode (**Igor Baliuk**).
|
||||
* ClickHouse binary is made self-extracting [#35775](https://github.com/ClickHouse/ClickHouse/pull/35775) ([Yakov Olkhovskiy, Arthur Filatenkov](https://github.com/yakov-olkhovskiy)).
|
||||
* Update tzdata to 2022b to support the new timezone changes. See https://github.com/google/cctz/pull/226. Chile's 2022 DST start is delayed from September 4 to September 11. Iran plans to stop observing DST permanently, after it falls back on 2022-09-21. There are corrections of the historical time zone of Asia/Tehran in the year 1977: Iran adopted standard time in 1935, not 1946. In 1977 it observed DST from 03-21 23:00 to 10-20 24:00; its 1978 transitions were on 03-24 and 08-05, not 03-20 and 10-20; and its spring 1979 transition was on 05-27, not 03-21 (https://data.iana.org/time-zones/tzdb/NEWS). ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Former packages used to install systemd.service file to `/etc`. The files there are marked as `conf` and are not cleaned out, and not updated automatically. This PR cleans them out. [#39323](https://github.com/ClickHouse/ClickHouse/pull/39323) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Update `tzdata` to 2022b to support the new timezone changes. See https://github.com/google/cctz/pull/226. Chile's 2022 DST start is delayed from September 4 to September 11. Iran plans to stop observing DST permanently after it falls back on 2022-09-21. There are corrections to the historical time zone of Asia/Tehran in the year 1977: Iran adopted standard time in 1935, not 1946. In 1977 it observed DST from 03-21 23:00 to 10-20 24:00; its 1978 transitions were on 03-24 and 08-05, not 03-20 and 10-20; and its spring 1979 transition was on 05-27, not 03-21 (https://data.iana.org/time-zones/tzdb/NEWS). ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Former packages used to install systemd.service file to `/etc`. The files there are marked as `conf` and are not cleaned out, and are not updated automatically. This PR cleans them out. [#39323](https://github.com/ClickHouse/ClickHouse/pull/39323) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Ensure LSan is effective. [#39430](https://github.com/ClickHouse/ClickHouse/pull/39430) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* TSAN has issues with clang-14 (https://github.com/google/sanitizers/issues/1552, https://github.com/google/sanitizers/issues/1540), so here we build the TSAN binaries with clang-15. [#39450](https://github.com/ClickHouse/ClickHouse/pull/39450) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Remove the option to build ClickHouse tools as separate executable programs. This fixes [#37847](https://github.com/ClickHouse/ClickHouse/issues/37847). [#39520](https://github.com/ClickHouse/ClickHouse/pull/39520) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Small preparations for build on s390x (which is big-endian). [#39627](https://github.com/ClickHouse/ClickHouse/pull/39627) ([Harry Lee](https://github.com/HarryLeeIBM)). [#39656](https://github.com/ClickHouse/ClickHouse/pull/39656) ([Harry Lee](https://github.com/HarryLeeIBM)). Fixed Endian issue in BitHelpers for s390x. [#39656](https://github.com/ClickHouse/ClickHouse/pull/39656) ([Harry Lee](https://github.com/HarryLeeIBM)). Implement a piece of code related to SipHash for s390x architecture (which is not supported by ClickHouse). [#39732](https://github.com/ClickHouse/ClickHouse/pull/39732) ([Harry Lee](https://github.com/HarryLeeIBM)). Fixed an Endian issue in Coordination snapshot code for s390x architecture (which is not supported by ClickHouse). [#39931](https://github.com/ClickHouse/ClickHouse/pull/39931) ([Harry Lee](https://github.com/HarryLeeIBM)). Fixed Endian issues in Codec code for s390x architecture (which is not supported by ClickHouse). [#40008](https://github.com/ClickHouse/ClickHouse/pull/40008) ([Harry Lee](https://github.com/HarryLeeIBM)). Fixed Endian issues in reading/writing BigEndian binary data in ReadHelpers and WriteHelpers code for s390x architecture (which is not supported by ClickHouse). [#40179](https://github.com/ClickHouse/ClickHouse/pull/40179) ([Harry Lee](https://github.com/HarryLeeIBM)).
|
||||
* Small preparations for build on s390x (which is big-endian). [#39627](https://github.com/ClickHouse/ClickHouse/pull/39627) ([Harry Lee](https://github.com/HarryLeeIBM)). [#39656](https://github.com/ClickHouse/ClickHouse/pull/39656) ([Harry Lee](https://github.com/HarryLeeIBM)). Fixed Endian issue in BitHelpers for s390x. [#39656](https://github.com/ClickHouse/ClickHouse/pull/39656) ([Harry Lee](https://github.com/HarryLeeIBM)). Implement a piece of code related to SipHash for s390x architecture (which is not supported by ClickHouse). [#39732](https://github.com/ClickHouse/ClickHouse/pull/39732) ([Harry Lee](https://github.com/HarryLeeIBM)). Fixed an Endian issue in the Coordination snapshot code for s390x architecture (which is not supported by ClickHouse). [#39931](https://github.com/ClickHouse/ClickHouse/pull/39931) ([Harry Lee](https://github.com/HarryLeeIBM)). Fixed Endian issues in Codec code for s390x architecture (which is not supported by ClickHouse). [#40008](https://github.com/ClickHouse/ClickHouse/pull/40008) ([Harry Lee](https://github.com/HarryLeeIBM)). Fixed Endian issues in reading/writing BigEndian binary data in ReadHelpers and WriteHelpers code for s390x architecture (which is not supported by ClickHouse). [#40179](https://github.com/ClickHouse/ClickHouse/pull/40179) ([Harry Lee](https://github.com/HarryLeeIBM)).
|
||||
* Support build with `clang-16` (trunk). This closes [#39949](https://github.com/ClickHouse/ClickHouse/issues/39949). [#40181](https://github.com/ClickHouse/ClickHouse/pull/40181) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Prepare RISC-V 64 build to run in CI. This is for [#40141](https://github.com/ClickHouse/ClickHouse/issues/40141). [#40197](https://github.com/ClickHouse/ClickHouse/pull/40197) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Simplified function registration macro interface (`FUNCTION_REGISTER*`) to eliminate the step to add and call an extern function in the registerFunctions.cpp, it also makes incremental builds of a new function faster. [#38615](https://github.com/ClickHouse/ClickHouse/pull/38615) ([Li Yin](https://github.com/liyinsg)).
|
||||
* Docker: Now entrypoint.sh in docker image creates and executes chown for all folders it found in config for multidisk setup [#17717](https://github.com/ClickHouse/ClickHouse/issues/17717). [#39121](https://github.com/ClickHouse/ClickHouse/pull/39121) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Docker: Now entrypoint.sh in docker image creates and executes chown for all folders it finds in the config for multidisk setup [#17717](https://github.com/ClickHouse/ClickHouse/issues/17717). [#39121](https://github.com/ClickHouse/ClickHouse/pull/39121) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
|
||||
#### Bug Fix
|
||||
|
||||
* Fix possible segfault in `CapnProto` input format. This bug was found and send through ClickHouse bug-bounty [program](https://github.com/ClickHouse/ClickHouse/issues/38986) by *kiojj*. [#40241](https://github.com/ClickHouse/ClickHouse/pull/40241) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Fix a very rare case of incorrect behavior of array subscript operator. This closes [#28720](https://github.com/ClickHouse/ClickHouse/issues/28720). [#40185](https://github.com/ClickHouse/ClickHouse/pull/40185) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix possible segfault in `CapnProto` input format. This bug was found and sent in through the ClickHouse bug-bounty [program](https://github.com/ClickHouse/ClickHouse/issues/38986) by *kiojj*. [#40241](https://github.com/ClickHouse/ClickHouse/pull/40241) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Fix a very rare case of incorrect behavior of the array subscript operator. This closes [#28720](https://github.com/ClickHouse/ClickHouse/issues/28720). [#40185](https://github.com/ClickHouse/ClickHouse/pull/40185) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix insufficient argument check for encryption functions (found by query fuzzer). This closes [#39987](https://github.com/ClickHouse/ClickHouse/issues/39987). [#40194](https://github.com/ClickHouse/ClickHouse/pull/40194) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix the case when the order of columns can be incorrect if the `IN` operator is used with a table with `ENGINE = Set` containing multiple columns. This fixes [#13014](https://github.com/ClickHouse/ClickHouse/issues/13014). [#40225](https://github.com/ClickHouse/ClickHouse/pull/40225) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix seeking while reading from encrypted disk. This PR fixes [#38381](https://github.com/ClickHouse/ClickHouse/issues/38381). [#39687](https://github.com/ClickHouse/ClickHouse/pull/39687) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
|
||||
@ -609,8 +609,6 @@ if (NATIVE_BUILD_TARGETS
|
||||
"-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}"
|
||||
"-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}"
|
||||
"-DENABLE_CCACHE=${ENABLE_CCACHE}"
|
||||
# Avoid overriding .cargo/config.toml with native toolchain.
|
||||
"-DENABLE_RUST=OFF"
|
||||
"-DENABLE_CLICKHOUSE_SELF_EXTRACTING=${ENABLE_CLICKHOUSE_SELF_EXTRACTING}"
|
||||
${CMAKE_SOURCE_DIR}
|
||||
WORKING_DIRECTORY "${NATIVE_BUILD_DIR}"
|
||||
|
||||
@ -40,11 +40,6 @@ else ()
|
||||
target_compile_definitions(common PUBLIC WITH_COVERAGE=0)
|
||||
endif ()
|
||||
|
||||
# FIXME: move libraries for line reading out from base
|
||||
if (TARGET ch_rust::skim)
|
||||
target_link_libraries(common PUBLIC ch_rust::skim)
|
||||
endif()
|
||||
|
||||
target_include_directories(common PUBLIC .. "${CMAKE_CURRENT_BINARY_DIR}/..")
|
||||
|
||||
if (OS_DARWIN AND NOT USE_STATIC_LIBRARIES)
|
||||
|
||||
@ -16,11 +16,9 @@
|
||||
#include <fstream>
|
||||
#include <filesystem>
|
||||
#include <fmt/format.h>
|
||||
#include "config.h" // USE_SKIM
|
||||
|
||||
#if USE_SKIM
|
||||
#include <skim.h>
|
||||
#endif
|
||||
#include <boost/algorithm/string/split.hpp>
|
||||
#include <boost/algorithm/string/replace.hpp>
|
||||
#include <boost/algorithm/string/classification.hpp> /// is_any_of
|
||||
|
||||
namespace
|
||||
{
|
||||
@ -41,6 +39,36 @@ std::string getEditor()
|
||||
return editor;
|
||||
}
|
||||
|
||||
std::pair<std::string, FuzzyFinderType> getFuzzyFinder()
|
||||
{
|
||||
const char * env_path = std::getenv("PATH"); // NOLINT(concurrency-mt-unsafe)
|
||||
|
||||
if (!env_path || !*env_path)
|
||||
return {};
|
||||
|
||||
std::vector<std::string> paths;
|
||||
boost::split(paths, env_path, boost::is_any_of(":"));
|
||||
for (const auto & path_str : paths)
|
||||
{
|
||||
std::filesystem::path path(path_str);
|
||||
std::filesystem::path sk_bin_path = path / "sk";
|
||||
if (!access(sk_bin_path.c_str(), X_OK))
|
||||
return {sk_bin_path, FUZZY_FINDER_SKIM};
|
||||
|
||||
std::filesystem::path fzf_bin_path = path / "fzf";
|
||||
if (!access(fzf_bin_path.c_str(), X_OK))
|
||||
return {fzf_bin_path, FUZZY_FINDER_FZF};
|
||||
}
|
||||
|
||||
return {"", FUZZY_FINDER_NONE};
|
||||
}
|
||||
|
||||
String escapeShellArgument(std::string arg)
|
||||
{
|
||||
boost::replace_all(arg, "'", "'\\''");
|
||||
return fmt::format("'{}'", arg);
|
||||
}
|
||||
|
||||
/// See comments in ShellCommand::executeImpl()
|
||||
/// (for the vfork via dlsym())
|
||||
int executeCommand(char * const argv[])
|
||||
@ -288,6 +316,8 @@ ReplxxLineReader::ReplxxLineReader(
|
||||
using namespace std::placeholders;
|
||||
using Replxx = replxx::Replxx;
|
||||
|
||||
std::tie(fuzzy_finder, fuzzy_finder_type) = getFuzzyFinder();
|
||||
|
||||
if (!history_file_path.empty())
|
||||
{
|
||||
history_file_fd = open(history_file_path.c_str(), O_RDWR);
|
||||
@ -392,30 +422,17 @@ ReplxxLineReader::ReplxxLineReader(
|
||||
};
|
||||
rx.bind_key(Replxx::KEY::meta('#'), insert_comment_action);
|
||||
|
||||
#if USE_SKIM
|
||||
auto interactive_history_search = [this](char32_t code)
|
||||
/// interactive search in history (requires fzf/sk)
|
||||
if (fuzzy_finder_type != FUZZY_FINDER_NONE)
|
||||
{
|
||||
std::vector<std::string> words;
|
||||
auto interactive_history_search = [this](char32_t code)
|
||||
{
|
||||
auto hs(rx.history_scan());
|
||||
while (hs.next())
|
||||
words.push_back(hs.get().text());
|
||||
}
|
||||
|
||||
std::string new_query(skim(words));
|
||||
if (!new_query.empty())
|
||||
rx.set_state(replxx::Replxx::State(new_query.c_str(), static_cast<int>(new_query.size())));
|
||||
|
||||
if (bracketed_paste_enabled)
|
||||
enableBracketedPaste();
|
||||
|
||||
rx.invoke(Replxx::ACTION::CLEAR_SELF, code);
|
||||
return rx.invoke(Replxx::ACTION::REPAINT, code);
|
||||
};
|
||||
|
||||
/// NOTE: You can use Ctrl-S for non-fuzzy complete.
|
||||
rx.bind_key(Replxx::KEY::control('R'), interactive_history_search);
|
||||
#endif
|
||||
openInteractiveHistorySearch();
|
||||
rx.invoke(Replxx::ACTION::CLEAR_SELF, code);
|
||||
return rx.invoke(Replxx::ACTION::REPAINT, code);
|
||||
};
|
||||
rx.bind_key(Replxx::KEY::control('R'), interactive_history_search);
|
||||
}
|
||||
}
|
||||
|
||||
ReplxxLineReader::~ReplxxLineReader()
|
||||
@ -484,6 +501,65 @@ void ReplxxLineReader::openEditor()
|
||||
enableBracketedPaste();
|
||||
}
|
||||
|
||||
void ReplxxLineReader::openInteractiveHistorySearch()
|
||||
{
|
||||
assert(!fuzzy_finder.empty());
|
||||
TemporaryFile history_file("clickhouse_client_history_in_XXXXXX.bin");
|
||||
auto hs(rx.history_scan());
|
||||
while (hs.next())
|
||||
{
|
||||
history_file.write(hs.get().text());
|
||||
history_file.write(std::string(1, '\0'));
|
||||
}
|
||||
history_file.close();
|
||||
|
||||
TemporaryFile output_file("clickhouse_client_history_out_XXXXXX.sql");
|
||||
output_file.close();
|
||||
|
||||
char sh[] = "sh";
|
||||
char sh_c[] = "-c";
|
||||
/// NOTE: You can use one of the following to configure the behaviour additionally:
|
||||
/// - SKIM_DEFAULT_OPTIONS
|
||||
/// - FZF_DEFAULT_OPTS
|
||||
///
|
||||
/// And also note, that fzf and skim is 95% compatible (at least option
|
||||
/// that is used here)
|
||||
std::string fuzzy_finder_command = fmt::format("{} --read0 --height=30%", fuzzy_finder);
|
||||
switch (fuzzy_finder_type)
|
||||
{
|
||||
case FUZZY_FINDER_SKIM:
|
||||
fuzzy_finder_command += " --tac --tiebreak=-score";
|
||||
break;
|
||||
case FUZZY_FINDER_FZF:
|
||||
fuzzy_finder_command += " --tac --tiebreak=index";
|
||||
break;
|
||||
case FUZZY_FINDER_NONE:
|
||||
/// assertion for !fuzzy_finder.empty() is enough
|
||||
break;
|
||||
}
|
||||
fuzzy_finder_command += fmt::format(" < {} > {}",
|
||||
escapeShellArgument(history_file.getPath()),
|
||||
escapeShellArgument(output_file.getPath()));
|
||||
char * const argv[] = {sh, sh_c, fuzzy_finder_command.data(), nullptr};
|
||||
|
||||
try
|
||||
{
|
||||
if (executeCommand(argv) == 0)
|
||||
{
|
||||
std::string new_query = readFile(output_file.getPath());
|
||||
rightTrim(new_query);
|
||||
rx.set_state(replxx::Replxx::State(new_query.c_str(), static_cast<int>(new_query.size())));
|
||||
}
|
||||
}
|
||||
catch (const std::runtime_error & e)
|
||||
{
|
||||
rx.print(e.what());
|
||||
}
|
||||
|
||||
if (bracketed_paste_enabled)
|
||||
enableBracketedPaste();
|
||||
}
|
||||
|
||||
void ReplxxLineReader::enableBracketedPaste()
|
||||
{
|
||||
bracketed_paste_enabled = true;
|
||||
|
||||
@ -4,6 +4,15 @@
|
||||
|
||||
#include <replxx.hxx>
|
||||
|
||||
enum FuzzyFinderType
|
||||
{
|
||||
FUZZY_FINDER_NONE,
|
||||
/// Use https://github.com/junegunn/fzf
|
||||
FUZZY_FINDER_FZF,
|
||||
/// Use https://github.com/lotabout/skim
|
||||
FUZZY_FINDER_SKIM,
|
||||
};
|
||||
|
||||
class ReplxxLineReader : public LineReader
|
||||
{
|
||||
public:
|
||||
@ -26,6 +35,7 @@ private:
|
||||
void addToHistory(const String & line) override;
|
||||
int executeEditor(const std::string & path);
|
||||
void openEditor();
|
||||
void openInteractiveHistorySearch();
|
||||
|
||||
replxx::Replxx rx;
|
||||
replxx::Replxx::highlighter_callback_t highlighter;
|
||||
@ -35,4 +45,6 @@ private:
|
||||
bool bracketed_paste_enabled = false;
|
||||
|
||||
std::string editor;
|
||||
std::string fuzzy_finder;
|
||||
FuzzyFinderType fuzzy_finder_type = FUZZY_FINDER_NONE;
|
||||
};
|
||||
|
||||
@ -10,6 +10,9 @@ else()
|
||||
endif()
|
||||
|
||||
option(ENABLE_RUST "Enable rust" ${DEFAULT_ENABLE_RUST})
|
||||
|
||||
message(STATUS ${ENABLE_RUST})
|
||||
|
||||
if(NOT ENABLE_RUST)
|
||||
message(STATUS "Not using rust")
|
||||
return()
|
||||
|
||||
@ -55,8 +55,7 @@ ccache --zero-stats ||:
|
||||
if [ "$BUILD_MUSL_KEEPER" == "1" ]
|
||||
then
|
||||
# build keeper with musl separately
|
||||
# and without rust bindings
|
||||
cmake --debug-trycompile -DENABLE_RUST=OFF -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
|
||||
cmake --debug-trycompile -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
|
||||
# shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty.
|
||||
ninja $NINJA_FLAGS clickhouse-keeper
|
||||
|
||||
|
||||
@ -127,23 +127,24 @@ EOL
|
||||
|
||||
function stop()
|
||||
{
|
||||
local max_tries=""
|
||||
if [ -n "$1" ]
|
||||
then
|
||||
max_tries="--max-tries $1"
|
||||
fi
|
||||
|
||||
local pid
|
||||
# Preserve the pid, since the server can hung after the PID will be deleted.
|
||||
pid="$(cat /var/run/clickhouse-server/clickhouse-server.pid)"
|
||||
|
||||
clickhouse stop $max_tries --do-not-kill && return
|
||||
|
||||
if [ -n "$1" ]
|
||||
then
|
||||
# temporarily disable it in BC check
|
||||
clickhouse stop --force
|
||||
return
|
||||
fi
|
||||
|
||||
# We failed to stop the server with SIGTERM. Maybe it hang, let's collect stacktraces.
|
||||
kill -TERM "$(pidof gdb)" ||:
|
||||
sleep 5
|
||||
echo "thread apply all backtrace (on stop)" >> /test_output/gdb.log
|
||||
gdb -batch -ex 'thread apply all backtrace' -p "$pid" | ts '%Y-%m-%d %H:%M:%S' >> /test_output/gdb.log
|
||||
timeout 30m gdb -batch -ex 'thread apply all backtrace' -p "$pid" | ts '%Y-%m-%d %H:%M:%S' >> /test_output/gdb.log
|
||||
clickhouse stop --force
|
||||
}
|
||||
|
||||
@ -431,7 +432,7 @@ else
|
||||
|
||||
clickhouse-client --query="SELECT 'Tables count:', count() FROM system.tables"
|
||||
|
||||
stop 180
|
||||
stop 1
|
||||
mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.backward.stress.log
|
||||
|
||||
# Start new server
|
||||
|
||||
@ -34,7 +34,7 @@ For a description of request parameters, see [request description](../../../sql-
|
||||
`columns` - a tuple with the names of columns where values will be summarized. Optional parameter.
|
||||
The columns must be of a numeric type and must not be in the primary key.
|
||||
|
||||
If `columns` not specified, ClickHouse summarizes the values in all columns with a numeric data type that are not in the primary key.
|
||||
If `columns` is not specified, ClickHouse summarizes the values in all columns with a numeric data type that are not in the primary key.
|
||||
|
||||
### Query clauses
|
||||
|
||||
|
||||
@ -92,7 +92,7 @@ Code: 452, e.displayText() = DB::Exception: Setting force_index_by_date should n
|
||||
**Note:** the `default` profile has special handling: all the constraints defined for the `default` profile become the default constraints, so they restrict all the users until they’re overridden explicitly for these users.
|
||||
|
||||
## Constraints on Merge Tree Settings
|
||||
It is possible to set constraints for [merge tree settings](merge-tree-settings.md). There constraints are applied when table with merge tree engine is created or its storage settings are altered. Name of merge tree setting must be prepended by `merge_tree_` prefix when referenced in `<constraint>` section.
|
||||
It is possible to set constraints for [merge tree settings](merge-tree-settings.md). These constraints are applied when table with merge tree engine is created or its storage settings are altered. Name of merge tree setting must be prepended by `merge_tree_` prefix when referenced in `<constraints>` section.
|
||||
|
||||
**Example:** Forbid to create new tables with explicitly specified `storage_policy`
|
||||
|
||||
|
||||
@ -1,3 +0,0 @@
|
||||
[env]
|
||||
CFLAGS = "@RUST_CFLAGS@"
|
||||
CXXFLAGS = "@RUST_CXXFLAGS@"
|
||||
0
rust/BLAKE3/CMakeLists.txt
Normal file → Executable file
0
rust/BLAKE3/CMakeLists.txt
Normal file → Executable file
@ -1,25 +1 @@
|
||||
function(configure_rustc)
|
||||
# NOTE: this can also be done by overriding rustc, but it not trivial with rustup.
|
||||
set(RUST_CFLAGS "${CMAKE_C_FLAGS}")
|
||||
|
||||
set(CXX_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm-project/libcxx/include")
|
||||
set(RUST_CXXFLAGS "${CMAKE_CXX_FLAGS} -isystem ${CXX_INCLUDE_DIR} ")
|
||||
|
||||
if (CMAKE_OSX_SYSROOT)
|
||||
set(RUST_CXXFLAGS "${RUST_CXXFLAGS} -isysroot ${CMAKE_OSX_SYSROOT}")
|
||||
set(RUST_CFLAGS "${RUST_CFLAGS} -isysroot ${CMAKE_OSX_SYSROOT}")
|
||||
elseif(CMAKE_SYSROOT)
|
||||
set(RUST_CXXFLAGS "${RUST_CXXFLAGS} --sysroot ${CMAKE_SYSROOT}")
|
||||
set(RUST_CFLAGS "${RUST_CFLAGS} --sysroot ${CMAKE_SYSROOT}")
|
||||
endif()
|
||||
|
||||
message(STATUS "RUST_CFLAGS: ${RUST_CFLAGS}")
|
||||
message(STATUS "RUST_CXXFLAGS: ${RUST_CXXFLAGS}")
|
||||
|
||||
# NOTE: requires RW access for the source dir
|
||||
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/.cargo/config.toml.in" "${CMAKE_CURRENT_SOURCE_DIR}/.cargo/config.toml" @ONLY)
|
||||
endfunction()
|
||||
configure_rustc()
|
||||
|
||||
add_subdirectory (BLAKE3)
|
||||
add_subdirectory (skim)
|
||||
|
||||
@ -1,2 +0,0 @@
|
||||
[env]
|
||||
CXXFLAGS = "@RUST_CXXFLAGS@"
|
||||
2
rust/skim/.gitignore
vendored
2
rust/skim/.gitignore
vendored
@ -1,2 +0,0 @@
|
||||
build.rs
|
||||
.cargo/config.toml
|
||||
@ -1,62 +0,0 @@
|
||||
if (OS_FREEBSD)
|
||||
# Right nix/libc requires fspacectl and it had been added only since FreeBSD14.
|
||||
# And sicne sysroot has older libararies you will got undefined reference for clickhouse binary.
|
||||
#
|
||||
# But likely everything should work without this syscall, however it is not
|
||||
# possible right now to gently override libraries versions for depdendcies,
|
||||
# and forking rust modules is a little bit too much for this thing.
|
||||
#
|
||||
# You can take a look at the details in the fillowing issue [1].
|
||||
#
|
||||
# [1]: https://github.com/rust-lang/cargo/issues/5640
|
||||
#
|
||||
message(STATUS "skim is disabled for FreeBSD")
|
||||
return()
|
||||
endif()
|
||||
|
||||
corrosion_import_crate(MANIFEST_PATH Cargo.toml NO_STD)
|
||||
|
||||
set(CXX_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm-project/libcxx/include")
|
||||
# -Wno-dollar-in-identifier-extension: cxx bridge complies names with '$'
|
||||
# -Wno-unused-macros: unused CXXBRIDGE1_RUST_STRING
|
||||
set(CXXBRIDGE_CXXFLAGS "-Wno-dollar-in-identifier-extension -Wno-unused-macros")
|
||||
set(RUST_CXXFLAGS "${CMAKE_CXX_FLAGS} -isystem ${CXX_INCLUDE_DIR} ${CXXBRIDGE_CXXFLAGS}")
|
||||
if (CMAKE_OSX_SYSROOT)
|
||||
set(RUST_CXXFLAGS "${RUST_CXXFLAGS} -isysroot ${CMAKE_OSX_SYSROOT}")
|
||||
elseif(CMAKE_SYSROOT)
|
||||
set(RUST_CXXFLAGS "${RUST_CXXFLAGS} --sysroot ${CMAKE_SYSROOT}")
|
||||
endif()
|
||||
message(STATUS "RUST_CXXFLAGS (for skim): ${RUST_CXXFLAGS}")
|
||||
# NOTE: requires RW access for the source dir
|
||||
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/build.rs.in" "${CMAKE_CURRENT_SOURCE_DIR}/build.rs" @ONLY)
|
||||
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/.cargo/config.toml.in" "${CMAKE_CURRENT_SOURCE_DIR}/.cargo/config.toml" @ONLY)
|
||||
|
||||
set (ffi_binding_generated_path
|
||||
${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/cargo/build/${Rust_CARGO_TARGET_CACHED}/cxxbridge/_ch_rust_skim_rust/src/lib.rs.cc)
|
||||
set (ffi_binding_final_path ${CMAKE_CURRENT_BINARY_DIR}/skim-ffi.cc)
|
||||
message(STATUS "Writing FFI Binding for skim: ${ffi_binding_generated_path} => ${ffi_binding_final_path}")
|
||||
|
||||
add_custom_command(OUTPUT ${ffi_binding_final_path}
|
||||
COMMAND ${CMAKE_COMMAND} -E copy ${ffi_binding_generated_path} ${ffi_binding_final_path}
|
||||
DEPENDS cargo-build__ch_rust_skim_rust)
|
||||
|
||||
add_library(_ch_rust_skim_ffi ${ffi_binding_final_path})
|
||||
if (USE_STATIC_LIBRARIES OR NOT SPLIT_SHARED_LIBRARIES)
|
||||
# static
|
||||
else()
|
||||
if (OS_DARWIN)
|
||||
target_link_libraries(_ch_rust_skim_ffi PRIVATE -Wl,-undefined,dynamic_lookup)
|
||||
else()
|
||||
target_link_libraries(_ch_rust_skim_ffi PRIVATE -Wl,--unresolved-symbols=ignore-all)
|
||||
endif()
|
||||
endif()
|
||||
# cxx bridge compiles such bindings
|
||||
set_target_properties(_ch_rust_skim_ffi PROPERTIES COMPILE_FLAGS "${CXXBRIDGE_CXXFLAGS}")
|
||||
|
||||
add_library(_ch_rust_skim INTERFACE)
|
||||
target_include_directories(_ch_rust_skim INTERFACE include)
|
||||
target_link_libraries(_ch_rust_skim INTERFACE
|
||||
_ch_rust_skim_rust
|
||||
_ch_rust_skim_ffi)
|
||||
|
||||
add_library(ch_rust::skim ALIAS _ch_rust_skim)
|
||||
982
rust/skim/Cargo.lock
generated
982
rust/skim/Cargo.lock
generated
@ -1,982 +0,0 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 3
|
||||
|
||||
[[package]]
|
||||
name = "_ch_rust_skim_rust"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"cxx",
|
||||
"cxx-build",
|
||||
"skim",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "0.7.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "android_system_properties"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arrayvec"
|
||||
version = "0.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
|
||||
|
||||
[[package]]
|
||||
name = "atty"
|
||||
version = "0.2.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
|
||||
|
||||
[[package]]
|
||||
name = "beef"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.77"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f"
|
||||
dependencies = [
|
||||
"iana-time-zone",
|
||||
"js-sys",
|
||||
"num-integer",
|
||||
"num-traits",
|
||||
"time 0.1.45",
|
||||
"wasm-bindgen",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "3.2.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"bitflags",
|
||||
"clap_lex",
|
||||
"indexmap",
|
||||
"once_cell",
|
||||
"strsim",
|
||||
"termcolor",
|
||||
"textwrap",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5"
|
||||
dependencies = [
|
||||
"os_str_bytes",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "codespan-reporting"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e"
|
||||
dependencies = [
|
||||
"termcolor",
|
||||
"unicode-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation-sys"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2801af0d36612ae591caa9568261fddce32ce6e08a7275ea334a06a4ad021a2c"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-channel",
|
||||
"crossbeam-deque",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-queue",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-channel"
|
||||
version = "0.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
"memoffset 0.7.1",
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-queue"
|
||||
version = "0.3.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cxx"
|
||||
version = "1.0.83"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bdf07d07d6531bfcdbe9b8b739b104610c6508dcc4d63b410585faf338241daf"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cxxbridge-flags",
|
||||
"cxxbridge-macro",
|
||||
"link-cplusplus",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cxx-build"
|
||||
version = "1.0.83"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2eb5b96ecdc99f72657332953d4d9c50135af1bac34277801cc3937906ebd39"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"codespan-reporting",
|
||||
"once_cell",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"scratch",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cxxbridge-flags"
|
||||
version = "1.0.83"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac040a39517fd1674e0f32177648334b0f4074625b5588a64519804ba0553b12"
|
||||
|
||||
[[package]]
|
||||
name = "cxxbridge-macro"
|
||||
version = "1.0.83"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1362b0ddcfc4eb0a1f57b68bd77dd99f0e826958a96abd0ae9bd092e114ffed6"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling"
|
||||
version = "0.14.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b0dd3cd20dc6b5a876612a6e5accfe7f3dd883db6d07acfbf14c128f61550dfa"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"darling_macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_core"
|
||||
version = "0.14.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a784d2ccaf7c98501746bf0be29b2022ba41fd62a2e622af997a03e9f972859f"
|
||||
dependencies = [
|
||||
"fnv",
|
||||
"ident_case",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"strsim",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_macro"
|
||||
version = "0.14.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7618812407e9402654622dd402b0a89dff9ba93badd6540781526117b92aab7e"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "defer-drop"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f613ec9fa66a6b28cdb1842b27f9adf24f39f9afc4dcdd9fdecee4aca7945c57"
|
||||
dependencies = [
|
||||
"crossbeam-channel",
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_builder"
|
||||
version = "0.11.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d07adf7be193b71cc36b193d0f5fe60b918a3a9db4dad0449f57bcfd519704a3"
|
||||
dependencies = [
|
||||
"derive_builder_macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_builder_core"
|
||||
version = "0.11.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1f91d4cfa921f1c05904dc3c57b4a32c38aed3340cce209f3a6fd1478babafc4"
|
||||
dependencies = [
|
||||
"darling",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_builder_macro"
|
||||
version = "0.11.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f0314b72bed045f3a68671b3c86328386762c93f82d98c65c3cb5e5f573dd68"
|
||||
dependencies = [
|
||||
"derive_builder_core",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-next"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"dirs-sys-next",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-sys-next"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"redox_users",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797"
|
||||
|
||||
[[package]]
|
||||
name = "env_logger"
|
||||
version = "0.9.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"humantime",
|
||||
"log",
|
||||
"regex",
|
||||
"termcolor",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fnv"
|
||||
version = "1.0.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
|
||||
|
||||
[[package]]
|
||||
name = "fuzzy-matcher"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "54614a3312934d066701a80f20f15fa3b56d67ac7722b39eea5b4c9dd1d66c94"
|
||||
dependencies = [
|
||||
"thread_local",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi 0.11.0+wasi-snapshot-preview1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
|
||||
|
||||
[[package]]
|
||||
name = "hermit-abi"
|
||||
version = "0.1.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "humantime"
|
||||
version = "2.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.53"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765"
|
||||
dependencies = [
|
||||
"android_system_properties",
|
||||
"core-foundation-sys",
|
||||
"iana-time-zone-haiku",
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone-haiku"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca"
|
||||
dependencies = [
|
||||
"cxx",
|
||||
"cxx-build",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ident_case"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "1.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"hashbrown",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.60"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47"
|
||||
dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.138"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8"
|
||||
|
||||
[[package]]
|
||||
name = "link-cplusplus"
|
||||
version = "1.0.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9272ab7b96c9046fbc5bc56c06c117cb639fe2d509df0c421cad82d2915cf369"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.6.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nix"
|
||||
version = "0.24.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nix"
|
||||
version = "0.25.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"bitflags",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"memoffset 0.6.5",
|
||||
"pin-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-integer"
|
||||
version = "0.1.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num_cpus"
|
||||
version = "1.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
|
||||
|
||||
[[package]]
|
||||
name = "os_str_bytes"
|
||||
version = "6.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee"
|
||||
|
||||
[[package]]
|
||||
name = "pin-utils"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7"
|
||||
dependencies = [
|
||||
"either",
|
||||
"rayon-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon-core"
|
||||
version = "1.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3"
|
||||
dependencies = [
|
||||
"crossbeam-channel",
|
||||
"crossbeam-deque",
|
||||
"crossbeam-utils",
|
||||
"num_cpus",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.2.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_users"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"redox_syscall",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.6.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
|
||||
|
||||
[[package]]
|
||||
name = "rustversion"
|
||||
version = "1.0.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8"
|
||||
|
||||
[[package]]
|
||||
name = "scopeguard"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
|
||||
|
||||
[[package]]
|
||||
name = "scratch"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.149"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "256b9932320c590e707b94576e3cc1f7c9024d0ee6612dfbcf1cb106cbe8e055"
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"
|
||||
|
||||
[[package]]
|
||||
name = "skim"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cebed5f897cd6c0d80fbe30adb36c0abf7400e93043a63ae56458495642b3485"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"beef",
|
||||
"bitflags",
|
||||
"chrono",
|
||||
"clap",
|
||||
"crossbeam",
|
||||
"defer-drop",
|
||||
"derive_builder",
|
||||
"env_logger",
|
||||
"fuzzy-matcher",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"nix 0.25.1",
|
||||
"rayon",
|
||||
"regex",
|
||||
"shlex",
|
||||
"time 0.3.17",
|
||||
"timer",
|
||||
"tuikit",
|
||||
"unicode-width",
|
||||
"vte",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.105"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "term"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f"
|
||||
dependencies = [
|
||||
"dirs-next",
|
||||
"rustversion",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "termcolor"
|
||||
version = "1.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
|
||||
dependencies = [
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "textwrap"
|
||||
version = "0.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d"
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "1.0.37"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "1.0.37"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thread_local"
|
||||
version = "1.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.1.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"wasi 0.10.0+wasi-snapshot-preview1",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.3.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"time-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time-core"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd"
|
||||
|
||||
[[package]]
|
||||
name = "timer"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "31d42176308937165701f50638db1c31586f183f1aab416268216577aec7306b"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tuikit"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e19c6ab038babee3d50c8c12ff8b910bdb2196f62278776422f50390d8e53d8"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"nix 0.24.3",
|
||||
"term",
|
||||
"unicode-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-width"
|
||||
version = "0.1.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372"
|
||||
|
||||
[[package]]
|
||||
name = "vte"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1aae21c12ad2ec2d168c236f369c38ff332bc1134f7246350dca641437365045"
|
||||
dependencies = [
|
||||
"arrayvec",
|
||||
"utf8parse",
|
||||
"vte_generate_state_changes",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "vte_generate_state_changes"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d257817081c7dffcdbab24b9e62d2def62e2ff7d00b1c20062551e6cccc145ff"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.10.0+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.11.0+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.83"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"wasm-bindgen-macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-backend"
|
||||
version = "0.2.83"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"log",
|
||||
"once_cell",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.83"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"wasm-bindgen-macro-support",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.83"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"wasm-bindgen-backend",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.83"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f"
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
||||
dependencies = [
|
||||
"winapi-i686-pc-windows-gnu",
|
||||
"winapi-x86_64-pc-windows-gnu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-i686-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-util"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-x86_64-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
@ -1,19 +0,0 @@
|
||||
[package]
|
||||
name = "_ch_rust_skim_rust"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
skim = "0.10.2"
|
||||
cxx = "1.0.83"
|
||||
|
||||
[build-dependencies]
|
||||
cxx-build = "1.0.83"
|
||||
|
||||
[lib]
|
||||
crate-type = ["staticlib"]
|
||||
|
||||
[profile.release]
|
||||
debug = true
|
||||
@ -1,8 +0,0 @@
|
||||
fn main() {
|
||||
let mut build = cxx_build::bridge("src/lib.rs");
|
||||
for flag in "@RUST_CXXFLAGS@".split(' ') {
|
||||
build.flag(flag);
|
||||
}
|
||||
build.compile("skim");
|
||||
println!("cargo:rerun-if-changed=src/lib.rs");
|
||||
}
|
||||
@ -1,90 +0,0 @@
|
||||
/// This header was compiled with:
|
||||
///
|
||||
/// $ cxxbridge rust/skim/src/lib.rs --header
|
||||
///
|
||||
/// For more info [1].
|
||||
///
|
||||
/// [1]: https://cxx.rs/build/other.html
|
||||
|
||||
#pragma once
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace rust {
|
||||
inline namespace cxxbridge1 {
|
||||
// #include "rust/cxx.h"
|
||||
|
||||
struct unsafe_bitcopy_t;
|
||||
|
||||
#ifndef CXXBRIDGE1_RUST_STRING
|
||||
#define CXXBRIDGE1_RUST_STRING
|
||||
class String final {
|
||||
public:
|
||||
String() noexcept;
|
||||
String(const String &) noexcept;
|
||||
String(String &&) noexcept;
|
||||
~String() noexcept;
|
||||
|
||||
String(const std::string &);
|
||||
String(const char *);
|
||||
String(const char *, std::size_t);
|
||||
String(const char16_t *);
|
||||
String(const char16_t *, std::size_t);
|
||||
|
||||
static String lossy(const std::string &) noexcept;
|
||||
static String lossy(const char *) noexcept;
|
||||
static String lossy(const char *, std::size_t) noexcept;
|
||||
static String lossy(const char16_t *) noexcept;
|
||||
static String lossy(const char16_t *, std::size_t) noexcept;
|
||||
|
||||
String &operator=(const String &) &noexcept;
|
||||
String &operator=(String &&) &noexcept;
|
||||
|
||||
explicit operator std::string() const;
|
||||
|
||||
const char *data() const noexcept;
|
||||
std::size_t size() const noexcept;
|
||||
std::size_t length() const noexcept;
|
||||
bool empty() const noexcept;
|
||||
|
||||
const char *c_str() noexcept;
|
||||
|
||||
std::size_t capacity() const noexcept;
|
||||
void reserve(size_t new_cap) noexcept;
|
||||
|
||||
using iterator = char *;
|
||||
iterator begin() noexcept;
|
||||
iterator end() noexcept;
|
||||
|
||||
using const_iterator = const char *;
|
||||
const_iterator begin() const noexcept;
|
||||
const_iterator end() const noexcept;
|
||||
const_iterator cbegin() const noexcept;
|
||||
const_iterator cend() const noexcept;
|
||||
|
||||
bool operator==(const String &) const noexcept;
|
||||
bool operator!=(const String &) const noexcept;
|
||||
bool operator<(const String &) const noexcept;
|
||||
bool operator<=(const String &) const noexcept;
|
||||
bool operator>(const String &) const noexcept;
|
||||
bool operator>=(const String &) const noexcept;
|
||||
|
||||
void swap(String &) noexcept;
|
||||
|
||||
String(unsafe_bitcopy_t, const String &) noexcept;
|
||||
|
||||
private:
|
||||
struct lossy_t;
|
||||
String(lossy_t, const char *, std::size_t) noexcept;
|
||||
String(lossy_t, const char16_t *, std::size_t) noexcept;
|
||||
friend void swap(String &lhs, String &rhs) noexcept { lhs.swap(rhs); }
|
||||
|
||||
std::array<std::uintptr_t, 3> repr;
|
||||
};
|
||||
#endif // CXXBRIDGE1_RUST_STRING
|
||||
} // namespace cxxbridge1
|
||||
} // namespace rust
|
||||
|
||||
::rust::String skim(::std::vector<::std::string> const &words) noexcept;
|
||||
@ -1,49 +0,0 @@
|
||||
use skim::prelude::*;
|
||||
use cxx::{CxxString, CxxVector};
|
||||
|
||||
#[cxx::bridge]
|
||||
mod ffi {
|
||||
extern "Rust" {
|
||||
fn skim(words: &CxxVector<CxxString>) -> String;
|
||||
}
|
||||
}
|
||||
|
||||
struct Item {
|
||||
text: String,
|
||||
}
|
||||
impl SkimItem for Item {
|
||||
fn text(&self) -> Cow<str> {
|
||||
return Cow::Borrowed(&self.text);
|
||||
}
|
||||
}
|
||||
|
||||
fn skim(words: &CxxVector<CxxString>) -> String {
|
||||
// TODO: configure colors
|
||||
let options = SkimOptionsBuilder::default()
|
||||
.height(Some("30%"))
|
||||
.tac(true)
|
||||
.tiebreak(Some("-score".to_string()))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let (tx, rx): (SkimItemSender, SkimItemReceiver) = unbounded();
|
||||
for word in words {
|
||||
tx.send(Arc::new(Item{ text: word.to_string() })).unwrap();
|
||||
}
|
||||
// so that skim could know when to stop waiting for more items.
|
||||
drop(tx);
|
||||
|
||||
let output = Skim::run_with(&options, Some(rx));
|
||||
if output.is_none() {
|
||||
return "".to_string();
|
||||
}
|
||||
let output = output.unwrap();
|
||||
if output.is_abort {
|
||||
return "".to_string();
|
||||
}
|
||||
|
||||
if output.selected_items.is_empty() {
|
||||
return "".to_string();
|
||||
}
|
||||
return output.selected_items[0].output().to_string();
|
||||
}
|
||||
@ -181,6 +181,7 @@ OperationID BackupsWorker::startMakingBackup(const ASTPtr & query, const Context
|
||||
/// For ON CLUSTER queries we will need to change some settings.
|
||||
/// For ASYNC queries we have to clone the context anyway.
|
||||
context_in_use = mutable_context = Context::createCopy(context);
|
||||
mutable_context->makeQueryContext();
|
||||
}
|
||||
|
||||
if (backup_settings.async)
|
||||
@ -400,6 +401,7 @@ OperationID BackupsWorker::startRestoring(const ASTPtr & query, ContextMutablePt
|
||||
/// For ON CLUSTER queries we will need to change some settings.
|
||||
/// For ASYNC queries we have to clone the context anyway.
|
||||
context_in_use = Context::createCopy(context);
|
||||
context_in_use->makeQueryContext();
|
||||
}
|
||||
|
||||
if (restore_settings.async)
|
||||
|
||||
@ -346,7 +346,7 @@ void RestorerFromBackup::findTableInBackup(const QualifiedTableName & table_name
|
||||
res_table_info.has_data = backup->hasFiles(data_path_in_backup);
|
||||
res_table_info.data_path_in_backup = data_path_in_backup;
|
||||
|
||||
tables_dependencies.addDependencies(table_name, getDependenciesFromCreateQuery(context->getGlobalContext(), table_name, create_table_query));
|
||||
tables_dependencies.addDependencies(table_name, getDependenciesFromCreateQuery(context, table_name, create_table_query));
|
||||
|
||||
if (partitions)
|
||||
{
|
||||
@ -674,6 +674,7 @@ void RestorerFromBackup::removeUnresolvedDependencies()
|
||||
void RestorerFromBackup::createTables()
|
||||
{
|
||||
/// We need to create tables considering their dependencies.
|
||||
tables_dependencies.log();
|
||||
auto tables_to_create = tables_dependencies.getTablesSortedByDependency();
|
||||
for (const auto & table_id : tables_to_create)
|
||||
{
|
||||
|
||||
@ -123,16 +123,13 @@ void ProgressIndication::writeFinalProgress()
|
||||
if (progress.read_rows < 1000)
|
||||
return;
|
||||
|
||||
UInt64 processed_rows = progress.read_rows + progress.written_rows;
|
||||
UInt64 processed_bytes = progress.read_bytes + progress.written_bytes;
|
||||
|
||||
std::cout << "Processed " << formatReadableQuantity(processed_rows) << " rows, "
|
||||
<< formatReadableSizeWithDecimalSuffix(processed_bytes);
|
||||
std::cout << "Processed " << formatReadableQuantity(progress.read_rows) << " rows, "
|
||||
<< formatReadableSizeWithDecimalSuffix(progress.read_bytes);
|
||||
|
||||
UInt64 elapsed_ns = getElapsedNanoseconds();
|
||||
if (elapsed_ns)
|
||||
std::cout << " (" << formatReadableQuantity(processed_rows * 1000000000.0 / elapsed_ns) << " rows/s., "
|
||||
<< formatReadableSizeWithDecimalSuffix(processed_bytes * 1000000000.0 / elapsed_ns) << "/s.)";
|
||||
std::cout << " (" << formatReadableQuantity(progress.read_rows * 1000000000.0 / elapsed_ns) << " rows/s., "
|
||||
<< formatReadableSizeWithDecimalSuffix(progress.read_bytes * 1000000000.0 / elapsed_ns) << "/s.)";
|
||||
else
|
||||
std::cout << ". ";
|
||||
}
|
||||
@ -167,18 +164,16 @@ void ProgressIndication::writeProgress(WriteBufferFromFileDescriptor & message)
|
||||
|
||||
size_t prefix_size = message.count();
|
||||
|
||||
UInt64 processed_rows = progress.read_rows + progress.written_rows;
|
||||
UInt64 processed_bytes = progress.read_bytes + progress.written_bytes;
|
||||
message << indicator << " Progress: ";
|
||||
message
|
||||
<< formatReadableQuantity(processed_rows) << " rows, "
|
||||
<< formatReadableSizeWithDecimalSuffix(processed_bytes);
|
||||
<< formatReadableQuantity(progress.read_rows) << " rows, "
|
||||
<< formatReadableSizeWithDecimalSuffix(progress.read_bytes);
|
||||
|
||||
UInt64 elapsed_ns = getElapsedNanoseconds();
|
||||
if (elapsed_ns)
|
||||
message << " ("
|
||||
<< formatReadableQuantity(processed_rows * 1000000000.0 / elapsed_ns) << " rows/s., "
|
||||
<< formatReadableSizeWithDecimalSuffix(processed_bytes * 1000000000.0 / elapsed_ns) << "/s.) ";
|
||||
<< formatReadableQuantity(progress.read_rows * 1000000000.0 / elapsed_ns) << " rows/s., "
|
||||
<< formatReadableSizeWithDecimalSuffix(progress.read_bytes * 1000000000.0 / elapsed_ns) << "/s.) ";
|
||||
else
|
||||
message << ". ";
|
||||
|
||||
|
||||
@ -52,5 +52,4 @@
|
||||
#cmakedefine01 USE_ODBC
|
||||
#cmakedefine01 USE_BORINGSSL
|
||||
#cmakedefine01 USE_BLAKE3
|
||||
#cmakedefine01 USE_SKIM
|
||||
#cmakedefine01 USE_OPENSSL_INTREE
|
||||
|
||||
@ -222,6 +222,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
|
||||
M(UInt64, max_concurrent_queries_for_user, 0, "The maximum number of concurrent requests per user.", 0) \
|
||||
\
|
||||
M(Bool, insert_deduplicate, true, "For INSERT queries in the replicated table, specifies that deduplication of insertings blocks should be performed", 0) \
|
||||
M(Bool, async_insert_deduplicate, false, "For async INSERT queries in the replicated table, specifies that deduplication of insertings blocks should be performed", 0) \
|
||||
\
|
||||
M(UInt64Auto, insert_quorum, 0, "For INSERT queries in the replicated table, wait writing for the specified number of replicas and linearize the addition of the data. 0 - disabled, 'auto' - use majority", 0) \
|
||||
M(Milliseconds, insert_quorum_timeout, 600000, "If the quorum of replicas did not meet in specified time (in milliseconds), exception will be thrown and insertion is aborted.", 0) \
|
||||
|
||||
@ -1,12 +1,17 @@
|
||||
#include <Databases/DDLDependencyVisitor.h>
|
||||
#include <Dictionaries/getDictionaryConfigurationFromAST.h>
|
||||
#include <Interpreters/Cluster.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Interpreters/InDepthNodeVisitor.h>
|
||||
#include <Interpreters/evaluateConstantExpression.h>
|
||||
#include <Interpreters/getClusterName.h>
|
||||
#include <Parsers/ASTCreateQuery.h>
|
||||
#include <Parsers/ASTFunction.h>
|
||||
#include <Parsers/ASTIdentifier.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
#include <Parsers/ASTSelectWithUnionQuery.h>
|
||||
#include <Parsers/ASTTablesInSelectQuery.h>
|
||||
#include <Common/KnownObjectNames.h>
|
||||
#include <Poco/String.h>
|
||||
|
||||
|
||||
@ -15,225 +20,401 @@ namespace DB
|
||||
|
||||
namespace
|
||||
{
|
||||
/// CREATE TABLE or CREATE DICTIONARY or CREATE VIEW or CREATE TEMPORARY TABLE or CREATE DATABASE query.
|
||||
void visitCreateQuery(const ASTCreateQuery & create, DDLDependencyVisitor::Data & data)
|
||||
/// Data for DDLDependencyVisitor.
|
||||
/// Used to visits ASTCreateQuery and extracts the names of all tables explicitly referenced in the create query.
|
||||
class DDLDependencyVisitorData
|
||||
{
|
||||
QualifiedTableName to_table{create.to_table_id.database_name, create.to_table_id.table_name};
|
||||
if (!to_table.table.empty())
|
||||
public:
|
||||
DDLDependencyVisitorData(const ContextPtr & context_, const QualifiedTableName & table_name_, const ASTPtr & ast_)
|
||||
: create_query(ast_), table_name(table_name_), current_database(context_->getCurrentDatabase()), context(context_)
|
||||
{
|
||||
/// TO target_table (for materialized views)
|
||||
if (to_table.database.empty())
|
||||
to_table.database = data.default_database;
|
||||
data.dependencies.emplace(to_table);
|
||||
}
|
||||
|
||||
QualifiedTableName as_table{create.as_database, create.as_table};
|
||||
if (!as_table.table.empty())
|
||||
/// Acquire the result of visiting the create query.
|
||||
TableNamesSet getDependencies() &&
|
||||
{
|
||||
/// AS table_name
|
||||
if (as_table.database.empty())
|
||||
as_table.database = data.default_database;
|
||||
data.dependencies.emplace(as_table);
|
||||
}
|
||||
}
|
||||
|
||||
/// ASTTableExpression represents a reference to a table in SELECT query.
|
||||
/// DDLDependencyVisitor should handle ASTTableExpression because some CREATE queries can contain SELECT queries after AS
|
||||
/// (for example, CREATE VIEW).
|
||||
void visitTableExpression(const ASTTableExpression & expr, DDLDependencyVisitor::Data & data)
|
||||
{
|
||||
if (!expr.database_and_table_name)
|
||||
return;
|
||||
|
||||
const ASTIdentifier * identifier = dynamic_cast<const ASTIdentifier *>(expr.database_and_table_name.get());
|
||||
if (!identifier)
|
||||
return;
|
||||
|
||||
auto table_identifier = identifier->createTable();
|
||||
if (!table_identifier)
|
||||
return;
|
||||
|
||||
QualifiedTableName qualified_name{table_identifier->getDatabaseName(), table_identifier->shortName()};
|
||||
if (qualified_name.table.empty())
|
||||
return;
|
||||
|
||||
if (qualified_name.database.empty())
|
||||
{
|
||||
/// It can be table/dictionary from default database or XML dictionary, but we cannot distinguish it here.
|
||||
qualified_name.database = data.default_database;
|
||||
dependencies.erase(table_name);
|
||||
return std::move(dependencies);
|
||||
}
|
||||
|
||||
data.dependencies.emplace(qualified_name);
|
||||
}
|
||||
bool needChildVisit(const ASTPtr & child) const { return !skip_asts.contains(child.get()); }
|
||||
|
||||
/// Extracts a table name with optional database written in the form db_name.table_name (as identifier) or 'db_name.table_name' (as string).
|
||||
void extractQualifiedTableNameFromArgument(const ASTFunction & function, DDLDependencyVisitor::Data & data, size_t arg_idx)
|
||||
{
|
||||
/// Just ignore incorrect arguments, proper exception will be thrown later
|
||||
if (!function.arguments || function.arguments->children.size() <= arg_idx)
|
||||
return;
|
||||
|
||||
QualifiedTableName qualified_name;
|
||||
|
||||
const auto * expr_list = function.arguments->as<ASTExpressionList>();
|
||||
if (!expr_list)
|
||||
return;
|
||||
|
||||
const auto * arg = expr_list->children[arg_idx].get();
|
||||
if (const auto * literal = arg->as<ASTLiteral>())
|
||||
void visit(const ASTPtr & ast)
|
||||
{
|
||||
if (literal->value.getType() != Field::Types::String)
|
||||
if (auto * create = ast->as<ASTCreateQuery>())
|
||||
{
|
||||
visitCreateQuery(*create);
|
||||
}
|
||||
else if (auto * dictionary = ast->as<ASTDictionary>())
|
||||
{
|
||||
visitDictionaryDef(*dictionary);
|
||||
}
|
||||
else if (auto * expr = ast->as<ASTTableExpression>())
|
||||
{
|
||||
visitTableExpression(*expr);
|
||||
}
|
||||
else if (const auto * function = ast->as<ASTFunction>())
|
||||
{
|
||||
if (function->kind == ASTFunction::Kind::TABLE_ENGINE)
|
||||
visitTableEngine(*function);
|
||||
else
|
||||
visitFunction(*function);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
ASTPtr create_query;
|
||||
std::unordered_set<const IAST *> skip_asts;
|
||||
QualifiedTableName table_name;
|
||||
String current_database;
|
||||
ContextPtr context;
|
||||
TableNamesSet dependencies;
|
||||
|
||||
/// CREATE TABLE or CREATE DICTIONARY or CREATE VIEW or CREATE TEMPORARY TABLE or CREATE DATABASE query.
|
||||
void visitCreateQuery(const ASTCreateQuery & create)
|
||||
{
|
||||
QualifiedTableName to_table{create.to_table_id.database_name, create.to_table_id.table_name};
|
||||
if (!to_table.table.empty())
|
||||
{
|
||||
/// TO target_table (for materialized views)
|
||||
if (to_table.database.empty())
|
||||
to_table.database = current_database;
|
||||
dependencies.emplace(to_table);
|
||||
}
|
||||
|
||||
QualifiedTableName as_table{create.as_database, create.as_table};
|
||||
if (!as_table.table.empty())
|
||||
{
|
||||
/// AS table_name
|
||||
if (as_table.database.empty())
|
||||
as_table.database = current_database;
|
||||
dependencies.emplace(as_table);
|
||||
}
|
||||
}
|
||||
|
||||
/// The definition of a dictionary: SOURCE(CLICKHOUSE(...)) LAYOUT(...) LIFETIME(...)
|
||||
void visitDictionaryDef(const ASTDictionary & dictionary)
|
||||
{
|
||||
if (!dictionary.source || dictionary.source->name != "clickhouse" || !dictionary.source->elements)
|
||||
return;
|
||||
|
||||
auto maybe_qualified_name = QualifiedTableName::tryParseFromString(literal->value.get<String>());
|
||||
/// Just return if name if invalid
|
||||
if (!maybe_qualified_name)
|
||||
auto config = getDictionaryConfigurationFromAST(create_query->as<ASTCreateQuery &>(), context);
|
||||
auto info = getInfoIfClickHouseDictionarySource(config, context);
|
||||
|
||||
/// We consider only dependencies on local tables.
|
||||
if (!info || !info->is_local)
|
||||
return;
|
||||
|
||||
qualified_name = std::move(*maybe_qualified_name);
|
||||
if (info->table_name.database.empty())
|
||||
info->table_name.database = current_database;
|
||||
dependencies.emplace(std::move(info->table_name));
|
||||
}
|
||||
else if (const auto * identifier = dynamic_cast<const ASTIdentifier *>(arg))
|
||||
|
||||
/// ASTTableExpression represents a reference to a table in SELECT query.
|
||||
/// DDLDependencyVisitor should handle ASTTableExpression because some CREATE queries can contain SELECT queries after AS
|
||||
/// (for example, CREATE VIEW).
|
||||
void visitTableExpression(const ASTTableExpression & expr)
|
||||
{
|
||||
/// ASTIdentifier or ASTTableIdentifier
|
||||
if (!expr.database_and_table_name)
|
||||
return;
|
||||
|
||||
const ASTIdentifier * identifier = dynamic_cast<const ASTIdentifier *>(expr.database_and_table_name.get());
|
||||
if (!identifier)
|
||||
return;
|
||||
|
||||
auto table_identifier = identifier->createTable();
|
||||
/// Just return if table identified is invalid
|
||||
if (!table_identifier)
|
||||
return;
|
||||
|
||||
qualified_name.database = table_identifier->getDatabaseName();
|
||||
qualified_name.table = table_identifier->shortName();
|
||||
}
|
||||
else
|
||||
{
|
||||
/// Just return because we don't validate AST in this function.
|
||||
return;
|
||||
QualifiedTableName qualified_name{table_identifier->getDatabaseName(), table_identifier->shortName()};
|
||||
if (qualified_name.table.empty())
|
||||
return;
|
||||
|
||||
if (qualified_name.database.empty())
|
||||
{
|
||||
/// It can be table/dictionary from default database or XML dictionary, but we cannot distinguish it here.
|
||||
qualified_name.database = current_database;
|
||||
}
|
||||
|
||||
dependencies.emplace(qualified_name);
|
||||
}
|
||||
|
||||
if (qualified_name.database.empty())
|
||||
/// Finds dependencies of a table engine.
|
||||
void visitTableEngine(const ASTFunction & table_engine)
|
||||
{
|
||||
/// It can be table/dictionary from default database or XML dictionary, but we cannot distinguish it here.
|
||||
qualified_name.database = data.default_database;
|
||||
}
|
||||
data.dependencies.emplace(std::move(qualified_name));
|
||||
}
|
||||
/// Dictionary(db_name.dictionary_name)
|
||||
if (table_engine.name == "Dictionary")
|
||||
addQualifiedNameFromArgument(table_engine, 0);
|
||||
|
||||
/// Extracts a table name with database written in the form 'db_name', 'table_name' (two strings).
|
||||
void extractDatabaseAndTableNameFromArguments(const ASTFunction & function, DDLDependencyVisitor::Data & data, size_t database_arg_idx, size_t table_arg_idx)
|
||||
/// Buffer('db_name', 'dest_table_name')
|
||||
if (table_engine.name == "Buffer")
|
||||
addDatabaseAndTableNameFromArguments(table_engine, 0, 1);
|
||||
|
||||
/// Distributed(cluster_name, db_name, table_name, ...)
|
||||
if (table_engine.name == "Distributed")
|
||||
visitDistributedTableEngine(table_engine);
|
||||
}
|
||||
|
||||
/// Distributed(cluster_name, database_name, table_name, ...)
|
||||
void visitDistributedTableEngine(const ASTFunction & table_engine)
|
||||
{
|
||||
/// We consider only dependencies on local tables.
|
||||
bool has_local_replicas = false;
|
||||
|
||||
if (auto cluster_name = tryGetClusterNameFromArgument(table_engine, 0))
|
||||
{
|
||||
auto cluster = context->tryGetCluster(*cluster_name);
|
||||
if (cluster && cluster->getLocalShardCount())
|
||||
has_local_replicas = true;
|
||||
}
|
||||
|
||||
if (has_local_replicas)
|
||||
addDatabaseAndTableNameFromArguments(table_engine, 1, 2);
|
||||
}
|
||||
|
||||
/// Finds dependencies of a function.
|
||||
void visitFunction(const ASTFunction & function)
|
||||
{
|
||||
if (function.name == "joinGet" || function.name == "dictHas" || function.name == "dictIsIn" || function.name.starts_with("dictGet"))
|
||||
{
|
||||
/// dictGet('dict_name', attr_names, id_expr)
|
||||
/// dictHas('dict_name', id_expr)
|
||||
/// joinGet(join_storage_table_name, `value_column`, join_keys)
|
||||
addQualifiedNameFromArgument(function, 0);
|
||||
}
|
||||
else if (function.name == "in" || function.name == "notIn" || function.name == "globalIn" || function.name == "globalNotIn")
|
||||
{
|
||||
/// in(x, table_name) - function for evaluating (x IN table_name)
|
||||
addQualifiedNameFromArgument(function, 1);
|
||||
}
|
||||
else if (function.name == "dictionary")
|
||||
{
|
||||
/// dictionary(dict_name)
|
||||
addQualifiedNameFromArgument(function, 0);
|
||||
}
|
||||
else if (function.name == "remote" || function.name == "remoteSecure")
|
||||
{
|
||||
visitRemoteFunction(function, /* is_cluster_function= */ false);
|
||||
}
|
||||
else if (function.name == "cluster" || function.name == "clusterAllReplicas")
|
||||
{
|
||||
visitRemoteFunction(function, /* is_cluster_function= */ true);
|
||||
}
|
||||
}
|
||||
|
||||
/// remote('addresses_expr', db_name.table_name, ...)
|
||||
/// remote('addresses_expr', 'db_name', 'table_name', ...)
|
||||
/// remote('addresses_expr', table_function(), ...)
|
||||
/// cluster('cluster_name', db_name.table_name, ...)
|
||||
/// cluster('cluster_name', 'db_name', 'table_name', ...)
|
||||
/// cluster('cluster_name', table_function(), ...)
|
||||
void visitRemoteFunction(const ASTFunction & function, bool is_cluster_function)
|
||||
{
|
||||
/// We consider dependencies on local tables only.
|
||||
bool has_local_replicas = false;
|
||||
|
||||
if (is_cluster_function)
|
||||
{
|
||||
if (auto cluster_name = tryGetClusterNameFromArgument(function, 0))
|
||||
{
|
||||
if (auto cluster = context->tryGetCluster(*cluster_name))
|
||||
{
|
||||
if (cluster->getLocalShardCount())
|
||||
has_local_replicas = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
/// remote() and remoteSecure() are not fully supported. To properly support them we would need to check the first
|
||||
/// argument to decide whether the host & port pattern specified in the first argument contains the local host or not
|
||||
/// which is not trivial. For now we just always assume that the host & port pattern doesn't contain the local host.
|
||||
}
|
||||
|
||||
if (!function.arguments)
|
||||
return;
|
||||
|
||||
ASTs & args = function.arguments->children;
|
||||
if (args.size() < 2)
|
||||
return;
|
||||
|
||||
const ASTFunction * table_function = nullptr;
|
||||
if (const auto * second_arg_as_function = args[1]->as<ASTFunction>();
|
||||
second_arg_as_function && KnownTableFunctionNames::instance().exists(second_arg_as_function->name))
|
||||
{
|
||||
table_function = second_arg_as_function;
|
||||
}
|
||||
|
||||
if (has_local_replicas && !table_function)
|
||||
{
|
||||
auto maybe_qualified_name = tryGetQualifiedNameFromArgument(function, 1, /* apply_current_database= */ false);
|
||||
if (!maybe_qualified_name)
|
||||
return;
|
||||
auto & qualified_name = *maybe_qualified_name;
|
||||
if (qualified_name.database.empty())
|
||||
{
|
||||
auto table = tryGetStringFromArgument(function, 2);
|
||||
if (!table)
|
||||
return;
|
||||
qualified_name.database = std::move(qualified_name.table);
|
||||
qualified_name.table = std::move(table).value();
|
||||
}
|
||||
dependencies.insert(qualified_name);
|
||||
}
|
||||
|
||||
if (!has_local_replicas && table_function)
|
||||
{
|
||||
/// `table function` will be executed remotely, so we won't check it or its arguments for dependencies.
|
||||
skip_asts.emplace(table_function);
|
||||
}
|
||||
}
|
||||
|
||||
/// Gets an argument as a string, evaluates constants if necessary.
|
||||
std::optional<String> tryGetStringFromArgument(const ASTFunction & function, size_t arg_idx) const
|
||||
{
|
||||
if (!function.arguments)
|
||||
return {};
|
||||
|
||||
const ASTs & args = function.arguments->children;
|
||||
if (arg_idx >= args.size())
|
||||
return {};
|
||||
|
||||
const auto & arg = args[arg_idx];
|
||||
ASTPtr evaluated;
|
||||
|
||||
try
|
||||
{
|
||||
evaluated = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
const auto * literal = evaluated->as<ASTLiteral>();
|
||||
if (!literal || (literal->value.getType() != Field::Types::String))
|
||||
return {};
|
||||
return literal->value.safeGet<String>();
|
||||
}
|
||||
|
||||
/// Gets an argument as a qualified table name.
|
||||
/// Accepts forms db_name.table_name (as an identifier) and 'db_name.table_name' (as a string).
|
||||
/// The function doesn't replace an empty database name with the current_database (the caller must do that).
|
||||
std::optional<QualifiedTableName>
|
||||
tryGetQualifiedNameFromArgument(const ASTFunction & function, size_t arg_idx, bool apply_current_database = true) const
|
||||
{
|
||||
if (!function.arguments)
|
||||
return {};
|
||||
|
||||
const ASTs & args = function.arguments->children;
|
||||
if (arg_idx >= args.size())
|
||||
return {};
|
||||
|
||||
const auto & arg = args[arg_idx];
|
||||
QualifiedTableName qualified_name;
|
||||
|
||||
if (const auto * identifier = dynamic_cast<const ASTIdentifier *>(arg.get()))
|
||||
{
|
||||
/// ASTIdentifier or ASTTableIdentifier
|
||||
auto table_identifier = identifier->createTable();
|
||||
if (!table_identifier)
|
||||
return {};
|
||||
|
||||
qualified_name.database = table_identifier->getDatabaseName();
|
||||
qualified_name.table = table_identifier->shortName();
|
||||
}
|
||||
else
|
||||
{
|
||||
auto qualified_name_as_string = tryGetStringFromArgument(function, arg_idx);
|
||||
if (!qualified_name_as_string)
|
||||
return {};
|
||||
|
||||
auto maybe_qualified_name = QualifiedTableName::tryParseFromString(*qualified_name_as_string);
|
||||
if (!maybe_qualified_name)
|
||||
return {};
|
||||
|
||||
qualified_name = std::move(maybe_qualified_name).value();
|
||||
}
|
||||
|
||||
if (qualified_name.database.empty() && apply_current_database)
|
||||
qualified_name.database = current_database;
|
||||
|
||||
return qualified_name;
|
||||
}
|
||||
|
||||
/// Adds a qualified table name from an argument to the collection of dependencies.
|
||||
/// Accepts forms db_name.table_name (as an identifier) and 'db_name.table_name' (as a string).
|
||||
void addQualifiedNameFromArgument(const ASTFunction & function, size_t arg_idx)
|
||||
{
|
||||
if (auto qualified_name = tryGetQualifiedNameFromArgument(function, arg_idx))
|
||||
dependencies.emplace(std::move(qualified_name).value());
|
||||
}
|
||||
|
||||
/// Returns a database name and a table name extracted from two separate arguments.
|
||||
std::optional<QualifiedTableName> tryGetDatabaseAndTableNameFromArguments(
|
||||
const ASTFunction & function, size_t database_arg_idx, size_t table_arg_idx, bool apply_current_database = true) const
|
||||
{
|
||||
auto database = tryGetStringFromArgument(function, database_arg_idx);
|
||||
if (!database)
|
||||
return {};
|
||||
|
||||
auto table = tryGetStringFromArgument(function, table_arg_idx);
|
||||
if (!table)
|
||||
return {};
|
||||
|
||||
QualifiedTableName qualified_name;
|
||||
qualified_name.database = std::move(database).value();
|
||||
qualified_name.table = std::move(table).value();
|
||||
|
||||
if (qualified_name.database.empty() && apply_current_database)
|
||||
qualified_name.database = current_database;
|
||||
|
||||
return qualified_name;
|
||||
}
|
||||
|
||||
/// Adds a database name and a table name from two separate arguments to the collection of dependencies.
|
||||
void addDatabaseAndTableNameFromArguments(const ASTFunction & function, size_t database_arg_idx, size_t table_arg_idx)
|
||||
{
|
||||
if (auto qualified_name = tryGetDatabaseAndTableNameFromArguments(function, database_arg_idx, table_arg_idx))
|
||||
dependencies.emplace(std::move(qualified_name).value());
|
||||
}
|
||||
|
||||
std::optional<String> tryGetClusterNameFromArgument(const ASTFunction & function, size_t arg_idx) const
|
||||
{
|
||||
if (!function.arguments)
|
||||
return {};
|
||||
|
||||
ASTs & args = function.arguments->children;
|
||||
if (arg_idx >= args.size())
|
||||
return {};
|
||||
|
||||
auto cluster_name = ::DB::tryGetClusterName(*args[arg_idx]);
|
||||
if (cluster_name)
|
||||
return cluster_name;
|
||||
|
||||
return tryGetStringFromArgument(function, arg_idx);
|
||||
}
|
||||
};
|
||||
|
||||
/// Visits ASTCreateQuery and extracts the names of all tables explicitly referenced in the create query.
|
||||
class DDLDependencyVisitor
|
||||
{
|
||||
/// Just ignore incorrect arguments, proper exception will be thrown later
|
||||
if (!function.arguments || (function.arguments->children.size() <= database_arg_idx)
|
||||
|| (function.arguments->children.size() <= table_arg_idx))
|
||||
return;
|
||||
public:
|
||||
using Data = DDLDependencyVisitorData;
|
||||
using Visitor = ConstInDepthNodeVisitor<DDLDependencyVisitor, /* top_to_bottom= */ true, /* need_child_accept_data= */ true>;
|
||||
|
||||
const auto * expr_list = function.arguments->as<ASTExpressionList>();
|
||||
if (!expr_list)
|
||||
return;
|
||||
|
||||
const auto * database_literal = expr_list->children[database_arg_idx]->as<ASTLiteral>();
|
||||
const auto * table_name_literal = expr_list->children[table_arg_idx]->as<ASTLiteral>();
|
||||
|
||||
if (!database_literal || !table_name_literal || (database_literal->value.getType() != Field::Types::String)
|
||||
|| (table_name_literal->value.getType() != Field::Types::String))
|
||||
return;
|
||||
|
||||
QualifiedTableName qualified_name{database_literal->value.get<String>(), table_name_literal->value.get<String>()};
|
||||
if (qualified_name.table.empty())
|
||||
return;
|
||||
|
||||
if (qualified_name.database.empty())
|
||||
qualified_name.database = data.default_database;
|
||||
|
||||
data.dependencies.emplace(qualified_name);
|
||||
}
|
||||
|
||||
void visitFunction(const ASTFunction & function, DDLDependencyVisitor::Data & data)
|
||||
{
|
||||
if (function.name == "joinGet" || function.name == "dictHas" || function.name == "dictIsIn" || function.name.starts_with("dictGet"))
|
||||
{
|
||||
/// dictGet('dict_name', attr_names, id_expr)
|
||||
/// dictHas('dict_name', id_expr)
|
||||
/// joinGet(join_storage_table_name, `value_column`, join_keys)
|
||||
extractQualifiedTableNameFromArgument(function, data, 0);
|
||||
}
|
||||
else if (function.name == "in" || function.name == "notIn" || function.name == "globalIn" || function.name == "globalNotIn")
|
||||
{
|
||||
/// in(x, table_name) - function for evaluating (x IN table_name)
|
||||
extractQualifiedTableNameFromArgument(function, data, 1);
|
||||
}
|
||||
else if (function.name == "dictionary")
|
||||
{
|
||||
/// dictionary(dict_name)
|
||||
extractQualifiedTableNameFromArgument(function, data, 0);
|
||||
}
|
||||
}
|
||||
|
||||
void visitTableEngine(const ASTFunction & table_engine, DDLDependencyVisitor::Data & data)
|
||||
{
|
||||
if (table_engine.name == "Dictionary")
|
||||
extractQualifiedTableNameFromArgument(table_engine, data, 0);
|
||||
|
||||
if (table_engine.name == "Buffer")
|
||||
extractDatabaseAndTableNameFromArguments(table_engine, data, 0, 1);
|
||||
}
|
||||
|
||||
void visitDictionaryDef(const ASTDictionary & dictionary, DDLDependencyVisitor::Data & data)
|
||||
{
|
||||
if (!dictionary.source || dictionary.source->name != "clickhouse" || !dictionary.source->elements)
|
||||
return;
|
||||
|
||||
auto config = getDictionaryConfigurationFromAST(data.create_query->as<ASTCreateQuery &>(), data.global_context);
|
||||
auto info = getInfoIfClickHouseDictionarySource(config, data.global_context);
|
||||
|
||||
if (!info || !info->is_local)
|
||||
return;
|
||||
|
||||
if (info->table_name.database.empty())
|
||||
info->table_name.database = data.default_database;
|
||||
data.dependencies.emplace(std::move(info->table_name));
|
||||
}
|
||||
static bool needChildVisit(const ASTPtr &, const ASTPtr & child, const Data & data) { return data.needChildVisit(child); }
|
||||
static void visit(const ASTPtr & ast, Data & data) { data.visit(ast); }
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
TableNamesSet getDependenciesFromCreateQuery(const ContextPtr & global_context, const QualifiedTableName & table_name, const ASTPtr & ast)
|
||||
TableNamesSet getDependenciesFromCreateQuery(const ContextPtr & context, const QualifiedTableName & table_name, const ASTPtr & ast)
|
||||
{
|
||||
assert(global_context == global_context->getGlobalContext());
|
||||
DDLDependencyVisitor::Data data;
|
||||
data.table_name = table_name;
|
||||
data.default_database = global_context->getCurrentDatabase();
|
||||
data.create_query = ast;
|
||||
data.global_context = global_context;
|
||||
DDLDependencyVisitor::Data data{context, table_name, ast};
|
||||
DDLDependencyVisitor::Visitor visitor{data};
|
||||
visitor.visit(ast);
|
||||
data.dependencies.erase(data.table_name);
|
||||
return data.dependencies;
|
||||
}
|
||||
|
||||
void DDLDependencyVisitor::visit(const ASTPtr & ast, Data & data)
|
||||
{
|
||||
if (auto * create = ast->as<ASTCreateQuery>())
|
||||
{
|
||||
visitCreateQuery(*create, data);
|
||||
}
|
||||
else if (auto * dictionary = ast->as<ASTDictionary>())
|
||||
{
|
||||
visitDictionaryDef(*dictionary, data);
|
||||
}
|
||||
else if (auto * expr = ast->as<ASTTableExpression>())
|
||||
{
|
||||
visitTableExpression(*expr, data);
|
||||
}
|
||||
else if (const auto * function = ast->as<ASTFunction>())
|
||||
{
|
||||
if (function->kind == ASTFunction::Kind::TABLE_ENGINE)
|
||||
visitTableEngine(*function, data);
|
||||
else
|
||||
visitFunction(*function, data);
|
||||
}
|
||||
}
|
||||
|
||||
bool DDLDependencyVisitor::needChildVisit(const ASTPtr &, const ASTPtr &)
|
||||
{
|
||||
return true;
|
||||
return std::move(data).getDependencies();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,8 +1,9 @@
|
||||
#pragma once
|
||||
|
||||
#include <Parsers/IAST_fwd.h>
|
||||
#include <Interpreters/InDepthNodeVisitor.h>
|
||||
#include <Core/QualifiedTableName.h>
|
||||
#include <Interpreters/Context_fwd.h>
|
||||
#include <Parsers/IAST_fwd.h>
|
||||
#include <unordered_set>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -12,25 +13,6 @@ using TableNamesSet = std::unordered_set<QualifiedTableName>;
|
||||
/// Returns a list of all tables explicitly referenced in the create query of a specified table.
|
||||
/// For example, a column default expression can use dictGet() and thus reference a dictionary.
|
||||
/// Does not validate AST, works a best-effort way.
|
||||
TableNamesSet getDependenciesFromCreateQuery(const ContextPtr & global_context, const QualifiedTableName & table_name, const ASTPtr & ast);
|
||||
|
||||
/// Visits ASTCreateQuery and extracts the names of all tables explicitly referenced in the create query.
|
||||
class DDLDependencyVisitor
|
||||
{
|
||||
public:
|
||||
struct Data
|
||||
{
|
||||
ASTPtr create_query;
|
||||
QualifiedTableName table_name;
|
||||
String default_database;
|
||||
ContextPtr global_context;
|
||||
TableNamesSet dependencies;
|
||||
};
|
||||
|
||||
using Visitor = ConstInDepthNodeVisitor<DDLDependencyVisitor, /* top_to_bottom= */ true>;
|
||||
|
||||
static void visit(const ASTPtr & ast, Data & data);
|
||||
static bool needChildVisit(const ASTPtr & node, const ASTPtr & child);
|
||||
};
|
||||
TableNamesSet getDependenciesFromCreateQuery(const ContextPtr & context, const QualifiedTableName & table_name, const ASTPtr & ast);
|
||||
|
||||
}
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#include <Databases/TablesDependencyGraph.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <boost/range/adaptor/reversed.hpp>
|
||||
|
||||
|
||||
@ -9,12 +10,13 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int INFINITE_LOOP;
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
|
||||
namespace
|
||||
{
|
||||
constexpr const size_t CYCLIC_LEVEL = static_cast<size_t>(-2);
|
||||
constexpr const size_t CYCLIC_LEVEL = std::numeric_limits<size_t>::max();
|
||||
}
|
||||
|
||||
|
||||
@ -40,7 +42,7 @@ TablesDependencyGraph::TablesDependencyGraph(TablesDependencyGraph && src) noexc
|
||||
|
||||
TablesDependencyGraph & TablesDependencyGraph::operator=(const TablesDependencyGraph & src)
|
||||
{
|
||||
if (&src != this)
|
||||
if (this != &src)
|
||||
{
|
||||
nodes = src.nodes;
|
||||
nodes_by_database_and_table_names = src.nodes_by_database_and_table_names;
|
||||
@ -54,11 +56,14 @@ TablesDependencyGraph & TablesDependencyGraph::operator=(const TablesDependencyG
|
||||
|
||||
TablesDependencyGraph & TablesDependencyGraph::operator=(TablesDependencyGraph && src) noexcept
|
||||
{
|
||||
nodes = std::exchange(src.nodes, decltype(nodes){});
|
||||
nodes_by_database_and_table_names = std::exchange(src.nodes_by_database_and_table_names, decltype(nodes_by_database_and_table_names){});
|
||||
nodes_by_uuid = std::exchange(src.nodes_by_uuid, decltype(nodes_by_uuid){});
|
||||
levels_calculated = std::exchange(src.levels_calculated, false);
|
||||
nodes_sorted_by_level_lazy = std::exchange(src.nodes_sorted_by_level_lazy, decltype(nodes_sorted_by_level_lazy){});
|
||||
if (this != &src)
|
||||
{
|
||||
nodes = std::exchange(src.nodes, decltype(nodes){});
|
||||
nodes_by_database_and_table_names = std::exchange(src.nodes_by_database_and_table_names, decltype(nodes_by_database_and_table_names){});
|
||||
nodes_by_uuid = std::exchange(src.nodes_by_uuid, decltype(nodes_by_uuid){});
|
||||
levels_calculated = std::exchange(src.levels_calculated, false);
|
||||
nodes_sorted_by_level_lazy = std::exchange(src.nodes_sorted_by_level_lazy, decltype(nodes_sorted_by_level_lazy){});
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
@ -89,11 +94,13 @@ void TablesDependencyGraph::addDependency(const StorageID & table_id, const Stor
|
||||
auto * table_node = addOrUpdateNode(table_id);
|
||||
auto * dependency_node = addOrUpdateNode(dependency);
|
||||
|
||||
if (table_node->dependencies.contains(dependency_node))
|
||||
return; /// Already have this dependency.
|
||||
bool inserted = table_node->dependencies.insert(dependency_node).second;
|
||||
if (!inserted)
|
||||
return; /// Not inserted because we already had this dependency.
|
||||
|
||||
table_node->dependencies.insert(dependency_node);
|
||||
dependency_node->dependents.insert(table_node);
|
||||
/// `dependency_node` must be updated too.
|
||||
[[maybe_unused]] bool inserted_to_set = dependency_node->dependents.insert(table_node).second;
|
||||
chassert(inserted_to_set);
|
||||
|
||||
setNeedRecalculateLevels();
|
||||
}
|
||||
@ -126,13 +133,19 @@ void TablesDependencyGraph::addDependencies(const StorageID & table_id, const st
|
||||
for (auto * dependency_node : old_dependency_nodes)
|
||||
{
|
||||
if (!new_dependency_nodes.contains(dependency_node))
|
||||
dependency_node->dependents.erase(table_node);
|
||||
{
|
||||
[[maybe_unused]] bool removed_from_set = dependency_node->dependents.erase(table_node);
|
||||
chassert(removed_from_set);
|
||||
}
|
||||
}
|
||||
|
||||
for (auto * dependency_node : new_dependency_nodes)
|
||||
{
|
||||
if (!old_dependency_nodes.contains(dependency_node))
|
||||
dependency_node->dependents.insert(table_node);
|
||||
{
|
||||
[[maybe_unused]] bool inserted_to_set = dependency_node->dependents.insert(table_node).second;
|
||||
chassert(inserted_to_set);
|
||||
}
|
||||
}
|
||||
|
||||
table_node->dependencies = std::move(new_dependency_nodes);
|
||||
@ -167,21 +180,28 @@ bool TablesDependencyGraph::removeDependency(const StorageID & table_id, const S
|
||||
|
||||
auto dependency_it = table_node->dependencies.find(dependency_node);
|
||||
if (dependency_it == table_node->dependencies.end())
|
||||
return false;
|
||||
return false; /// No such dependency, nothing to remove.
|
||||
|
||||
table_node->dependencies.erase(dependency_it);
|
||||
dependency_node->dependents.erase(table_node);
|
||||
bool table_node_removed = false;
|
||||
|
||||
/// `dependency_node` must be updated too.
|
||||
[[maybe_unused]] bool removed_from_set = dependency_node->dependents.erase(table_node);
|
||||
chassert(removed_from_set);
|
||||
|
||||
if (remove_isolated_tables && dependency_node->dependencies.empty() && dependency_node->dependents.empty())
|
||||
{
|
||||
/// The dependency table has no dependencies and no dependents now, so we will remove it from the graph.
|
||||
removeNode(dependency_node);
|
||||
if (table_node == dependency_node)
|
||||
table_node_removed = true;
|
||||
}
|
||||
|
||||
if (remove_isolated_tables && !table_node_removed && table_node->dependencies.empty() && table_node->dependents.empty())
|
||||
{
|
||||
/// The table `table_id` has no dependencies and no dependents now, so we will remove it from the graph.
|
||||
removeNode(table_node);
|
||||
}
|
||||
|
||||
setNeedRecalculateLevels();
|
||||
return true;
|
||||
@ -203,19 +223,28 @@ std::vector<StorageID> TablesDependencyGraph::removeDependencies(const StorageID
|
||||
|
||||
for (auto * dependency_node : dependency_nodes)
|
||||
{
|
||||
/// We're gathering the list of dependencies the table `table_id` had in the graph to return from the function.
|
||||
dependencies.emplace_back(dependency_node->storage_id);
|
||||
dependency_node->dependents.erase(table_node);
|
||||
|
||||
/// Update `dependency_node`.
|
||||
[[maybe_unused]] bool removed_from_set = dependency_node->dependents.erase(table_node);
|
||||
chassert(removed_from_set);
|
||||
|
||||
if (remove_isolated_tables && dependency_node->dependencies.empty() && dependency_node->dependents.empty())
|
||||
{
|
||||
/// The dependency table has no dependencies and no dependents now, so we will remove it from the graph.
|
||||
removeNode(dependency_node);
|
||||
if (table_node == dependency_node)
|
||||
table_node_removed = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (remove_isolated_tables && !table_node_removed && table_node->dependencies.empty() && table_node->dependents.empty())
|
||||
chassert(table_node->dependencies.empty());
|
||||
if (remove_isolated_tables && !table_node_removed && table_node->dependents.empty())
|
||||
{
|
||||
/// The table `table_id` has no dependencies and no dependents now, so we will remove it from the graph.
|
||||
removeNode(table_node);
|
||||
}
|
||||
|
||||
setNeedRecalculateLevels();
|
||||
return dependencies;
|
||||
@ -251,7 +280,12 @@ TablesDependencyGraph::Node * TablesDependencyGraph::findNode(const StorageID &
|
||||
{
|
||||
auto * node = it->second;
|
||||
if (table_id.hasUUID() && node->storage_id.hasUUID() && (table_id.uuid != node->storage_id.uuid))
|
||||
return nullptr; /// UUID is different, it's not the node we're looking for.
|
||||
{
|
||||
/// We found a table with specified database and table names in the graph, but surprisingly it has a different UUID.
|
||||
/// Maybe an "EXCHANGE TABLES" command has been executed somehow without changing the graph?
|
||||
LOG_WARNING(getLogger(), "Found table {} in the graph with unexpected UUID {}", table_id, node->storage_id.uuid);
|
||||
return nullptr; /// Act like it's not found.
|
||||
}
|
||||
return node; /// Found by table name.
|
||||
}
|
||||
}
|
||||
@ -268,7 +302,8 @@ TablesDependencyGraph::Node * TablesDependencyGraph::addOrUpdateNode(const Stora
|
||||
if (table_id.hasUUID() && !node->storage_id.hasUUID())
|
||||
{
|
||||
node->storage_id.uuid = table_id.uuid;
|
||||
nodes_by_uuid.emplace(node->storage_id.uuid, node);
|
||||
[[maybe_unused]] bool inserted_to_map = nodes_by_uuid.emplace(node->storage_id.uuid, node).second;
|
||||
chassert(inserted_to_map);
|
||||
}
|
||||
|
||||
if (!table_id.table_name.empty() && ((table_id.table_name != node->storage_id.table_name) || (table_id.database_name != node->storage_id.database_name)))
|
||||
@ -283,7 +318,8 @@ TablesDependencyGraph::Node * TablesDependencyGraph::addOrUpdateNode(const Stora
|
||||
nodes_by_database_and_table_names.erase(node->storage_id);
|
||||
node->storage_id.database_name = table_id.database_name;
|
||||
node->storage_id.table_name = table_id.table_name;
|
||||
nodes_by_database_and_table_names.emplace(node->storage_id, node);
|
||||
[[maybe_unused]] bool inserted_to_map = nodes_by_database_and_table_names.emplace(node->storage_id, node).second;
|
||||
chassert(inserted_to_map);
|
||||
}
|
||||
}
|
||||
else
|
||||
@ -303,9 +339,15 @@ TablesDependencyGraph::Node * TablesDependencyGraph::addOrUpdateNode(const Stora
|
||||
nodes.insert(node_ptr);
|
||||
node = node_ptr.get();
|
||||
if (table_id.hasUUID())
|
||||
nodes_by_uuid.emplace(table_id.uuid, node);
|
||||
{
|
||||
[[maybe_unused]] bool inserted_to_map = nodes_by_uuid.emplace(table_id.uuid, node).second;
|
||||
chassert(inserted_to_map);
|
||||
}
|
||||
if (!table_id.table_name.empty())
|
||||
nodes_by_database_and_table_names.emplace(table_id, node);
|
||||
{
|
||||
[[maybe_unused]] bool inserted_to_map = nodes_by_database_and_table_names.emplace(table_id, node).second;
|
||||
chassert(inserted_to_map);
|
||||
}
|
||||
}
|
||||
return node;
|
||||
}
|
||||
@ -313,22 +355,39 @@ TablesDependencyGraph::Node * TablesDependencyGraph::addOrUpdateNode(const Stora
|
||||
|
||||
void TablesDependencyGraph::removeNode(Node * node)
|
||||
{
|
||||
chassert(node);
|
||||
auto dependency_nodes = std::move(node->dependencies);
|
||||
auto dependent_nodes = std::move(node->dependents);
|
||||
|
||||
if (node->storage_id.hasUUID())
|
||||
nodes_by_uuid.erase(node->storage_id.uuid);
|
||||
{
|
||||
[[maybe_unused]] bool removed_from_map = nodes_by_uuid.erase(node->storage_id.uuid);
|
||||
chassert(removed_from_map);
|
||||
}
|
||||
|
||||
if (!node->storage_id.table_name.empty())
|
||||
nodes_by_database_and_table_names.erase(node->storage_id);
|
||||
{
|
||||
[[maybe_unused]]bool removed_from_map = nodes_by_database_and_table_names.erase(node->storage_id);
|
||||
chassert(removed_from_map);
|
||||
}
|
||||
|
||||
for (auto * dependency_node : dependency_nodes)
|
||||
dependency_node->dependents.erase(node);
|
||||
{
|
||||
[[maybe_unused]] bool removed_from_set = dependency_node->dependents.erase(node);
|
||||
chassert(removed_from_set);
|
||||
}
|
||||
|
||||
for (auto * dependent_node : dependent_nodes)
|
||||
dependent_node->dependencies.erase(node);
|
||||
{
|
||||
[[maybe_unused]] bool removed_from_set = dependent_node->dependencies.erase(node);
|
||||
chassert(removed_from_set);
|
||||
}
|
||||
|
||||
nodes.erase(node->shared_from_this());
|
||||
auto it = nodes.find(node);
|
||||
chassert(it != nodes.end());
|
||||
nodes.erase(it);
|
||||
|
||||
nodes_sorted_by_level_lazy.clear();
|
||||
}
|
||||
|
||||
|
||||
@ -533,7 +592,7 @@ String TablesDependencyGraph::describeCyclicDependencies() const
|
||||
}
|
||||
|
||||
|
||||
void TablesDependencyGraph::setNeedRecalculateLevels()
|
||||
void TablesDependencyGraph::setNeedRecalculateLevels() const
|
||||
{
|
||||
levels_calculated = false;
|
||||
nodes_sorted_by_level_lazy.clear();
|
||||
@ -546,49 +605,73 @@ void TablesDependencyGraph::calculateLevels() const
|
||||
return;
|
||||
levels_calculated = true;
|
||||
|
||||
/// First find tables with no dependencies, add them to `nodes_sorted_by_level_lazy`.
|
||||
/// Then remove those tables from the dependency graph (we imitate that removing by decrementing `num_dependencies_to_count`),
|
||||
/// and find which tables have no dependencies now.
|
||||
/// Repeat until we have tables with no dependencies.
|
||||
/// In the end we expect all nodes from `nodes` to be added to `nodes_sorted_by_level_lazy`.
|
||||
/// If some nodes are still not added to `nodes_sorted_by_level_lazy` in the end then there is a cyclic dependency.
|
||||
/// Complexity: O(V + E)
|
||||
|
||||
nodes_sorted_by_level_lazy.clear();
|
||||
nodes_sorted_by_level_lazy.reserve(nodes.size());
|
||||
|
||||
std::unordered_set<const Node *> nodes_to_process;
|
||||
for (const auto & node_ptr : nodes)
|
||||
nodes_to_process.emplace(node_ptr.get());
|
||||
|
||||
size_t current_level = 0;
|
||||
|
||||
while (!nodes_to_process.empty())
|
||||
/// Find tables with no dependencies.
|
||||
for (const auto & node_ptr : nodes)
|
||||
{
|
||||
size_t old_num_sorted = nodes_sorted_by_level_lazy.size();
|
||||
|
||||
for (auto it = nodes_to_process.begin(); it != nodes_to_process.end();)
|
||||
const Node * node = node_ptr.get();
|
||||
node->num_dependencies_to_count = node->dependencies.size();
|
||||
if (!node->num_dependencies_to_count)
|
||||
{
|
||||
const auto * current_node = *(it++);
|
||||
bool has_dependencies = false;
|
||||
for (const auto * dependency : current_node->dependencies)
|
||||
{
|
||||
if (nodes_to_process.contains(dependency))
|
||||
has_dependencies = true;
|
||||
}
|
||||
node->level = current_level;
|
||||
nodes_sorted_by_level_lazy.emplace_back(node);
|
||||
}
|
||||
}
|
||||
|
||||
if (!has_dependencies)
|
||||
size_t num_nodes_without_dependencies = nodes_sorted_by_level_lazy.size();
|
||||
++current_level;
|
||||
|
||||
while (num_nodes_without_dependencies)
|
||||
{
|
||||
size_t begin = nodes_sorted_by_level_lazy.size() - num_nodes_without_dependencies;
|
||||
size_t end = nodes_sorted_by_level_lazy.size();
|
||||
|
||||
/// Decrement number of dependencies for each dependent table.
|
||||
for (size_t i = begin; i != end; ++i)
|
||||
{
|
||||
const Node * current_node = nodes_sorted_by_level_lazy[i];
|
||||
for (const Node * dependent_node : current_node->dependents)
|
||||
{
|
||||
current_node->level = current_level;
|
||||
nodes_sorted_by_level_lazy.emplace_back(current_node);
|
||||
if (!dependent_node->num_dependencies_to_count)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "{}: Trying to decrement 0 dependencies counter for {}. It's a bug", name_for_logging, dependent_node->storage_id);
|
||||
|
||||
if (!--dependent_node->num_dependencies_to_count)
|
||||
{
|
||||
dependent_node->level = current_level;
|
||||
nodes_sorted_by_level_lazy.emplace_back(dependent_node);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (nodes_sorted_by_level_lazy.size() == old_num_sorted)
|
||||
break;
|
||||
|
||||
for (size_t i = old_num_sorted; i != nodes_sorted_by_level_lazy.size(); ++i)
|
||||
nodes_to_process.erase(nodes_sorted_by_level_lazy[i]);
|
||||
if (nodes_sorted_by_level_lazy.size() > nodes.size())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "{}: Some tables were found more than once while passing through the dependency graph. It's a bug", name_for_logging);
|
||||
|
||||
num_nodes_without_dependencies = nodes_sorted_by_level_lazy.size() - end;
|
||||
++current_level;
|
||||
}
|
||||
|
||||
for (const auto * node_with_cyclic_dependencies : nodes_to_process)
|
||||
if (nodes_sorted_by_level_lazy.size() < nodes.size())
|
||||
{
|
||||
node_with_cyclic_dependencies->level = CYCLIC_LEVEL;
|
||||
nodes_sorted_by_level_lazy.emplace_back(node_with_cyclic_dependencies);
|
||||
for (const auto & node_ptr : nodes)
|
||||
{
|
||||
const Node * node = node_ptr.get();
|
||||
if (node->num_dependencies_to_count)
|
||||
{
|
||||
node->level = CYCLIC_LEVEL;
|
||||
nodes_sorted_by_level_lazy.emplace_back(node);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -630,7 +713,7 @@ std::vector<std::vector<StorageID>> TablesDependencyGraph::getTablesSortedByDepe
|
||||
|
||||
void TablesDependencyGraph::log() const
|
||||
{
|
||||
if (empty())
|
||||
if (nodes.empty())
|
||||
{
|
||||
LOG_TEST(getLogger(), "No tables");
|
||||
return;
|
||||
|
||||
@ -20,11 +20,11 @@ using TableNamesSet = std::unordered_set<QualifiedTableName>;
|
||||
///
|
||||
/// This class is used to represent various types of table-table dependencies:
|
||||
/// 1. View dependencies: "source_table -> materialized_view".
|
||||
/// Data inserted to a source table is also inserted to corresponding materialized views.
|
||||
/// Data inserted to a source table is also inserted to corresponding materialized views.
|
||||
/// 2. Loading dependencies: specify in which order tables must be loaded during startup.
|
||||
/// For example a dictionary should be loaded after it's source table and it's written in the graph as "dictionary -> source_table".
|
||||
/// For example a dictionary should be loaded after it's source table and it's written in the graph as "dictionary -> source_table".
|
||||
/// 3. Referential dependencies: "table -> all tables mentioned in its definition".
|
||||
/// Referential dependencies are checked to decide if it's safe to drop a table (it can be unsafe if the table is used by another table).
|
||||
/// Referential dependencies are checked to decide if it's safe to drop a table (it can be unsafe if the table is used by another table).
|
||||
///
|
||||
/// WARNING: This class doesn't have an embedded mutex, so it must be synchronized outside.
|
||||
class TablesDependencyGraph
|
||||
@ -98,8 +98,8 @@ public:
|
||||
/// Cyclic dependencies are dependencies like "A->A" or "A->B->C->D->A".
|
||||
void checkNoCyclicDependencies() const;
|
||||
bool hasCyclicDependencies() const;
|
||||
std::vector<StorageID> getTablesWithCyclicDependencies() const;
|
||||
String describeCyclicDependencies() const;
|
||||
std::vector<StorageID> getTablesWithCyclicDependencies() const;
|
||||
|
||||
/// Returns a list of tables sorted by their dependencies:
|
||||
/// tables without dependencies first, then
|
||||
@ -113,8 +113,12 @@ public:
|
||||
/// Outputs information about this graph as a bunch of logging messages.
|
||||
void log() const;
|
||||
|
||||
/// Calculates levels - this is required for checking cyclic dependencies, to sort tables by dependency, and to log the graph.
|
||||
/// This function is called automatically by the functions which need it, but can be invoked directly.
|
||||
void calculateLevels() const;
|
||||
|
||||
private:
|
||||
struct Node : public std::enable_shared_from_this<Node>
|
||||
struct Node
|
||||
{
|
||||
StorageID storage_id;
|
||||
|
||||
@ -128,28 +132,38 @@ private:
|
||||
/// Calculated lazily.
|
||||
mutable size_t level = 0;
|
||||
|
||||
/// Number of dependencies left, used only while we're calculating levels.
|
||||
mutable size_t num_dependencies_to_count = 0;
|
||||
|
||||
explicit Node(const StorageID & storage_id_) : storage_id(storage_id_) {}
|
||||
};
|
||||
|
||||
using NodeSharedPtr = std::shared_ptr<Node>;
|
||||
|
||||
struct LessByLevel
|
||||
struct Hash
|
||||
{
|
||||
bool operator()(const Node * left, const Node * right) { return left->level < right->level; }
|
||||
using is_transparent = void;
|
||||
size_t operator()(const Node * node) const { return std::hash<const Node *>{}(node); }
|
||||
size_t operator()(const NodeSharedPtr & node_ptr) const { return operator()(node_ptr.get()); }
|
||||
};
|
||||
|
||||
std::unordered_set<NodeSharedPtr> nodes;
|
||||
struct Equal
|
||||
{
|
||||
using is_transparent = void;
|
||||
size_t operator()(const NodeSharedPtr & left, const Node * right) const { return left.get() == right; }
|
||||
size_t operator()(const NodeSharedPtr & left, const NodeSharedPtr & right) const { return left == right; }
|
||||
};
|
||||
|
||||
std::unordered_set<NodeSharedPtr, Hash, Equal> nodes;
|
||||
|
||||
/// Nodes can be found either by UUID or by database name & table name. That's why we need two maps here.
|
||||
std::unordered_map<StorageID, Node *, StorageID::DatabaseAndTableNameHash, StorageID::DatabaseAndTableNameEqual> nodes_by_database_and_table_names;
|
||||
std::unordered_map<UUID, Node *> nodes_by_uuid;
|
||||
|
||||
/// This is set if both `level` inside each node and `nodes_sorted_by_level_lazy` are calculated.
|
||||
mutable bool levels_calculated = false;
|
||||
|
||||
/// Nodes sorted by their level. Calculated lazily.
|
||||
using NodesSortedByLevel = std::vector<const Node *>;
|
||||
mutable NodesSortedByLevel nodes_sorted_by_level_lazy;
|
||||
mutable bool levels_calculated = false;
|
||||
|
||||
const String name_for_logging;
|
||||
mutable Poco::Logger * logger = nullptr;
|
||||
@ -161,8 +175,7 @@ private:
|
||||
static std::vector<StorageID> getDependencies(const Node & node);
|
||||
static std::vector<StorageID> getDependents(const Node & node);
|
||||
|
||||
void setNeedRecalculateLevels();
|
||||
void calculateLevels() const;
|
||||
void setNeedRecalculateLevels() const;
|
||||
const NodesSortedByLevel & getNodesSortedByLevel() const;
|
||||
|
||||
Poco::Logger * getLogger() const;
|
||||
|
||||
@ -624,20 +624,21 @@ getInfoIfClickHouseDictionarySource(DictionaryConfigurationPtr & config, Context
|
||||
{
|
||||
ClickHouseDictionarySourceInfo info;
|
||||
|
||||
String host = config->getString("dictionary.source.clickhouse.host", "");
|
||||
UInt16 port = config->getUInt("dictionary.source.clickhouse.port", 0);
|
||||
bool secure = config->getBool("dictionary.source.clickhouse.secure", false);
|
||||
UInt16 default_port = secure ? global_context->getTCPPortSecure().value_or(0) : global_context->getTCPPort();
|
||||
|
||||
String host = config->getString("dictionary.source.clickhouse.host", "localhost");
|
||||
UInt16 port = config->getUInt("dictionary.source.clickhouse.port", default_port);
|
||||
String database = config->getString("dictionary.source.clickhouse.db", "");
|
||||
String table = config->getString("dictionary.source.clickhouse.table", "");
|
||||
bool secure = config->getBool("dictionary.source.clickhouse.secure", false);
|
||||
|
||||
if (host.empty() || port == 0 || table.empty())
|
||||
if (table.empty())
|
||||
return {};
|
||||
|
||||
info.table_name = {database, table};
|
||||
|
||||
try
|
||||
{
|
||||
UInt16 default_port = secure ? global_context->getTCPPortSecure().value_or(0) : global_context->getTCPPort();
|
||||
if (isLocalAddress({host, port}, default_port))
|
||||
info.is_local = true;
|
||||
}
|
||||
|
||||
@ -106,7 +106,7 @@ public:
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
if (arguments.size() < 1 || arguments.size() > 2)
|
||||
if (arguments.empty() || arguments.size() > 2)
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} takes one or two arguments", name);
|
||||
|
||||
if (!isInteger(arguments[0].type))
|
||||
@ -126,7 +126,7 @@ public:
|
||||
const auto & col = *src.column;
|
||||
|
||||
if (!checkAndGetColumn<ColumnVector<T>>(col))
|
||||
return 0;
|
||||
return false;
|
||||
|
||||
auto & result_data = result_column->getData();
|
||||
|
||||
@ -135,7 +135,7 @@ public:
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
result_data[i] = source_data[i];
|
||||
|
||||
return 1;
|
||||
return true;
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
|
||||
@ -225,7 +225,7 @@ HashJoin::HashJoin(std::shared_ptr<TableJoin> table_join_, const Block & right_s
|
||||
, right_sample_block(right_sample_block_)
|
||||
, log(&Poco::Logger::get("HashJoin"))
|
||||
{
|
||||
LOG_DEBUG(log, "Datatype: {}, kind: {}, strictness: {}", data->type, kind, strictness);
|
||||
LOG_DEBUG(log, "Datatype: {}, kind: {}, strictness: {}, right header: {}", data->type, kind, strictness, right_sample_block.dumpStructure());
|
||||
LOG_DEBUG(log, "Keys: {}", TableJoin::formatClauses(table_join->getClauses(), true));
|
||||
|
||||
if (isCrossOrComma(kind))
|
||||
|
||||
@ -672,6 +672,11 @@ String TableJoin::renamedRightColumnName(const String & name) const
|
||||
return name;
|
||||
}
|
||||
|
||||
void TableJoin::setRename(const String & from, const String & to)
|
||||
{
|
||||
renames[from] = to;
|
||||
}
|
||||
|
||||
void TableJoin::addKey(const String & left_name, const String & right_name, const ASTPtr & left_ast, const ASTPtr & right_ast)
|
||||
{
|
||||
clauses.back().key_names_left.emplace_back(left_name);
|
||||
|
||||
@ -334,6 +334,7 @@ public:
|
||||
Block getRequiredRightKeys(const Block & right_table_keys, std::vector<String> & keys_sources) const;
|
||||
|
||||
String renamedRightColumnName(const String & name) const;
|
||||
void setRename(const String & from, const String & to);
|
||||
|
||||
void resetKeys();
|
||||
void resetToCross();
|
||||
|
||||
@ -18,18 +18,31 @@ namespace ErrorCodes
|
||||
|
||||
|
||||
std::string getClusterName(const IAST & node)
|
||||
{
|
||||
auto name = tryGetClusterName(node);
|
||||
if (!name)
|
||||
throw Exception("Illegal expression instead of cluster name.", ErrorCodes::BAD_ARGUMENTS);
|
||||
return std::move(name).value();
|
||||
}
|
||||
|
||||
|
||||
std::optional<std::string> tryGetClusterName(const IAST & node)
|
||||
{
|
||||
if (const auto * ast_id = node.as<ASTIdentifier>())
|
||||
return ast_id->name();
|
||||
|
||||
if (const auto * ast_lit = node.as<ASTLiteral>())
|
||||
return checkAndGetLiteralArgument<String>(*ast_lit, "cluster_name");
|
||||
{
|
||||
if (ast_lit->value.getType() != Field::Types::String)
|
||||
return {};
|
||||
return ast_lit->value.safeGet<String>();
|
||||
}
|
||||
|
||||
/// A hack to support hyphens in cluster names.
|
||||
if (const auto * ast_func = node.as<ASTFunction>())
|
||||
{
|
||||
if (ast_func->name != "minus" || !ast_func->arguments || ast_func->arguments->children.size() < 2)
|
||||
throw Exception("Illegal expression instead of cluster name.", ErrorCodes::BAD_ARGUMENTS);
|
||||
return {};
|
||||
|
||||
String name;
|
||||
for (const auto & arg : ast_func->arguments->children)
|
||||
@ -43,7 +56,7 @@ std::string getClusterName(const IAST & node)
|
||||
return name;
|
||||
}
|
||||
|
||||
throw Exception("Illegal expression instead of cluster name.", ErrorCodes::BAD_ARGUMENTS);
|
||||
return {};
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -15,6 +15,7 @@ namespace DB
|
||||
* Therefore, consider this case separately.
|
||||
*/
|
||||
std::string getClusterName(const IAST & node);
|
||||
std::optional<std::string> tryGetClusterName(const IAST & node);
|
||||
|
||||
std::string getClusterNameAndMakeLiteral(ASTPtr & node);
|
||||
|
||||
|
||||
@ -45,8 +45,9 @@ namespace DB
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int LOGICAL_ERROR;
|
||||
extern const int INCOMPATIBLE_TYPE_OF_JOIN;
|
||||
extern const int INVALID_JOIN_ON_EXPRESSION;
|
||||
extern const int LOGICAL_ERROR;
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
@ -671,9 +672,23 @@ std::shared_ptr<IJoin> chooseJoinAlgorithm(std::shared_ptr<TableJoin> & table_jo
|
||||
{
|
||||
trySetStorageInTableJoin(right_table_expression, table_join);
|
||||
|
||||
auto & right_table_expression_data = planner_context->getTableExpressionDataOrThrow(right_table_expression);
|
||||
|
||||
/// JOIN with JOIN engine.
|
||||
if (auto storage = table_join->getStorageJoin())
|
||||
{
|
||||
for (const auto & result_column : right_table_expression_header)
|
||||
{
|
||||
const auto * source_column_name = right_table_expression_data.getColumnNameOrNull(result_column.name);
|
||||
if (!source_column_name)
|
||||
throw Exception(ErrorCodes::INCOMPATIBLE_TYPE_OF_JOIN,
|
||||
"JOIN with 'Join' table engine should be performed by storage keys [{}], but column '{}' was found",
|
||||
fmt::join(storage->getKeyNames(), ", "), result_column.name);
|
||||
|
||||
table_join->setRename(*source_column_name, result_column.name);
|
||||
}
|
||||
return storage->getJoinLocked(table_join, planner_context->getQueryContext());
|
||||
}
|
||||
|
||||
/** JOIN with constant.
|
||||
* Example: SELECT * FROM test_table AS t1 INNER JOIN test_table AS t2 ON 1;
|
||||
|
||||
@ -172,7 +172,7 @@ ColumnWithTypeAndName RPNBuilderTreeNode::getConstantColumn() const
|
||||
|
||||
if (ast_node)
|
||||
{
|
||||
const auto * literal = assert_cast<const ASTLiteral *>(ast_node);
|
||||
const auto * literal = typeid_cast<const ASTLiteral *>(ast_node);
|
||||
if (literal)
|
||||
{
|
||||
result.type = applyVisitor(FieldToDataType(), literal->value);
|
||||
|
||||
@ -386,8 +386,13 @@ void ReplicatedMergeTreeRestartingThread::setReadonly(bool on_shutdown)
|
||||
CurrentMetrics::add(CurrentMetrics::ReadonlyReplica);
|
||||
|
||||
/// Replica was already readonly, but we should decrement the metric, because we are detaching/dropping table.
|
||||
if (on_shutdown)
|
||||
/// if first pass wasn't done we don't have to decrement because it wasn't incremented in the first place
|
||||
/// the task should be deactivated if it's full shutdown so no race is present
|
||||
if (!first_time && on_shutdown)
|
||||
{
|
||||
CurrentMetrics::sub(CurrentMetrics::ReadonlyReplica);
|
||||
assert(CurrentMetrics::get(CurrentMetrics::ReadonlyReplica) >= 0);
|
||||
}
|
||||
}
|
||||
|
||||
void ReplicatedMergeTreeRestartingThread::setNotReadonly()
|
||||
@ -397,7 +402,10 @@ void ReplicatedMergeTreeRestartingThread::setNotReadonly()
|
||||
/// because we don't want to change this metric if replication is started successfully.
|
||||
/// So we should not decrement it when replica stopped being readonly on startup.
|
||||
if (storage.is_readonly.compare_exchange_strong(old_val, false) && !first_time)
|
||||
{
|
||||
CurrentMetrics::sub(CurrentMetrics::ReadonlyReplica);
|
||||
assert(CurrentMetrics::get(CurrentMetrics::ReadonlyReplica) >= 0);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -174,6 +174,9 @@ HashJoinPtr StorageJoin::getJoinLocked(std::shared_ptr<TableJoin> analyzed_join,
|
||||
"Table {} needs the same join_use_nulls setting as present in LEFT or FULL JOIN",
|
||||
getStorageID().getNameForLogs());
|
||||
|
||||
if (analyzed_join->getClauses().size() != 1)
|
||||
throw Exception(ErrorCodes::INCOMPATIBLE_TYPE_OF_JOIN, "JOIN keys should match to the Join engine keys [{}]", fmt::join(getKeyNames(), ", "));
|
||||
|
||||
const auto & join_on = analyzed_join->getOnlyClause();
|
||||
if (join_on.on_filter_condition_left || join_on.on_filter_condition_right)
|
||||
throw Exception(ErrorCodes::INCOMPATIBLE_TYPE_OF_JOIN, "ON section of JOIN with filter conditions is not implemented");
|
||||
@ -211,9 +214,9 @@ HashJoinPtr StorageJoin::getJoinLocked(std::shared_ptr<TableJoin> analyzed_join,
|
||||
left_key_names_resorted.push_back(key_names_left[key_position]);
|
||||
}
|
||||
|
||||
/// Set names qualifiers: table.column -> column
|
||||
/// It's required because storage join stores non-qualified names
|
||||
/// Qualifies will be added by join implementation (HashJoin)
|
||||
/// Set qualified identifiers to original names (table.column -> column).
|
||||
/// It's required because storage join stores non-qualified names.
|
||||
/// Qualifies will be added by join implementation (TableJoin contains a rename mapping).
|
||||
analyzed_join->setRightKeys(key_names);
|
||||
analyzed_join->setLeftKeys(left_key_names_resorted);
|
||||
|
||||
|
||||
@ -633,6 +633,8 @@ void StorageReplicatedMergeTree::createNewZooKeeperNodes()
|
||||
futures.push_back(zookeeper->asyncTryCreateNoThrow(zookeeper_path + "/pinned_part_uuids", getPinnedPartUUIDs()->toString(), zkutil::CreateMode::Persistent));
|
||||
/// For ALTER PARTITION with multi-leaders
|
||||
futures.push_back(zookeeper->asyncTryCreateNoThrow(zookeeper_path + "/alter_partition_version", String(), zkutil::CreateMode::Persistent));
|
||||
/// For deduplication of async inserts
|
||||
futures.push_back(zookeeper->asyncTryCreateNoThrow(zookeeper_path + "/async_blocks", String(), zkutil::CreateMode::Persistent));
|
||||
|
||||
/// As for now, "/temp" node must exist, but we want to be able to remove it in future
|
||||
if (zookeeper->exists(zookeeper_path + "/temp"))
|
||||
@ -4535,7 +4537,7 @@ SinkToStoragePtr StorageReplicatedMergeTree::write(const ASTPtr & /*query*/, con
|
||||
const auto storage_settings_ptr = getSettings();
|
||||
const Settings & query_settings = local_context->getSettingsRef();
|
||||
bool deduplicate = storage_settings_ptr->replicated_deduplication_window != 0 && query_settings.insert_deduplicate;
|
||||
bool async_deduplicate = query_settings.async_insert && storage_settings_ptr->replicated_deduplication_window_for_async_inserts != 0 && query_settings.insert_deduplicate;
|
||||
bool async_deduplicate = query_settings.async_insert && query_settings.async_insert_deduplicate && storage_settings_ptr->replicated_deduplication_window_for_async_inserts != 0 && query_settings.insert_deduplicate;
|
||||
if (async_deduplicate)
|
||||
return std::make_shared<ReplicatedMergeTreeSinkWithAsyncDeduplicate>(
|
||||
*this, metadata_snapshot, query_settings.insert_quorum.valueOr(0),
|
||||
@ -6562,7 +6564,7 @@ void StorageReplicatedMergeTree::getClearBlocksInPartitionOpsImpl(
|
||||
{
|
||||
Strings blocks;
|
||||
if (Coordination::Error::ZOK != zookeeper.tryGetChildren(fs::path(zookeeper_path) / blocks_dir_name, blocks))
|
||||
throw Exception(zookeeper_path + "/" + blocks_dir_name + "blocks doesn't exist", ErrorCodes::NOT_FOUND_NODE);
|
||||
throw Exception(ErrorCodes::NOT_FOUND_NODE, "Node {}/{} doesn't exist", zookeeper_path, blocks_dir_name);
|
||||
|
||||
String partition_prefix = partition_id + "_";
|
||||
Strings paths_to_get;
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
#include <Access/ContextAccess.h>
|
||||
#include <Storages/System/StorageSystemDatabases.h>
|
||||
#include <Parsers/ASTCreateQuery.h>
|
||||
#include <Common/logger_useful.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -49,6 +50,7 @@ static String getEngineFull(const DatabasePtr & database)
|
||||
return {};
|
||||
|
||||
guard.reset();
|
||||
LOG_TRACE(&Poco::Logger::get("StorageSystemDatabases"), "Failed to lock database {} ({}), will retry", name, database->getUUID());
|
||||
}
|
||||
|
||||
ASTPtr ast = database->getCreateDatabaseQuery();
|
||||
|
||||
@ -22,9 +22,6 @@ endif()
|
||||
if (TARGET ch_rust::blake3)
|
||||
set(USE_BLAKE3 1)
|
||||
endif()
|
||||
if (TARGET ch_rust::skim)
|
||||
set(USE_SKIM 1)
|
||||
endif()
|
||||
if (TARGET OpenSSL::SSL)
|
||||
set(USE_SSL 1)
|
||||
endif()
|
||||
|
||||
@ -145,7 +145,7 @@ if __name__ == "__main__":
|
||||
)
|
||||
logging.info("Going to run func tests: %s", run_command)
|
||||
|
||||
with TeePopen(run_command, run_log_path) as process:
|
||||
with TeePopen(run_command, run_log_path, timeout=60 * 150) as process:
|
||||
retcode = process.wait()
|
||||
if retcode == 0:
|
||||
logging.info("Run successfully")
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from io import TextIOWrapper
|
||||
from subprocess import Popen, PIPE, STDOUT
|
||||
from threading import Thread
|
||||
from time import sleep
|
||||
@ -14,15 +15,23 @@ import sys
|
||||
# it finishes. stderr and stdout will be redirected both to specified file and
|
||||
# stdout.
|
||||
class TeePopen:
|
||||
# pylint: disable=W0102
|
||||
def __init__(self, command, log_file, env=os.environ.copy(), timeout=None):
|
||||
def __init__(
|
||||
self,
|
||||
command: str,
|
||||
log_file: str,
|
||||
env: Optional[dict] = None,
|
||||
timeout: Optional[int] = None,
|
||||
):
|
||||
self.command = command
|
||||
self.log_file = log_file
|
||||
self.env = env
|
||||
self._log_file_name = log_file
|
||||
self._log_file = None # type: Optional[TextIOWrapper]
|
||||
self.env = env or os.environ.copy()
|
||||
self._process = None # type: Optional[Popen]
|
||||
self.timeout = timeout
|
||||
|
||||
def _check_timeout(self):
|
||||
def _check_timeout(self) -> None:
|
||||
if self.timeout is None:
|
||||
return
|
||||
sleep(self.timeout)
|
||||
while self.process.poll() is None:
|
||||
logging.warning(
|
||||
@ -33,7 +42,7 @@ class TeePopen:
|
||||
os.killpg(self.process.pid, 9)
|
||||
sleep(10)
|
||||
|
||||
def __enter__(self):
|
||||
def __enter__(self) -> "TeePopen":
|
||||
self.process = Popen(
|
||||
self.command,
|
||||
shell=True,
|
||||
@ -44,25 +53,21 @@ class TeePopen:
|
||||
stdout=PIPE,
|
||||
bufsize=1,
|
||||
)
|
||||
self.log_file = open(self.log_file, "w", encoding="utf-8")
|
||||
if self.timeout is not None and self.timeout > 0:
|
||||
t = Thread(target=self._check_timeout)
|
||||
t.daemon = True # does not block the program from exit
|
||||
t.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, t, value, traceback):
|
||||
for line in self.process.stdout: # type: ignore
|
||||
sys.stdout.write(line)
|
||||
self.log_file.write(line)
|
||||
|
||||
self.process.wait()
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
self.wait()
|
||||
self.log_file.close()
|
||||
|
||||
def wait(self):
|
||||
for line in self.process.stdout: # type: ignore
|
||||
sys.stdout.write(line)
|
||||
self.log_file.write(line)
|
||||
if self.process.stdout is not None:
|
||||
for line in self.process.stdout:
|
||||
sys.stdout.write(line)
|
||||
self.log_file.write(line)
|
||||
|
||||
return self.process.wait()
|
||||
|
||||
@ -75,3 +80,9 @@ class TeePopen:
|
||||
@process.setter
|
||||
def process(self, process: Popen) -> None:
|
||||
self._process = process
|
||||
|
||||
@property
|
||||
def log_file(self) -> TextIOWrapper:
|
||||
if self._log_file is None:
|
||||
self._log_file = open(self._log_file_name, "w", encoding="utf-8")
|
||||
return self._log_file
|
||||
|
||||
@ -1180,6 +1180,9 @@ def test_tables_dependency():
|
||||
t4 = random_table_names[3]
|
||||
t5 = random_table_names[4]
|
||||
t6 = random_table_names[5]
|
||||
t7 = random_table_names[6]
|
||||
t8 = random_table_names[7]
|
||||
t9 = random_table_names[8]
|
||||
|
||||
# Create a materialized view and a dictionary with a local table as source.
|
||||
instance.query(
|
||||
@ -1193,7 +1196,7 @@ def test_tables_dependency():
|
||||
instance.query(f"CREATE MATERIALIZED VIEW {t3} TO {t2} AS SELECT x, y FROM {t1}")
|
||||
|
||||
instance.query(
|
||||
f"CREATE DICTIONARY {t4} (x Int64, y String) PRIMARY KEY x SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE '{t1.split('.')[1]}' DB '{t1.split('.')[0]}')) LAYOUT(FLAT()) LIFETIME(0)"
|
||||
f"CREATE DICTIONARY {t4} (x Int64, y String) PRIMARY KEY x SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() TABLE '{t1.split('.')[1]}' DB '{t1.split('.')[0]}')) LAYOUT(FLAT()) LIFETIME(4)"
|
||||
)
|
||||
|
||||
instance.query(f"CREATE TABLE {t5} AS dictionary({t4})")
|
||||
@ -1202,12 +1205,25 @@ def test_tables_dependency():
|
||||
f"CREATE TABLE {t6}(x Int64, y String DEFAULT dictGet({t4}, 'y', x)) ENGINE=MergeTree ORDER BY tuple()"
|
||||
)
|
||||
|
||||
instance.query(f"CREATE VIEW {t7} AS SELECT sum(x) FROM (SELECT x FROM {t6})")
|
||||
|
||||
instance.query(
|
||||
f"CREATE TABLE {t8} AS {t2} ENGINE = Buffer({t2.split('.')[0]}, {t2.split('.')[1]}, 16, 10, 100, 10000, 1000000, 10000000, 100000000)"
|
||||
)
|
||||
|
||||
instance.query(
|
||||
f"CREATE DICTIONARY {t9} (x Int64, y String) PRIMARY KEY x SOURCE(CLICKHOUSE(TABLE '{t1.split('.')[1]}' DB '{t1.split('.')[0]}')) LAYOUT(FLAT()) LIFETIME(9)"
|
||||
)
|
||||
|
||||
# Make backup.
|
||||
backup_name = new_backup_name()
|
||||
instance.query(f"BACKUP DATABASE test, DATABASE test2 TO {backup_name}")
|
||||
|
||||
# Drop everything in reversive order.
|
||||
def drop():
|
||||
instance.query(f"DROP DICTIONARY {t9}")
|
||||
instance.query(f"DROP TABLE {t8} NO DELAY")
|
||||
instance.query(f"DROP TABLE {t7} NO DELAY")
|
||||
instance.query(f"DROP TABLE {t6} NO DELAY")
|
||||
instance.query(f"DROP TABLE {t5} NO DELAY")
|
||||
instance.query(f"DROP DICTIONARY {t4}")
|
||||
@ -1219,11 +1235,36 @@ def test_tables_dependency():
|
||||
|
||||
drop()
|
||||
|
||||
# Restore everything and check.
|
||||
# Restore everything.
|
||||
instance.query(f"RESTORE ALL FROM {backup_name}")
|
||||
|
||||
# Check everything is restored.
|
||||
assert instance.query(
|
||||
"SELECT concat(database, '.', name) AS c FROM system.tables WHERE database IN ['test', 'test2'] ORDER BY c"
|
||||
) == TSV(sorted([t1, t2, t3, t4, t5, t6]))
|
||||
) == TSV(sorted([t1, t2, t3, t4, t5, t6, t7, t8, t9]))
|
||||
|
||||
# Check logs.
|
||||
instance.query("SYSTEM FLUSH LOGS")
|
||||
expect_in_logs = [
|
||||
f"Table {t1} has no dependencies (level 0)",
|
||||
f"Table {t2} has no dependencies (level 0)",
|
||||
(
|
||||
f"Table {t3} has 2 dependencies: {t1}, {t2} (level 1)",
|
||||
f"Table {t3} has 2 dependencies: {t2}, {t1} (level 1)",
|
||||
),
|
||||
f"Table {t4} has 1 dependencies: {t1} (level 1)",
|
||||
f"Table {t5} has 1 dependencies: {t4} (level 2)",
|
||||
f"Table {t6} has 1 dependencies: {t4} (level 2)",
|
||||
f"Table {t7} has 1 dependencies: {t6} (level 3)",
|
||||
f"Table {t8} has 1 dependencies: {t2} (level 1)",
|
||||
f"Table {t9} has 1 dependencies: {t1} (level 1)",
|
||||
]
|
||||
for expect in expect_in_logs:
|
||||
assert any(
|
||||
[
|
||||
instance.contains_in_log(f"RestorerFromBackup: {x}")
|
||||
for x in tuple(expect)
|
||||
]
|
||||
)
|
||||
|
||||
drop()
|
||||
|
||||
@ -796,6 +796,84 @@ def test_mutation():
|
||||
node1.query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}")
|
||||
|
||||
|
||||
def test_tables_dependency():
|
||||
node1.query("CREATE DATABASE mydb ON CLUSTER 'cluster3'")
|
||||
|
||||
node1.query(
|
||||
"CREATE TABLE mydb.src ON CLUSTER 'cluster' (x Int64, y String) ENGINE=MergeTree ORDER BY tuple()"
|
||||
)
|
||||
|
||||
node1.query(
|
||||
"CREATE DICTIONARY mydb.dict ON CLUSTER 'cluster' (x Int64, y String) PRIMARY KEY x "
|
||||
"SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() DB 'mydb' TABLE 'src')) LAYOUT(FLAT()) LIFETIME(0)"
|
||||
)
|
||||
|
||||
node1.query(
|
||||
"CREATE TABLE mydb.dist1 (x Int64) ENGINE=Distributed('cluster', 'mydb', 'src')"
|
||||
)
|
||||
|
||||
node3.query(
|
||||
"CREATE TABLE mydb.dist2 (x Int64) ENGINE=Distributed(cluster, 'mydb', 'src')"
|
||||
)
|
||||
|
||||
node1.query("CREATE TABLE mydb.clusterfunc1 AS cluster('cluster', 'mydb.src')")
|
||||
node1.query("CREATE TABLE mydb.clusterfunc2 AS cluster(cluster, mydb.src)")
|
||||
node1.query("CREATE TABLE mydb.clusterfunc3 AS cluster(cluster, 'mydb', 'src')")
|
||||
node1.query(
|
||||
"CREATE TABLE mydb.clusterfunc4 AS cluster(cluster, dictionary(mydb.dict))"
|
||||
)
|
||||
node1.query(
|
||||
"CREATE TABLE mydb.clusterfunc5 AS clusterAllReplicas(cluster, dictionary(mydb.dict))"
|
||||
)
|
||||
|
||||
node3.query("CREATE TABLE mydb.clusterfunc6 AS cluster('cluster', 'mydb.src')")
|
||||
node3.query("CREATE TABLE mydb.clusterfunc7 AS cluster(cluster, mydb.src)")
|
||||
node3.query("CREATE TABLE mydb.clusterfunc8 AS cluster(cluster, 'mydb', 'src')")
|
||||
node3.query(
|
||||
"CREATE TABLE mydb.clusterfunc9 AS cluster(cluster, dictionary(mydb.dict))"
|
||||
)
|
||||
node3.query(
|
||||
"CREATE TABLE mydb.clusterfunc10 AS clusterAllReplicas(cluster, dictionary(mydb.dict))"
|
||||
)
|
||||
|
||||
backup_name = new_backup_name()
|
||||
node3.query(f"BACKUP DATABASE mydb ON CLUSTER 'cluster3' TO {backup_name}")
|
||||
|
||||
node3.query("DROP DATABASE mydb")
|
||||
|
||||
node3.query(f"RESTORE DATABASE mydb ON CLUSTER 'cluster3' FROM {backup_name}")
|
||||
|
||||
node3.query("SYSTEM FLUSH LOGS ON CLUSTER 'cluster3'")
|
||||
expect_in_logs_1 = [
|
||||
"Table mydb.src has no dependencies (level 0)",
|
||||
"Table mydb.dict has 1 dependencies: mydb.src (level 1)",
|
||||
"Table mydb.dist1 has 1 dependencies: mydb.src (level 1)",
|
||||
"Table mydb.clusterfunc1 has 1 dependencies: mydb.src (level 1)",
|
||||
"Table mydb.clusterfunc2 has 1 dependencies: mydb.src (level 1)",
|
||||
"Table mydb.clusterfunc3 has 1 dependencies: mydb.src (level 1)",
|
||||
"Table mydb.clusterfunc4 has 1 dependencies: mydb.dict (level 2)",
|
||||
"Table mydb.clusterfunc5 has 1 dependencies: mydb.dict (level 2)",
|
||||
]
|
||||
expect_in_logs_2 = [
|
||||
"Table mydb.src has no dependencies (level 0)",
|
||||
"Table mydb.dict has 1 dependencies: mydb.src (level 1)",
|
||||
]
|
||||
expect_in_logs_3 = [
|
||||
"Table mydb.dist2 has no dependencies (level 0)",
|
||||
"Table mydb.clusterfunc6 has no dependencies (level 0)",
|
||||
"Table mydb.clusterfunc7 has no dependencies (level 0)",
|
||||
"Table mydb.clusterfunc8 has no dependencies (level 0)",
|
||||
"Table mydb.clusterfunc9 has no dependencies (level 0)",
|
||||
"Table mydb.clusterfunc10 has no dependencies (level 0)",
|
||||
]
|
||||
for expect in expect_in_logs_1:
|
||||
assert node1.contains_in_log(f"RestorerFromBackup: {expect}")
|
||||
for expect in expect_in_logs_2:
|
||||
assert node2.contains_in_log(f"RestorerFromBackup: {expect}")
|
||||
for expect in expect_in_logs_3:
|
||||
assert node3.contains_in_log(f"RestorerFromBackup: {expect}")
|
||||
|
||||
|
||||
def test_get_error_from_other_host():
|
||||
node1.query("CREATE TABLE tbl (`x` UInt8) ENGINE = MergeTree ORDER BY x")
|
||||
node1.query("INSERT INTO tbl VALUES (3)")
|
||||
|
||||
@ -39,7 +39,7 @@ RENAME TABLE test_01155_ordinary.mv1 TO test_01155_atomic.mv1;
|
||||
RENAME TABLE test_01155_ordinary.mv2 TO test_01155_atomic.mv2;
|
||||
RENAME TABLE test_01155_ordinary.dst TO test_01155_atomic.dst;
|
||||
RENAME TABLE test_01155_ordinary.src TO test_01155_atomic.src;
|
||||
SET check_table_dependencies=0; -- Otherwise we'll get error "test_01155_atomic.dict depends on test_01155_ordinary.dist" in the next line.
|
||||
SET check_table_dependencies=0; -- Otherwise we'll get error "test_01155_ordinary.dict depends on test_01155_ordinary.dist" in the next line.
|
||||
RENAME TABLE test_01155_ordinary.dist TO test_01155_atomic.dist;
|
||||
SET check_table_dependencies=1;
|
||||
RENAME DICTIONARY test_01155_ordinary.dict TO test_01155_atomic.dict;
|
||||
@ -65,7 +65,7 @@ SELECT dictGet('test_01155_ordinary.dict', 'x', 'after renaming database');
|
||||
SELECT database, substr(name, 1, 10) FROM system.tables WHERE database like 'test_01155_%';
|
||||
|
||||
-- Move tables back
|
||||
SET check_table_dependencies=0; -- Otherwise we'll get error "test_01155_atomic.dict depends on test_01155_ordinary.dist" in the next line.
|
||||
SET check_table_dependencies=0; -- Otherwise we'll get error "test_01155_ordinary.dict depends on test_01155_ordinary.dist" in the next line.
|
||||
RENAME DATABASE test_01155_ordinary TO test_01155_atomic;
|
||||
SET check_table_dependencies=1;
|
||||
|
||||
|
||||
@ -34,8 +34,8 @@ DROP VIEW IF EXISTS test_view_different_db;
|
||||
CREATE VIEW test_view_different_db AS SELECT id, value, dictGet('2025_test_db.test_dictionary', 'value', id) FROM 2025_test_db.view_table;
|
||||
SELECT * FROM test_view_different_db;
|
||||
|
||||
DROP TABLE 2025_test_db.test_table;
|
||||
DROP DICTIONARY 2025_test_db.test_dictionary;
|
||||
DROP TABLE 2025_test_db.test_table;
|
||||
DROP TABLE 2025_test_db.view_table;
|
||||
|
||||
DROP VIEW test_view_different_db;
|
||||
|
||||
@ -68,7 +68,7 @@ def generate_data(q, total_number):
|
||||
def fetch_and_insert_data(q, client):
|
||||
while True:
|
||||
insert = q.get()
|
||||
client.query(insert, settings = {"async_insert": 1, "wait_for_async_insert": 0, "async_insert_busy_timeout_ms": 1500, "insert_keeper_fault_injection_probability": 0})
|
||||
client.query(insert, settings = {"async_insert": 1, "async_insert_deduplicate": 1, "wait_for_async_insert": 0, "async_insert_busy_timeout_ms": 1500, "insert_keeper_fault_injection_probability": 0})
|
||||
q.task_done()
|
||||
sleep_time = random.randint(50, 500)
|
||||
time.sleep(sleep_time/1000.0)
|
||||
|
||||
215
tests/queries/0_stateless/02495_analyzer_storage_join.reference
Normal file
215
tests/queries/0_stateless/02495_analyzer_storage_join.reference
Normal file
@ -0,0 +1,215 @@
|
||||
--- no name clashes ---
|
||||
id2 id1 val key2 key1 a b x y
|
||||
0 0 0 6 -6 60 600 6000 60000
|
||||
5 -5 55 5 -5 50 500 5000 50000
|
||||
4 -4 44 4 -4 40 400 4000 40000
|
||||
3 -3 33 3 -3 30 300 3000 30000
|
||||
2 -2 22 2 -2 20 200 2000 20000
|
||||
id1 val key1 b x
|
||||
0 0 -6 600 6000
|
||||
-5 55 -5 500 5000
|
||||
-4 44 -4 400 4000
|
||||
-3 33 -3 300 3000
|
||||
-2 22 -2 200 2000
|
||||
id1 val key1 b x
|
||||
0 0 -6 600 6000
|
||||
-5 55 -5 500 5000
|
||||
-4 44 -4 400 4000
|
||||
-3 33 -3 300 3000
|
||||
-2 22 -2 200 2000
|
||||
val b x
|
||||
0 600 6000
|
||||
55 500 5000
|
||||
44 400 4000
|
||||
33 300 3000
|
||||
22 200 2000
|
||||
val
|
||||
0
|
||||
55
|
||||
44
|
||||
33
|
||||
22
|
||||
x
|
||||
6000
|
||||
5000
|
||||
4000
|
||||
3000
|
||||
2000
|
||||
--- name clashes ---
|
||||
-- using --
|
||||
key1 key2 t.b t.x val a tj.b tj.x y
|
||||
-6 6 0 0 0 60 600 6000 60000
|
||||
-5 5 55 555 5555 50 500 5000 50000
|
||||
-4 4 44 444 4444 40 400 4000 40000
|
||||
-3 3 33 333 2222 30 300 3000 30000
|
||||
-2 2 22 222 2222 20 200 2000 20000
|
||||
key1
|
||||
-6
|
||||
-5
|
||||
-4
|
||||
-3
|
||||
-2
|
||||
t.key1 tj.key1
|
||||
0 -6
|
||||
-5 -5
|
||||
-4 -4
|
||||
-3 -3
|
||||
-2 -2
|
||||
t.key2 tj.key2
|
||||
0 6
|
||||
5 5
|
||||
4 4
|
||||
3 3
|
||||
2 2
|
||||
t.b tj.b
|
||||
0 600
|
||||
55 500
|
||||
44 400
|
||||
33 300
|
||||
22 200
|
||||
t.x tj.b
|
||||
0 600
|
||||
555 500
|
||||
444 400
|
||||
333 300
|
||||
222 200
|
||||
a
|
||||
60
|
||||
50
|
||||
40
|
||||
30
|
||||
20
|
||||
tj.b
|
||||
600
|
||||
500
|
||||
400
|
||||
300
|
||||
200
|
||||
tj.x
|
||||
6000
|
||||
5000
|
||||
4000
|
||||
3000
|
||||
2000
|
||||
y
|
||||
60000
|
||||
50000
|
||||
40000
|
||||
30000
|
||||
20000
|
||||
a
|
||||
60
|
||||
50
|
||||
40
|
||||
30
|
||||
20
|
||||
y
|
||||
60000
|
||||
50000
|
||||
40000
|
||||
30000
|
||||
20000
|
||||
val
|
||||
0
|
||||
5555
|
||||
4444
|
||||
2222
|
||||
2222
|
||||
val
|
||||
0
|
||||
5555
|
||||
4444
|
||||
2222
|
||||
2222
|
||||
-- on --
|
||||
t.key2 t.key1 t.b t.x val tj.key2 tj.key1 a tj.b tj.x y
|
||||
5 -5 55 555 5555 5 -5 50 500 5000 50000
|
||||
4 -4 44 444 4444 4 -4 40 400 4000 40000
|
||||
3 -3 33 333 2222 3 -3 30 300 3000 30000
|
||||
2 -2 22 222 2222 2 -2 20 200 2000 20000
|
||||
0 0 0 0 0 6 -6 60 600 6000 60000
|
||||
t.key1 tj.key1
|
||||
-5 -5
|
||||
-4 -4
|
||||
-3 -3
|
||||
-2 -2
|
||||
0 -6
|
||||
t.key2 tj.key2
|
||||
5 5
|
||||
4 4
|
||||
3 3
|
||||
2 2
|
||||
0 6
|
||||
t.b tj.b
|
||||
55 500
|
||||
44 400
|
||||
33 300
|
||||
22 200
|
||||
0 600
|
||||
t.x tj.b
|
||||
555 500
|
||||
444 400
|
||||
333 300
|
||||
222 200
|
||||
0 600
|
||||
a
|
||||
50
|
||||
40
|
||||
30
|
||||
20
|
||||
60
|
||||
tj.b
|
||||
500
|
||||
400
|
||||
300
|
||||
200
|
||||
600
|
||||
tj.x
|
||||
5000
|
||||
4000
|
||||
3000
|
||||
2000
|
||||
6000
|
||||
y
|
||||
50000
|
||||
40000
|
||||
30000
|
||||
20000
|
||||
60000
|
||||
a
|
||||
50
|
||||
40
|
||||
30
|
||||
20
|
||||
60
|
||||
y
|
||||
50000
|
||||
40000
|
||||
30000
|
||||
20000
|
||||
60000
|
||||
val
|
||||
5555
|
||||
4444
|
||||
2222
|
||||
2222
|
||||
0
|
||||
val
|
||||
5555
|
||||
4444
|
||||
2222
|
||||
2222
|
||||
0
|
||||
--- unsupported and illegal conditions ---
|
||||
t.key2 t.key1 t.b t.x val tj.key2 tj.key1 a tj.b tj.x y
|
||||
5 -5 55 555 5555 4 -4 40 400 4000 40000
|
||||
4 -4 44 444 4444 3 -3 30 300 3000 30000
|
||||
3 -3 33 333 2222 2 -2 20 200 2000 20000
|
||||
0 0 0 0 0 5 -5 50 500 5000 50000
|
||||
0 0 0 0 0 6 -6 60 600 6000 60000
|
||||
t.key2 t.key1 t.b t.x val tj.key2 tj.key1 a tj.b tj.x y
|
||||
2 -2 22 222 2222 2 -2 20 200 2000 20000
|
||||
0 0 0 0 0 3 -3 30 300 3000 30000
|
||||
0 0 0 0 0 4 -4 40 400 4000 40000
|
||||
0 0 0 0 0 5 -5 50 500 5000 50000
|
||||
0 0 0 0 0 6 -6 60 600 6000 60000
|
||||
88
tests/queries/0_stateless/02495_analyzer_storage_join.sql
Normal file
88
tests/queries/0_stateless/02495_analyzer_storage_join.sql
Normal file
@ -0,0 +1,88 @@
|
||||
DROP TABLE IF EXISTS t;
|
||||
DROP TABLE IF EXISTS t1;
|
||||
DROP TABLE IF EXISTS tj;
|
||||
|
||||
SET allow_experimental_analyzer = 1;
|
||||
|
||||
CREATE TABLE tj (key2 UInt64, key1 Int64, a UInt64, b UInt64, x UInt64, y UInt64) ENGINE = Join(ALL, RIGHT, key1, key2);
|
||||
INSERT INTO tj VALUES (2, -2, 20, 200, 2000, 20000), (3, -3, 30, 300, 3000, 30000), (4, -4, 40, 400, 4000, 40000), (5, -5, 50, 500, 5000, 50000), (6, -6, 60, 600, 6000, 60000);
|
||||
|
||||
SELECT '--- no name clashes ---';
|
||||
|
||||
CREATE TABLE t1 (id2 UInt64, id1 Int64, val UInt64) ENGINE = Memory;
|
||||
INSERT INTO t1 VALUES (1, -1, 11), (2, -2, 22), (3, -3, 33), (4, -4, 44), (5, -5, 55);
|
||||
|
||||
SELECT * FROM t1 ALL RIGHT JOIN tj ON t1.id1 == tj.key1 AND t1.id2 == tj.key2 ORDER BY key1 FORMAT TSVWithNames;
|
||||
SELECT id1, val, key1, b, x FROM t1 ALL RIGHT JOIN tj ON t1.id1 == tj.key1 AND t1.id2 == tj.key2 ORDER BY key1 FORMAT TSVWithNames;
|
||||
SELECT t1.id1, t1.val, tj.key1, tj.b, tj.x FROM t1 ALL RIGHT JOIN tj ON t1.id1 == tj.key1 AND t1.id2 == tj.key2 ORDER BY key1 FORMAT TSVWithNames;
|
||||
SELECT val, b, x FROM t1 ALL RIGHT JOIN tj ON t1.id1 == tj.key1 AND t1.id2 == tj.key2 ORDER BY key1 FORMAT TSVWithNames;
|
||||
SELECT val FROM t1 ALL RIGHT JOIN tj ON t1.id1 == tj.key1 AND t1.id2 == tj.key2 ORDER BY key1 FORMAT TSVWithNames;
|
||||
SELECT x FROM t1 ALL RIGHT JOIN tj ON t1.id1 == tj.key1 AND t1.id2 == tj.key2 ORDER BY key1 FORMAT TSVWithNames;
|
||||
|
||||
SELECT '--- name clashes ---';
|
||||
|
||||
CREATE TABLE t (key2 UInt64, key1 Int64, b UInt64, x UInt64, val UInt64) ENGINE = Memory;
|
||||
INSERT INTO t VALUES (1, -1, 11, 111, 1111), (2, -2, 22, 222, 2222), (3, -3, 33, 333, 2222), (4, -4, 44, 444, 4444), (5, -5, 55, 555, 5555)
|
||||
|
||||
SELECT '-- using --';
|
||||
|
||||
SELECT * FROM t ALL RIGHT JOIN tj USING (key1, key2) ORDER BY key1 FORMAT TSVWithNames;
|
||||
SELECT key1 FROM t ALL RIGHT JOIN tj USING (key1, key2) ORDER BY key1 FORMAT TSVWithNames;
|
||||
SELECT t.key1, tj.key1 FROM t ALL RIGHT JOIN tj USING (key1, key2) ORDER BY key1 FORMAT TSVWithNames;
|
||||
SELECT t.key2, tj.key2 FROM t ALL RIGHT JOIN tj USING (key1, key2) ORDER BY key1 FORMAT TSVWithNames;
|
||||
SELECT t.b, tj.b FROM t ALL RIGHT JOIN tj USING (key1, key2) ORDER BY key1 FORMAT TSVWithNames;
|
||||
SELECT t.x, tj.b FROM t ALL RIGHT JOIN tj USING (key1, key2) ORDER BY key1 FORMAT TSVWithNames;
|
||||
SELECT tj.a FROM t ALL RIGHT JOIN tj USING (key1, key2) ORDER BY key1 FORMAT TSVWithNames;
|
||||
SELECT tj.b FROM t ALL RIGHT JOIN tj USING (key1, key2) ORDER BY key1 FORMAT TSVWithNames;
|
||||
SELECT tj.x FROM t ALL RIGHT JOIN tj USING (key1, key2) ORDER BY key1 FORMAT TSVWithNames;
|
||||
SELECT tj.y FROM t ALL RIGHT JOIN tj USING (key1, key2) ORDER BY key1 FORMAT TSVWithNames;
|
||||
SELECT a FROM t ALL RIGHT JOIN tj USING (key1, key2) ORDER BY key1 FORMAT TSVWithNames;
|
||||
SELECT b FROM t ALL RIGHT JOIN tj USING (key1, key2) ORDER BY key1 FORMAT TSVWithNames; -- { serverError AMBIGUOUS_IDENTIFIER }
|
||||
SELECT x FROM t ALL RIGHT JOIN tj USING (key1, key2) ORDER BY key1 FORMAT TSVWithNames; -- { serverError AMBIGUOUS_IDENTIFIER }
|
||||
SELECT y FROM t ALL RIGHT JOIN tj USING (key1, key2) ORDER BY key1 FORMAT TSVWithNames;
|
||||
SELECT t.val FROM t ALL RIGHT JOIN tj USING (key1, key2) ORDER BY key1 FORMAT TSVWithNames;
|
||||
SELECT val FROM t ALL RIGHT JOIN tj USING (key1, key2) ORDER BY key1 FORMAT TSVWithNames;
|
||||
|
||||
SELECT '-- on --';
|
||||
|
||||
SELECT * FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 ORDER BY t.key1 FORMAT TSVWithNames;
|
||||
SELECT key1 FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 ORDER BY t.key1 FORMAT TSVWithNames; -- { serverError AMBIGUOUS_IDENTIFIER }
|
||||
SELECT t.key1, tj.key1 FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 ORDER BY t.key1 FORMAT TSVWithNames;
|
||||
SELECT t.key2, tj.key2 FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 ORDER BY t.key1 FORMAT TSVWithNames;
|
||||
SELECT t.b, tj.b FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 ORDER BY t.key1 FORMAT TSVWithNames;
|
||||
SELECT t.x, tj.b FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 ORDER BY t.key1 FORMAT TSVWithNames;
|
||||
SELECT tj.a FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 ORDER BY t.key1 FORMAT TSVWithNames;
|
||||
SELECT tj.b FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 ORDER BY t.key1 FORMAT TSVWithNames;
|
||||
SELECT tj.x FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 ORDER BY t.key1 FORMAT TSVWithNames;
|
||||
SELECT tj.y FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 ORDER BY t.key1 FORMAT TSVWithNames;
|
||||
SELECT a FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 ORDER BY t.key1 FORMAT TSVWithNames;
|
||||
SELECT b FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 ORDER BY t.key1 FORMAT TSVWithNames; -- { serverError AMBIGUOUS_IDENTIFIER }
|
||||
SELECT x FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 ORDER BY t.key1 FORMAT TSVWithNames; -- { serverError AMBIGUOUS_IDENTIFIER }
|
||||
SELECT y FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 ORDER BY t.key1 FORMAT TSVWithNames;
|
||||
SELECT t.val FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 ORDER BY t.key1 FORMAT TSVWithNames;
|
||||
SELECT val FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 ORDER BY t.key1 FORMAT TSVWithNames;
|
||||
|
||||
SELECT '--- unsupported and illegal conditions ---';
|
||||
|
||||
SELECT * FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 + 1 FORMAT TSVWithNames; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
SELECT * FROM t ALL RIGHT JOIN tj ON t.key1 + 1 == tj.key1 AND toUInt64(t.key2 - 1) == tj.key2 ORDER BY t.key1, tj.key2 FORMAT TSVWithNames; -- Ok: expression on the left table
|
||||
|
||||
SELECT * FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 AND 1 == 1 FORMAT TSVWithNames; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
SELECT * FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 AND 1 == 2 FORMAT TSVWithNames; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
|
||||
SELECT * FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 AND tj.a == 20 FORMAT TSVWithNames; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
SELECT * FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 AND t.b == 22 ORDER BY t.key1, tj.key2 FORMAT TSVWithNames; -- Ok: t.b from the left table
|
||||
|
||||
SELECT * FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 AND 1 != 1 FORMAT TSVWithNames; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
SELECT * FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 AND NULL FORMAT TSVWithNames; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
|
||||
SELECT * FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 AND 'aaa' FORMAT TSVWithNames; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
|
||||
SELECT * FROM t ALL RIGHT JOIN tj ON 'aaa' FORMAT TSVWithNames; -- { serverError INVALID_JOIN_ON_EXPRESSION }
|
||||
|
||||
SELECT * FROM t ALL RIGHT JOIN tj ON t.key1 == tj.key1 AND t.key2 == tj.key2 AND 1 FORMAT TSVWithNames; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
SELECT * FROM t ALL RIGHT JOIN tj ON 0 FORMAT TSVWithNames; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
SELECT * FROM t ALL RIGHT JOIN tj ON 1 FORMAT TSVWithNames; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
|
||||
DROP TABLE IF EXISTS t;
|
||||
DROP TABLE IF EXISTS t1;
|
||||
DROP TABLE IF EXISTS tj;
|
||||
@ -1,2 +1,4 @@
|
||||
2 2 2
|
||||
3 0 3
|
||||
2 2 2
|
||||
3 0 3
|
||||
|
||||
@ -7,4 +7,8 @@ CREATE TABLE t2 (key UInt64, a UInt64) ENGINE = Join(ALL, RIGHT, key);
|
||||
INSERT INTO t1 VALUES (1, 1), (2, 2);
|
||||
INSERT INTO t2 VALUES (2, 2), (3, 3);
|
||||
|
||||
SET allow_experimental_analyzer = 0;
|
||||
SELECT * FROM t1 ALL RIGHT JOIN t2 USING (key) ORDER BY key;
|
||||
|
||||
SET allow_experimental_analyzer = 1;
|
||||
SELECT * FROM t1 ALL RIGHT JOIN t2 USING (key) ORDER BY key;
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
{% for _ in range(2) -%}
|
||||
--- using ---
|
||||
21 22 23 2000
|
||||
31 32 33 3000
|
||||
41 42 43 4000
|
||||
@ -14,6 +16,11 @@
|
||||
31 32 33 3000
|
||||
41 42 43 4000
|
||||
51 52 53 5000
|
||||
21 22 23 2000
|
||||
31 32 33 3000
|
||||
41 42 43 4000
|
||||
51 52 53 5000
|
||||
--- on ---
|
||||
21 22 23 22 21 23 2000
|
||||
31 32 33 32 31 33 3000
|
||||
41 42 43 42 41 43 4000
|
||||
@ -30,6 +37,7 @@
|
||||
31 32 33 32 31 33 3000
|
||||
41 42 43 42 41 43 4000
|
||||
51 52 53 52 51 53 5000
|
||||
--- on different name ---
|
||||
23 21 22 22 21 23 2000
|
||||
33 31 32 32 31 33 3000
|
||||
43 41 42 42 41 43 4000
|
||||
@ -42,6 +50,8 @@
|
||||
33 31 32 32 31 33 3000
|
||||
43 41 42 42 41 43 4000
|
||||
53 51 52 52 51 53 5000
|
||||
--- incompatible ---
|
||||
--- reuse column from left ---
|
||||
11 12 13 11 11 11 1000
|
||||
21 22 23 21 21 21 2000
|
||||
31 32 33 31 31 31 3000
|
||||
@ -52,3 +62,4 @@
|
||||
31 32 33 31 31 31 3000
|
||||
41 42 43 41 41 41 4000
|
||||
51 52 53 51 51 51 5000
|
||||
{% endfor -%}
|
||||
@ -8,20 +8,38 @@ INSERT INTO t1 VALUES (11, 12, 13), (21, 22, 23), (31, 32, 33), (41, 42, 43), (5
|
||||
CREATE TABLE tj (key2 UInt64, key1 UInt64, key3 UInt64, attr UInt64) ENGINE = Join(ALL, INNER, key3, key2, key1);
|
||||
INSERT INTO tj VALUES (22, 21, 23, 2000), (32, 31, 33, 3000), (42, 41, 43, 4000), (52, 51, 53, 5000), (62, 61, 63, 6000);
|
||||
|
||||
CREATE TABLE tjj (key2 UInt64, key1 UInt64, key3 UInt64, attr UInt64) ENGINE = Join(ALL, INNER, key3, key2, key1);
|
||||
INSERT INTO tjj VALUES (11, 11, 11, 1000), (21, 21, 21, 2000), (31, 31, 31, 3000), (41, 41, 41, 4000), (51, 51, 51, 5000), (61, 61, 61, 6000);
|
||||
|
||||
|
||||
{% for use_analyzer in [0, 1] -%}
|
||||
SET allow_experimental_analyzer = '{{ use_analyzer }}';
|
||||
|
||||
SELECT '--- using ---';
|
||||
SELECT * FROM t1 ALL INNER JOIN tj USING (key1, key2, key3) ORDER BY key1;
|
||||
SELECT * FROM t1 ALL INNER JOIN tj USING (key2, key3, key1) ORDER BY key1;
|
||||
SELECT * FROM t1 ALL INNER JOIN tj USING (key3, key2, key1) ORDER BY key1;
|
||||
SELECT * FROM t1 ALL INNER JOIN tj USING (key1, key3, key2) ORDER BY key1;
|
||||
SELECT key1, key2, key3, attr FROM t1 ALL INNER JOIN tj USING (key1, key2, key3) ORDER BY key1;
|
||||
SELECT key1, key2, key3, attr FROM t1 ALL INNER JOIN tj USING (key2, key3, key1) ORDER BY key1;
|
||||
SELECT key1, key2, key3, attr FROM t1 ALL INNER JOIN tj USING (key3, key2, key1) ORDER BY key1;
|
||||
SELECT key1, key2, key3, attr FROM t1 ALL INNER JOIN tj USING (key1, key3, key2) ORDER BY key1;
|
||||
|
||||
SELECT * FROM t1 ALL INNER JOIN tj ON t1.key3 = tj.key3 AND t1.key2 = tj.key2 AND t1.key1 = tj.key1 ORDER BY key1;
|
||||
SELECT * FROM t1 ALL INNER JOIN tj ON t1.key2 = tj.key2 AND t1.key3 = tj.key3 AND t1.key1 = tj.key1 ORDER BY key1;
|
||||
SELECT * FROM t1 ALL INNER JOIN tj ON t1.key3 = tj.key3 AND t1.key1 = tj.key1 AND t1.key2 = tj.key2 ORDER BY key1;
|
||||
SELECT * FROM t1 ALL INNER JOIN tj ON t1.key1 = tj.key1 AND t1.key3 = tj.key3 AND t1.key2 = tj.key2 ORDER BY key1;
|
||||
SELECT '--- on ---';
|
||||
SELECT * FROM t1 ALL INNER JOIN tj ON t1.key3 = tj.key3 AND t1.key2 = tj.key2 AND t1.key1 = tj.key1 ORDER BY t1.key1;
|
||||
SELECT * FROM t1 ALL INNER JOIN tj ON t1.key2 = tj.key2 AND t1.key3 = tj.key3 AND t1.key1 = tj.key1 ORDER BY t1.key1;
|
||||
SELECT * FROM t1 ALL INNER JOIN tj ON t1.key3 = tj.key3 AND t1.key1 = tj.key1 AND t1.key2 = tj.key2 ORDER BY t1.key1;
|
||||
SELECT * FROM t1 ALL INNER JOIN tj ON t1.key1 = tj.key1 AND t1.key3 = tj.key3 AND t1.key2 = tj.key2 ORDER BY t1.key1;
|
||||
|
||||
SELECT '--- on different name ---';
|
||||
SELECT * FROM (SELECT key3 AS c, key1 AS a, key2 AS b FROM t1) AS t1 ALL INNER JOIN tj ON t1.a = tj.key1 AND t1.c = tj.key3 AND t1.b = tj.key2 ORDER BY t1.a;
|
||||
SELECT * FROM (SELECT key3 AS c, key1 AS a, key2 AS b FROM t1) AS t1 ALL INNER JOIN tj ON t1.a = tj.key1 AND t1.b = tj.key2 AND t1.c = tj.key3 ORDER BY t1.a;
|
||||
SELECT * FROM (SELECT key3 AS c, key1 AS a, key2 AS b FROM t1) AS t1 ALL INNER JOIN tj ON t1.c = tj.key3 AND t1.a = tj.key1 AND t1.b = tj.key2 ORDER BY t1.a;
|
||||
|
||||
-- TODO (vdimir): uncomment after https://github.com/ClickHouse/ClickHouse/pull/44016
|
||||
-- SELECT * FROM t1 ALL INNER JOIN tj ON t1.key1 = tj.key1 AND t1.key3 = tj.key3 AND t1.key2 = tj.key2 AND 1; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
-- SELECT * FROM t1 ALL INNER JOIN tj ON t1.key1 = tj.key1 AND t1.key3 = tj.key3 AND t1.key2 = tj.key2 AND 0; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
-- SELECT * FROM t1 ALL INNER JOIN tj ON t1.key1 = tj.key1 AND t1.key3 = tj.key3 AND t1.key2 = tj.key2 AND 1 == 1; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
-- SELECT * FROM t1 ALL INNER JOIN tj ON t1.key1 = tj.key1 AND t1.key3 = tj.key3 AND t1.key2 = tj.key2 AND 1 > 1; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
|
||||
SELECT '--- incompatible ---';
|
||||
SELECT * FROM t1 ALL INNER JOIN tj ON 1; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
SELECT * FROM t1 ALL INNER JOIN tj ON 0; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
SELECT * FROM t1 ALL INNER JOIN tj ON NULL; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
@ -29,19 +47,20 @@ SELECT * FROM t1 ALL INNER JOIN tj ON 1 == 1; -- { serverError INCOMPATIBLE_TYPE
|
||||
SELECT * FROM t1 ALL INNER JOIN tj ON 1 != 1; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
|
||||
SELECT * FROM t1 ALL INNER JOIN tj USING (key2, key3); -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
SELECT * FROM t1 ALL INNER JOIN tj USING (key1, key2, attr); -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
SELECT * FROM t1 ALL INNER JOIN tj USING (key1, key2, key3, attr); -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
SELECT * FROM t1 ALL INNER JOIN tj USING (key1, key2, attr) SETTINGS allow_experimental_analyzer = 0; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
SELECT * FROM t1 ALL INNER JOIN tj USING (key1, key2, attr) SETTINGS allow_experimental_analyzer = 1; -- { serverError UNKNOWN_IDENTIFIER }
|
||||
SELECT * FROM t1 ALL INNER JOIN tj USING (key1, key2, key3, attr) SETTINGS allow_experimental_analyzer = 0; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
SELECT * FROM t1 ALL INNER JOIN tj USING (key1, key2, key3, attr) SETTINGS allow_experimental_analyzer = 1; -- { serverError UNKNOWN_IDENTIFIER }
|
||||
SELECT * FROM t1 ALL INNER JOIN tj ON t1.key1 = tj.attr; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
SELECT * FROM t1 ALL INNER JOIN tj ON t1.key1 = tj.key1; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
SELECT * FROM t1 ALL INNER JOIN tj ON t1.key1 = tj.key1 AND t1.key2 = tj.key2 AND t1.key3 = tj.attr; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
SELECT * FROM t1 ALL INNER JOIN tj ON t1.key1 = tj.key1 AND t1.key2 = tj.key2 AND t1.key3 = tj.key3 AND t1.key1 = tj.key1; -- { serverError INCOMPATIBLE_TYPE_OF_JOIN }
|
||||
|
||||
SELECT '--- reuse column from left ---';
|
||||
SELECT * FROM t1 ALL INNER JOIN tjj ON t1.key1 = tjj.key1 AND t1.key1 = tjj.key2 AND t1.key1 = tjj.key3 ORDER BY t1.key1;
|
||||
SELECT * FROM t1 ALL INNER JOIN tjj ON t1.key1 = tjj.key1 AND t1.key1 = tjj.key3 AND t1.key1 = tjj.key2 ORDER BY t1.key1;
|
||||
|
||||
CREATE TABLE tjj (key2 UInt64, key1 UInt64, key3 UInt64, attr UInt64) ENGINE = Join(ALL, INNER, key3, key2, key1);
|
||||
INSERT INTO tjj VALUES (11, 11, 11, 1000), (21, 21, 21, 2000), (31, 31, 31, 3000), (41, 41, 41, 4000), (51, 51, 51, 5000), (61, 61, 61, 6000);
|
||||
|
||||
SELECT * FROM t1 ALL INNER JOIN tjj ON t1.key1 = tjj.key1 AND t1.key1 = tjj.key2 AND t1.key1 = tjj.key3 ORDER BY key1;
|
||||
SELECT * FROM t1 ALL INNER JOIN tjj ON t1.key1 = tjj.key1 AND t1.key1 = tjj.key3 AND t1.key1 = tjj.key2 ORDER BY key1;
|
||||
{% endfor -%}
|
||||
|
||||
DROP TABLE IF EXISTS t1;
|
||||
DROP TABLE IF EXISTS tj;
|
||||
@ -0,0 +1,7 @@
|
||||
SET allow_deprecated_syntax_for_merge_tree=1;
|
||||
DROP TABLE IF EXISTS test54378;
|
||||
CREATE TABLE test54378 (`part_date` Date, `pk_date` Date, `date` Date) ENGINE = MergeTree(part_date, pk_date, 8192);
|
||||
INSERT INTO test54378 values ('2018-04-19', '2018-04-19', '2018-04-19');
|
||||
SELECT 232 FROM test54378 PREWHERE (part_date = (SELECT toDate('2018-04-19'))) IN (SELECT toDate('2018-04-19')) GROUP BY toDate(toDate(-2147483649, NULL), NULL), -inf;
|
||||
DROP TABLE test54378;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user