diff --git a/.gitmodules b/.gitmodules
index de7250166b8..f7dcf5f4ac1 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -133,7 +133,7 @@
url = https://github.com/unicode-org/icu.git
[submodule "contrib/flatbuffers"]
path = contrib/flatbuffers
- url = https://github.com/google/flatbuffers.git
+ url = https://github.com/ClickHouse-Extras/flatbuffers.git
[submodule "contrib/libc-headers"]
path = contrib/libc-headers
url = https://github.com/ClickHouse-Extras/libc-headers.git
@@ -221,6 +221,9 @@
[submodule "contrib/NuRaft"]
path = contrib/NuRaft
url = https://github.com/ClickHouse-Extras/NuRaft.git
+[submodule "contrib/nanodbc"]
+ path = contrib/nanodbc
+ url = https://github.com/ClickHouse-Extras/nanodbc.git
[submodule "contrib/datasketches-cpp"]
path = contrib/datasketches-cpp
url = https://github.com/ClickHouse-Extras/datasketches-cpp.git
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 43531b60267..cc1ec835a7b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,156 @@
+## ClickHouse release 21.4
+
+### ClickHouse release 21.4.1 2021-04-12
+
+#### Backward Incompatible Change
+
+* The `toStartOfIntervalFunction` will align hour intervals to the midnight (in previous versions they were aligned to the start of unix epoch). For example, `toStartOfInterval(x, INTERVAL 11 HOUR)` will split every day into three intervals: `00:00:00..10:59:59`, `11:00:00..21:59:59` and `22:00:00..23:59:59`. This behaviour is more suited for practical needs. This closes [#9510](https://github.com/ClickHouse/ClickHouse/issues/9510). [#22060](https://github.com/ClickHouse/ClickHouse/pull/22060) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* `Age` and `Precision` in graphite rollup configs should increase from retention to retention. Now it's checked and the wrong config raises an exception. [#21496](https://github.com/ClickHouse/ClickHouse/pull/21496) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
+* Fix `cutToFirstSignificantSubdomainCustom()`/`firstSignificantSubdomainCustom()` returning wrong result for 3+ level domains present in custom top-level domain list. For input domains matching these custom top-level domains, the third-level domain was considered to be the first significant one. This is now fixed. This change may introduce incompatibility if the function is used in e.g. the sharding key. [#21946](https://github.com/ClickHouse/ClickHouse/pull/21946) ([Azat Khuzhin](https://github.com/azat)).
+* Column `keys` in table `system.dictionaries` was replaced to columns `key.names` and `key.types`. Columns `key.names`, `key.types`, `attribute.names`, `attribute.types` from `system.dictionaries` table does not require dictionary to be loaded. [#21884](https://github.com/ClickHouse/ClickHouse/pull/21884) ([Maksim Kita](https://github.com/kitaisreal)).
+* Now replicas that are processing the `ALTER TABLE ATTACH PART[ITION]` command search in their `detached/` folders before fetching the data from other replicas. As an implementation detail, a new command `ATTACH_PART` is introduced in the replicated log. Parts are searched and compared by their checksums. [#18978](https://github.com/ClickHouse/ClickHouse/pull/18978) ([Mike Kot](https://github.com/myrrc)). **Note**:
+ * `ATTACH PART[ITION]` queries may not work during cluster upgrade.
+ * It's not possible to rollback to older ClickHouse version after executing `ALTER ... ATTACH` query in new version as the old servers would fail to pass the `ATTACH_PART` entry in the replicated log.
+* In this version, empty `` will block all access to remote hosts while in previous versions it did nothing. If you want to keep old behaviour and you have empty `remote_url_allow_hosts` element in configuration file, remove it. [#20058](https://github.com/ClickHouse/ClickHouse/pull/20058) ([Vladimir Chebotarev](https://github.com/excitoon)).
+
+
+#### New Feature
+
+* Extended range of `DateTime64` to support dates from year 1925 to 2283. Improved support of `DateTime` around zero date (`1970-01-01`). [#9404](https://github.com/ClickHouse/ClickHouse/pull/9404) ([alexey-milovidov](https://github.com/alexey-milovidov), [Vasily Nemkov](https://github.com/Enmk)). Not every time and date functions are working for extended range of dates.
+* Added support of Kerberos authentication for preconfigured users and HTTP requests (GSS-SPNEGO). [#14995](https://github.com/ClickHouse/ClickHouse/pull/14995) ([Denis Glazachev](https://github.com/traceon)).
+* Add `prefer_column_name_to_alias` setting to use original column names instead of aliases. it is needed to be more compatible with common databases' aliasing rules. This is for [#9715](https://github.com/ClickHouse/ClickHouse/issues/9715) and [#9887](https://github.com/ClickHouse/ClickHouse/issues/9887). [#22044](https://github.com/ClickHouse/ClickHouse/pull/22044) ([Amos Bird](https://github.com/amosbird)).
+* Added functions `dictGetChildren(dictionary, key)`, `dictGetDescendants(dictionary, key, level)`. Function `dictGetChildren` return all children as an array if indexes. It is a inverse transformation for `dictGetHierarchy`. Function `dictGetDescendants` return all descendants as if `dictGetChildren` was applied `level` times recursively. Zero `level` value is equivalent to infinity. Closes [#14656](https://github.com/ClickHouse/ClickHouse/issues/14656). [#22096](https://github.com/ClickHouse/ClickHouse/pull/22096) ([Maksim Kita](https://github.com/kitaisreal)).
+* Added `executable_pool` dictionary source. Close [#14528](https://github.com/ClickHouse/ClickHouse/issues/14528). [#21321](https://github.com/ClickHouse/ClickHouse/pull/21321) ([Maksim Kita](https://github.com/kitaisreal)).
+* Added table function `dictionary`. It works the same way as `Dictionary` engine. Closes [#21560](https://github.com/ClickHouse/ClickHouse/issues/21560). [#21910](https://github.com/ClickHouse/ClickHouse/pull/21910) ([Maksim Kita](https://github.com/kitaisreal)).
+* Support `Nullable` type for `PolygonDictionary` attribute. [#21890](https://github.com/ClickHouse/ClickHouse/pull/21890) ([Maksim Kita](https://github.com/kitaisreal)).
+* Functions `dictGet`, `dictHas` use current database name if it is not specified for dictionaries created with DDL. Closes [#21632](https://github.com/ClickHouse/ClickHouse/issues/21632). [#21859](https://github.com/ClickHouse/ClickHouse/pull/21859) ([Maksim Kita](https://github.com/kitaisreal)).
+* Added function `dictGetOrNull`. It works like `dictGet`, but return `Null` in case key was not found in dictionary. Closes [#22375](https://github.com/ClickHouse/ClickHouse/issues/22375). [#22413](https://github.com/ClickHouse/ClickHouse/pull/22413) ([Maksim Kita](https://github.com/kitaisreal)).
+* Added async update in `ComplexKeyCache`, `SSDCache`, `SSDComplexKeyCache` dictionaries. Added support for `Nullable` type in `Cache`, `ComplexKeyCache`, `SSDCache`, `SSDComplexKeyCache` dictionaries. Added support for multiple attributes fetch with `dictGet`, `dictGetOrDefault` functions. Fixes [#21517](https://github.com/ClickHouse/ClickHouse/issues/21517). [#20595](https://github.com/ClickHouse/ClickHouse/pull/20595) ([Maksim Kita](https://github.com/kitaisreal)).
+* Support `dictHas` function for `RangeHashedDictionary`. Fixes [#6680](https://github.com/ClickHouse/ClickHouse/issues/6680). [#19816](https://github.com/ClickHouse/ClickHouse/pull/19816) ([Maksim Kita](https://github.com/kitaisreal)).
+* Add function `timezoneOf` that returns the timezone name of `DateTime` or `DateTime64` data types. This does not close [#9959](https://github.com/ClickHouse/ClickHouse/issues/9959). Fix inconsistencies in function names: add aliases `timezone` and `timeZone` as well as `toTimezone` and `toTimeZone` and `timezoneOf` and `timeZoneOf`. [#22001](https://github.com/ClickHouse/ClickHouse/pull/22001) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Add new optional clause `GRANTEES` for `CREATE/ALTER USER` commands. It specifies users or roles which are allowed to receive grants from this user on condition this user has also all required access granted with grant option. By default `GRANTEES ANY` is used which means a user with grant option can grant to anyone. Syntax: `CREATE USER ... GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]`. [#21641](https://github.com/ClickHouse/ClickHouse/pull/21641) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Add new column `slowdowns_count` to `system.clusters`. When using hedged requests, it shows how many times we switched to another replica because this replica was responding slowly. Also show actual value of `errors_count` in `system.clusters`. [#21480](https://github.com/ClickHouse/ClickHouse/pull/21480) ([Kruglov Pavel](https://github.com/Avogar)).
+* Add `_partition_id` virtual column for `MergeTree*` engines. Allow to prune partitions by `_partition_id`. Add `partitionID()` function to calculate partition id string. [#21401](https://github.com/ClickHouse/ClickHouse/pull/21401) ([Amos Bird](https://github.com/amosbird)).
+* Add function `isIPAddressInRange` to test if an IPv4 or IPv6 address is contained in a given CIDR network prefix. [#21329](https://github.com/ClickHouse/ClickHouse/pull/21329) ([PHO](https://github.com/depressed-pho)).
+* Added new SQL command `ALTER TABLE 'table_name' UNFREEZE [PARTITION 'part_expr'] WITH NAME 'backup_name'`. This command is needed to properly remove 'freezed' partitions from all disks. [#21142](https://github.com/ClickHouse/ClickHouse/pull/21142) ([Pavel Kovalenko](https://github.com/Jokser)).
+* Supports implicit key type conversion for JOIN. [#19885](https://github.com/ClickHouse/ClickHouse/pull/19885) ([Vladimir](https://github.com/vdimir)).
+
+#### Experimental Feature
+
+* Support `RANGE OFFSET` frame (for window functions) for floating point types. Implement `lagInFrame`/`leadInFrame` window functions, which are analogous to `lag`/`lead`, but respect the window frame. They are identical when the frame is `between unbounded preceding and unbounded following`. This closes [#5485](https://github.com/ClickHouse/ClickHouse/issues/5485). [#21895](https://github.com/ClickHouse/ClickHouse/pull/21895) ([Alexander Kuzmenkov](https://github.com/akuzm)).
+* Zero-copy replication for `ReplicatedMergeTree` over S3 storage. [#16240](https://github.com/ClickHouse/ClickHouse/pull/16240) ([ianton-ru](https://github.com/ianton-ru)).
+* Added possibility to migrate existing S3 disk to the schema with backup-restore capabilities. [#22070](https://github.com/ClickHouse/ClickHouse/pull/22070) ([Pavel Kovalenko](https://github.com/Jokser)).
+
+#### Performance Improvement
+
+* Supported parallel formatting in `clickhouse-local` and everywhere else. [#21630](https://github.com/ClickHouse/ClickHouse/pull/21630) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Support parallel parsing for `CSVWithNames` and `TSVWithNames` formats. This closes [#21085](https://github.com/ClickHouse/ClickHouse/issues/21085). [#21149](https://github.com/ClickHouse/ClickHouse/pull/21149) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Enable read with mmap IO for file ranges from 64 MiB (the settings `min_bytes_to_use_mmap_io`). It may lead to moderate performance improvement. [#22326](https://github.com/ClickHouse/ClickHouse/pull/22326) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Add cache for files read with `min_bytes_to_use_mmap_io` setting. It makes significant (2x and more) performance improvement when the value of the setting is small by avoiding frequent mmap/munmap calls and the consequent page faults. Note that mmap IO has major drawbacks that makes it less reliable in production (e.g. hung or SIGBUS on faulty disks; less controllable memory usage). Nevertheless it is good in benchmarks. [#22206](https://github.com/ClickHouse/ClickHouse/pull/22206) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Avoid unnecessary data copy when using codec `NONE`. Please note that codec `NONE` is mostly useless - it's recommended to always use compression (`LZ4` is by default). Despite the common belief, disabling compression may not improve performance (the opposite effect is possible). The `NONE` codec is useful in some cases: - when data is uncompressable; - for synthetic benchmarks. [#22145](https://github.com/ClickHouse/ClickHouse/pull/22145) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Faster `GROUP BY` with small `max_rows_to_group_by` and `group_by_overflow_mode='any'`. [#21856](https://github.com/ClickHouse/ClickHouse/pull/21856) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Optimize performance of queries like `SELECT ... FINAL ... WHERE`. Now in queries with `FINAL` it's allowed to move to `PREWHERE` columns, which are in sorting key. [#21830](https://github.com/ClickHouse/ClickHouse/pull/21830) ([foolchi](https://github.com/foolchi)).
+* Improved performance by replacing `memcpy` to another implementation. This closes [#18583](https://github.com/ClickHouse/ClickHouse/issues/18583). [#21520](https://github.com/ClickHouse/ClickHouse/pull/21520) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Improve performance of aggregation in order of sorting key (with enabled setting `optimize_aggregation_in_order`). [#19401](https://github.com/ClickHouse/ClickHouse/pull/19401) ([Anton Popov](https://github.com/CurtizJ)).
+
+#### Improvement
+
+* Add connection pool for PostgreSQL table/database engine and dictionary source. Should fix [#21444](https://github.com/ClickHouse/ClickHouse/issues/21444). [#21839](https://github.com/ClickHouse/ClickHouse/pull/21839) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Support non-default table schema for postgres storage/table-function. Closes [#21701](https://github.com/ClickHouse/ClickHouse/issues/21701). [#21711](https://github.com/ClickHouse/ClickHouse/pull/21711) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Support replicas priority for postgres dictionary source. [#21710](https://github.com/ClickHouse/ClickHouse/pull/21710) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Introduce a new merge tree setting `min_bytes_to_rebalance_partition_over_jbod` which allows assigning new parts to different disks of a JBOD volume in a balanced way. [#16481](https://github.com/ClickHouse/ClickHouse/pull/16481) ([Amos Bird](https://github.com/amosbird)).
+* Added `Grant`, `Revoke` and `System` values of `query_kind` column for corresponding queries in `system.query_log`. [#21102](https://github.com/ClickHouse/ClickHouse/pull/21102) ([Vasily Nemkov](https://github.com/Enmk)).
+* Allow customizing timeouts for HTTP connections used for replication independently from other HTTP timeouts. [#20088](https://github.com/ClickHouse/ClickHouse/pull/20088) ([nvartolomei](https://github.com/nvartolomei)).
+* Better exception message in client in case of exception while server is writing blocks. In previous versions client may get misleading message like `Data compressed with different methods`. [#22427](https://github.com/ClickHouse/ClickHouse/pull/22427) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix error `Directory tmp_fetch_XXX already exists` which could happen after failed fetch part. Delete temporary fetch directory if it already exists. Fixes [#14197](https://github.com/ClickHouse/ClickHouse/issues/14197). [#22411](https://github.com/ClickHouse/ClickHouse/pull/22411) ([nvartolomei](https://github.com/nvartolomei)).
+* Fix MSan report for function `range` with `UInt256` argument (support for large integers is experimental). This closes [#22157](https://github.com/ClickHouse/ClickHouse/issues/22157). [#22387](https://github.com/ClickHouse/ClickHouse/pull/22387) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Add `current_database` column to `system.processes` table. It contains the current database of the query. [#22365](https://github.com/ClickHouse/ClickHouse/pull/22365) ([Alexander Kuzmenkov](https://github.com/akuzm)).
+* Add case-insensitive history search/navigation and subword movement features to `clickhouse-client`. [#22105](https://github.com/ClickHouse/ClickHouse/pull/22105) ([Amos Bird](https://github.com/amosbird)).
+* If tuple of NULLs, e.g. `(NULL, NULL)` is on the left hand side of `IN` operator with tuples of non-NULLs on the right hand side, e.g. `SELECT (NULL, NULL) IN ((0, 0), (3, 1))` return 0 instead of throwing an exception about incompatible types. The expression may also appear due to optimization of something like `SELECT (NULL, NULL) = (8, 0) OR (NULL, NULL) = (3, 2) OR (NULL, NULL) = (0, 0) OR (NULL, NULL) = (3, 1)`. This closes [#22017](https://github.com/ClickHouse/ClickHouse/issues/22017). [#22063](https://github.com/ClickHouse/ClickHouse/pull/22063) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Update used version of simdjson to 0.9.1. This fixes [#21984](https://github.com/ClickHouse/ClickHouse/issues/21984). [#22057](https://github.com/ClickHouse/ClickHouse/pull/22057) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Added case insensitive aliases for `CONNECTION_ID()` and `VERSION()` functions. This fixes [#22028](https://github.com/ClickHouse/ClickHouse/issues/22028). [#22042](https://github.com/ClickHouse/ClickHouse/pull/22042) ([Eugene Klimov](https://github.com/Slach)).
+* Add option `strict_increase` to `windowFunnel` function to calculate each event once (resolve [#21835](https://github.com/ClickHouse/ClickHouse/issues/21835)). [#22025](https://github.com/ClickHouse/ClickHouse/pull/22025) ([Vladimir](https://github.com/vdimir)).
+* If partition key of a `MergeTree` table does not include `Date` or `DateTime` columns but includes exactly one `DateTime64` column, expose its values in the `min_time` and `max_time` columns in `system.parts` and `system.parts_columns` tables. Add `min_time` and `max_time` columns to `system.parts_columns` table (these was inconsistency to the `system.parts` table). This closes [#18244](https://github.com/ClickHouse/ClickHouse/issues/18244). [#22011](https://github.com/ClickHouse/ClickHouse/pull/22011) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Supported `replication_alter_partitions_sync=1` setting in `clickhouse-copier` for moving partitions from helping table to destination. Decreased default timeouts. Fixes [#21911](https://github.com/ClickHouse/ClickHouse/issues/21911). [#21912](https://github.com/ClickHouse/ClickHouse/pull/21912) ([turbo jason](https://github.com/songenjie)).
+* Show path to data directory of `EmbeddedRocksDB` tables in system tables. [#21903](https://github.com/ClickHouse/ClickHouse/pull/21903) ([tavplubix](https://github.com/tavplubix)).
+* Add profile event `HedgedRequestsChangeReplica`, change read data timeout from sec to ms. [#21886](https://github.com/ClickHouse/ClickHouse/pull/21886) ([Kruglov Pavel](https://github.com/Avogar)).
+* DiskS3 (experimental feature under development). Fixed bug with the impossibility to move directory if the destination is not empty and cache disk is used. [#21837](https://github.com/ClickHouse/ClickHouse/pull/21837) ([Pavel Kovalenko](https://github.com/Jokser)).
+* Better formatting for `Array` and `Map` data types in Web UI. [#21798](https://github.com/ClickHouse/ClickHouse/pull/21798) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Update clusters only if their configurations were updated. [#21685](https://github.com/ClickHouse/ClickHouse/pull/21685) ([Kruglov Pavel](https://github.com/Avogar)).
+* Propagate query and session settings for distributed DDL queries. Set `distributed_ddl_entry_format_version` to 2 to enable this. Added `distributed_ddl_output_mode` setting. Supported modes: `none`, `throw` (default), `null_status_on_timeout` and `never_throw`. Miscellaneous fixes and improvements for `Replicated` database engine. [#21535](https://github.com/ClickHouse/ClickHouse/pull/21535) ([tavplubix](https://github.com/tavplubix)).
+* If `PODArray` was instantiated with element size that is neither a fraction or a multiple of 16, buffer overflow was possible. No bugs in current releases exist. [#21533](https://github.com/ClickHouse/ClickHouse/pull/21533) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Add `last_error_time`/`last_error_message`/`last_error_stacktrace`/`remote` columns for `system.errors`. [#21529](https://github.com/ClickHouse/ClickHouse/pull/21529) ([Azat Khuzhin](https://github.com/azat)).
+* Add aliases `simpleJSONExtract/simpleJSONHas` to `visitParam/visitParamExtract{UInt, Int, Bool, Float, Raw, String}`. Fixes #21383. [#21519](https://github.com/ClickHouse/ClickHouse/pull/21519) ([fastio](https://github.com/fastio)).
+* Add setting `optimize_skip_unused_shards_limit` to limit the number of sharding key values for `optimize_skip_unused_shards`. [#21512](https://github.com/ClickHouse/ClickHouse/pull/21512) ([Azat Khuzhin](https://github.com/azat)).
+* Improve `clickhouse-format` to not throw exception when there are extra spaces or comment after the last query, and throw exception early with readable message when format `ASTInsertQuery` with data . [#21311](https://github.com/ClickHouse/ClickHouse/pull/21311) ([flynn](https://github.com/ucasFL)).
+* Improve support of integer keys in data type `Map`. [#21157](https://github.com/ClickHouse/ClickHouse/pull/21157) ([Anton Popov](https://github.com/CurtizJ)).
+* MaterializeMySQL: attempt to reconnect to MySQL if the connection is lost. [#20961](https://github.com/ClickHouse/ClickHouse/pull/20961) ([Håvard Kvålen](https://github.com/havardk)).
+* Support more cases to rewrite `CROSS JOIN` to `INNER JOIN`. [#20392](https://github.com/ClickHouse/ClickHouse/pull/20392) ([Vladimir](https://github.com/vdimir)).
+* Do not create empty parts on INSERT when `optimize_on_insert` setting enabled. Fixes [#20304](https://github.com/ClickHouse/ClickHouse/issues/20304). [#20387](https://github.com/ClickHouse/ClickHouse/pull/20387) ([Kruglov Pavel](https://github.com/Avogar)).
+* `MaterializeMySQL`: add minmax skipping index for `_version` column. [#20382](https://github.com/ClickHouse/ClickHouse/pull/20382) ([Stig Bakken](https://github.com/stigsb)).
+* Add option `--backslash` for `clickhouse-format`, which can add a backslash at the end of each line of the formatted query. [#21494](https://github.com/ClickHouse/ClickHouse/pull/21494) ([flynn](https://github.com/ucasFL)).
+* Now clickhouse will not throw `LOGICAL_ERROR` exception when we try to mutate the already covered part. Fixes [#22013](https://github.com/ClickHouse/ClickHouse/issues/22013). [#22291](https://github.com/ClickHouse/ClickHouse/pull/22291) ([alesapin](https://github.com/alesapin)).
+
+#### Bug Fix
+
+* Remove socket from epoll before cancelling packet receiver in `HedgedConnections` to prevent possible race. Fixes [#22161](https://github.com/ClickHouse/ClickHouse/issues/22161). [#22443](https://github.com/ClickHouse/ClickHouse/pull/22443) ([Kruglov Pavel](https://github.com/Avogar)).
+* Add (missing) memory accounting in parallel parsing routines. In previous versions OOM was possible when the resultset contains very large blocks of data. This closes [#22008](https://github.com/ClickHouse/ClickHouse/issues/22008). [#22425](https://github.com/ClickHouse/ClickHouse/pull/22425) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Fix exception which may happen when `SELECT` has constant `WHERE` condition and source table has columns which names are digits. [#22270](https://github.com/ClickHouse/ClickHouse/pull/22270) ([LiuNeng](https://github.com/liuneng1994)).
+* Fix query cancellation with `use_hedged_requests=0` and `async_socket_for_remote=1`. [#22183](https://github.com/ClickHouse/ClickHouse/pull/22183) ([Azat Khuzhin](https://github.com/azat)).
+* Fix uncaught exception in `InterserverIOHTTPHandler`. [#22146](https://github.com/ClickHouse/ClickHouse/pull/22146) ([Azat Khuzhin](https://github.com/azat)).
+* Fix docker entrypoint in case `http_port` is not in the config. [#22132](https://github.com/ClickHouse/ClickHouse/pull/22132) ([Ewout](https://github.com/devwout)).
+* Fix error `Invalid number of rows in Chunk` in `JOIN` with `TOTALS` and `arrayJoin`. Closes [#19303](https://github.com/ClickHouse/ClickHouse/issues/19303). [#22129](https://github.com/ClickHouse/ClickHouse/pull/22129) ([Vladimir](https://github.com/vdimir)).
+* Fix the background thread pool name which used to poll message from Kafka. The Kafka engine with the broken thread pool will not consume the message from message queue. [#22122](https://github.com/ClickHouse/ClickHouse/pull/22122) ([fastio](https://github.com/fastio)).
+* Fix waiting for `OPTIMIZE` and `ALTER` queries for `ReplicatedMergeTree` table engines. Now the query will not hang when the table was detached or restarted. [#22118](https://github.com/ClickHouse/ClickHouse/pull/22118) ([alesapin](https://github.com/alesapin)).
+* Disable `async_socket_for_remote`/`use_hedged_requests` for buggy Linux kernels. [#22109](https://github.com/ClickHouse/ClickHouse/pull/22109) ([Azat Khuzhin](https://github.com/azat)).
+* Docker entrypoint: avoid chown of `.` in case when `LOG_PATH` is empty. Closes [#22100](https://github.com/ClickHouse/ClickHouse/issues/22100). [#22102](https://github.com/ClickHouse/ClickHouse/pull/22102) ([filimonov](https://github.com/filimonov)).
+* The function `decrypt` was lacking a check for the minimal size of data encrypted in `AEAD` mode. This closes [#21897](https://github.com/ClickHouse/ClickHouse/issues/21897). [#22064](https://github.com/ClickHouse/ClickHouse/pull/22064) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* In rare case, merge for `CollapsingMergeTree` may create granule with `index_granularity + 1` rows. Because of this, internal check, added in [#18928](https://github.com/ClickHouse/ClickHouse/issues/18928) (affects 21.2 and 21.3), may fail with error `Incomplete granules are not allowed while blocks are granules size`. This error did not allow parts to merge. [#21976](https://github.com/ClickHouse/ClickHouse/pull/21976) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Reverted [#15454](https://github.com/ClickHouse/ClickHouse/issues/15454) that may cause significant increase in memory usage while loading external dictionaries of hashed type. This closes [#21935](https://github.com/ClickHouse/ClickHouse/issues/21935). [#21948](https://github.com/ClickHouse/ClickHouse/pull/21948) ([Maksim Kita](https://github.com/kitaisreal)).
+* Prevent hedged connections overlaps (`Unknown packet 9 from server` error). [#21941](https://github.com/ClickHouse/ClickHouse/pull/21941) ([Azat Khuzhin](https://github.com/azat)).
+* Fix reading the HTTP POST request with "multipart/form-data" content type in some cases. [#21936](https://github.com/ClickHouse/ClickHouse/pull/21936) ([Ivan](https://github.com/abyss7)).
+* Fix wrong `ORDER BY` results when a query contains window functions, and optimization for reading in primary key order is applied. Fixes [#21828](https://github.com/ClickHouse/ClickHouse/issues/21828). [#21915](https://github.com/ClickHouse/ClickHouse/pull/21915) ([Alexander Kuzmenkov](https://github.com/akuzm)).
+* Fix deadlock in first catboost model execution. Closes [#13832](https://github.com/ClickHouse/ClickHouse/issues/13832). [#21844](https://github.com/ClickHouse/ClickHouse/pull/21844) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix incorrect query result (and possible crash) which could happen when `WHERE` or `HAVING` condition is pushed before `GROUP BY`. Fixes [#21773](https://github.com/ClickHouse/ClickHouse/issues/21773). [#21841](https://github.com/ClickHouse/ClickHouse/pull/21841) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Better error handling and logging in `WriteBufferFromS3`. [#21836](https://github.com/ClickHouse/ClickHouse/pull/21836) ([Pavel Kovalenko](https://github.com/Jokser)).
+* Fix possible crashes in aggregate functions with combinator `Distinct`, while using two-level aggregation. This is a follow-up fix of [#18365](https://github.com/ClickHouse/ClickHouse/pull/18365) . Can only reproduced in production env. [#21818](https://github.com/ClickHouse/ClickHouse/pull/21818) ([Amos Bird](https://github.com/amosbird)).
+* Fix scalar subquery index analysis. This fixes [#21717](https://github.com/ClickHouse/ClickHouse/issues/21717) , which was introduced in [#18896](https://github.com/ClickHouse/ClickHouse/pull/18896). [#21766](https://github.com/ClickHouse/ClickHouse/pull/21766) ([Amos Bird](https://github.com/amosbird)).
+* Fix bug for `ReplicatedMerge` table engines when `ALTER MODIFY COLUMN` query doesn't change the type of `Decimal` column if its size (32 bit or 64 bit) doesn't change. [#21728](https://github.com/ClickHouse/ClickHouse/pull/21728) ([alesapin](https://github.com/alesapin)).
+* Fix possible infinite waiting when concurrent `OPTIMIZE` and `DROP` are run for `ReplicatedMergeTree`. [#21716](https://github.com/ClickHouse/ClickHouse/pull/21716) ([Azat Khuzhin](https://github.com/azat)).
+* Fix function `arrayElement` with type `Map` for constant integer arguments. [#21699](https://github.com/ClickHouse/ClickHouse/pull/21699) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix SIGSEGV on not existing attributes from `ip_trie` with `access_to_key_from_attributes`. [#21692](https://github.com/ClickHouse/ClickHouse/pull/21692) ([Azat Khuzhin](https://github.com/azat)).
+* Server now start accepting connections only after `DDLWorker` and dictionaries initialization. [#21676](https://github.com/ClickHouse/ClickHouse/pull/21676) ([Azat Khuzhin](https://github.com/azat)).
+* Add type conversion for keys of tables of type `Join` (previously led to SIGSEGV). [#21646](https://github.com/ClickHouse/ClickHouse/pull/21646) ([Azat Khuzhin](https://github.com/azat)).
+* Fix distributed requests cancellation (for example simple select from multiple shards with limit, i.e. `select * from remote('127.{2,3}', system.numbers) limit 100`) with `async_socket_for_remote=1`. [#21643](https://github.com/ClickHouse/ClickHouse/pull/21643) ([Azat Khuzhin](https://github.com/azat)).
+* Fix `fsync_part_directory` for horizontal merge. [#21642](https://github.com/ClickHouse/ClickHouse/pull/21642) ([Azat Khuzhin](https://github.com/azat)).
+* Remove unknown columns from joined table in `WHERE` for queries to external database engines (MySQL, PostgreSQL). close [#14614](https://github.com/ClickHouse/ClickHouse/issues/14614), close [#19288](https://github.com/ClickHouse/ClickHouse/issues/19288) (dup), close [#19645](https://github.com/ClickHouse/ClickHouse/issues/19645) (dup). [#21640](https://github.com/ClickHouse/ClickHouse/pull/21640) ([Vladimir](https://github.com/vdimir)).
+* `std::terminate` was called if there is an error writing data into s3. [#21624](https://github.com/ClickHouse/ClickHouse/pull/21624) ([Vladimir](https://github.com/vdimir)).
+* Fix possible error `Cannot find column` when `optimize_skip_unused_shards` is enabled and zero shards are used. [#21579](https://github.com/ClickHouse/ClickHouse/pull/21579) ([Azat Khuzhin](https://github.com/azat)).
+* In case if query has constant `WHERE` condition, and setting `optimize_skip_unused_shards` enabled, all shards may be skipped and query could return incorrect empty result. [#21550](https://github.com/ClickHouse/ClickHouse/pull/21550) ([Amos Bird](https://github.com/amosbird)).
+* Fix table function `clusterAllReplicas` returns wrong `_shard_num`. close [#21481](https://github.com/ClickHouse/ClickHouse/issues/21481). [#21498](https://github.com/ClickHouse/ClickHouse/pull/21498) ([flynn](https://github.com/ucasFL)).
+* Fix that S3 table holds old credentials after config update. [#21457](https://github.com/ClickHouse/ClickHouse/pull/21457) ([Grigory Pervakov](https://github.com/GrigoryPervakov)).
+* Fixed race on SSL object inside `SecureSocket` in Poco. [#21456](https://github.com/ClickHouse/ClickHouse/pull/21456) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Fix `Avro` format parsing for `Kafka`. Fixes [#21437](https://github.com/ClickHouse/ClickHouse/issues/21437). [#21438](https://github.com/ClickHouse/ClickHouse/pull/21438) ([Ilya Golshtein](https://github.com/ilejn)).
+* Fix receive and send timeouts and non-blocking read in secure socket. [#21429](https://github.com/ClickHouse/ClickHouse/pull/21429) ([Kruglov Pavel](https://github.com/Avogar)).
+* `force_drop_table` flag didn't work for `MATERIALIZED VIEW`, it's fixed. Fixes [#18943](https://github.com/ClickHouse/ClickHouse/issues/18943). [#20626](https://github.com/ClickHouse/ClickHouse/pull/20626) ([tavplubix](https://github.com/tavplubix)).
+* Fix name clashes in `PredicateRewriteVisitor`. It caused incorrect `WHERE` filtration after full join. Close [#20497](https://github.com/ClickHouse/ClickHouse/issues/20497). [#20622](https://github.com/ClickHouse/ClickHouse/pull/20622) ([Vladimir](https://github.com/vdimir)).
+
+#### Build/Testing/Packaging Improvement
+
+* Add [Jepsen](https://github.com/jepsen-io/jepsen) tests for ClickHouse Keeper. [#21677](https://github.com/ClickHouse/ClickHouse/pull/21677) ([alesapin](https://github.com/alesapin)).
+* Run stateless tests in parallel in CI. Depends on [#22181](https://github.com/ClickHouse/ClickHouse/issues/22181). [#22300](https://github.com/ClickHouse/ClickHouse/pull/22300) ([alesapin](https://github.com/alesapin)).
+* Enable status check for [SQLancer](https://github.com/sqlancer/sqlancer) CI run. [#22015](https://github.com/ClickHouse/ClickHouse/pull/22015) ([Ilya Yatsishin](https://github.com/qoega)).
+* Multiple preparations for PowerPC builds: Enable the bundled openldap on `ppc64le`. [#22487](https://github.com/ClickHouse/ClickHouse/pull/22487) ([Kfir Itzhak](https://github.com/mastertheknife)). Enable compiling on `ppc64le` with Clang. [#22476](https://github.com/ClickHouse/ClickHouse/pull/22476) ([Kfir Itzhak](https://github.com/mastertheknife)). Fix compiling boost on `ppc64le`. [#22474](https://github.com/ClickHouse/ClickHouse/pull/22474) ([Kfir Itzhak](https://github.com/mastertheknife)). Fix CMake error about internal CMake variable `CMAKE_ASM_COMPILE_OBJECT` not set on `ppc64le`. [#22469](https://github.com/ClickHouse/ClickHouse/pull/22469) ([Kfir Itzhak](https://github.com/mastertheknife)). Fix Fedora/RHEL/CentOS not finding `libclang_rt.builtins` on `ppc64le`. [#22458](https://github.com/ClickHouse/ClickHouse/pull/22458) ([Kfir Itzhak](https://github.com/mastertheknife)). Enable building with `jemalloc` on `ppc64le`. [#22447](https://github.com/ClickHouse/ClickHouse/pull/22447) ([Kfir Itzhak](https://github.com/mastertheknife)). Fix ClickHouse's config embedding and cctz's timezone embedding on `ppc64le`. [#22445](https://github.com/ClickHouse/ClickHouse/pull/22445) ([Kfir Itzhak](https://github.com/mastertheknife)). Fixed compiling on `ppc64le` and use the correct instruction pointer register on `ppc64le`. [#22430](https://github.com/ClickHouse/ClickHouse/pull/22430) ([Kfir Itzhak](https://github.com/mastertheknife)).
+* Re-enable the S3 (AWS) library on `aarch64`. [#22484](https://github.com/ClickHouse/ClickHouse/pull/22484) ([Kfir Itzhak](https://github.com/mastertheknife)).
+* Add `tzdata` to Docker containers because reading `ORC` formats requires it. This closes [#14156](https://github.com/ClickHouse/ClickHouse/issues/14156). [#22000](https://github.com/ClickHouse/ClickHouse/pull/22000) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Introduce 2 arguments for `clickhouse-server` image Dockerfile: `deb_location` & `single_binary_location`. [#21977](https://github.com/ClickHouse/ClickHouse/pull/21977) ([filimonov](https://github.com/filimonov)).
+* Allow to use clang-tidy with release builds by enabling assertions if it is used. [#21914](https://github.com/ClickHouse/ClickHouse/pull/21914) ([alexey-milovidov](https://github.com/alexey-milovidov)).
+* Add llvm-12 binaries name to search in cmake scripts. Implicit constants conversions to mute clang warnings. Updated submodules to build with CMake 3.19. Mute recursion in macro expansion in `readpassphrase` library. Deprecated `-fuse-ld` changed to `--ld-path` for clang. [#21597](https://github.com/ClickHouse/ClickHouse/pull/21597) ([Ilya Yatsishin](https://github.com/qoega)).
+* Updating `docker/test/testflows/runner/dockerd-entrypoint.sh` to use Yandex dockerhub-proxy, because Docker Hub has enabled very restrictive rate limits [#21551](https://github.com/ClickHouse/ClickHouse/pull/21551) ([vzakaznikov](https://github.com/vzakaznikov)).
+* Fix macOS shared lib build. [#20184](https://github.com/ClickHouse/ClickHouse/pull/20184) ([nvartolomei](https://github.com/nvartolomei)).
+* Add `ctime` option to `zookeeper-dump-tree`. It allows to dump node creation time. [#21842](https://github.com/ClickHouse/ClickHouse/pull/21842) ([Ilya](https://github.com/HumanUser)).
+
+
## ClickHouse release 21.3 (LTS)
### ClickHouse release v21.3, 2021-03-12
@@ -26,7 +179,7 @@
#### Experimental feature
* Add experimental `Replicated` database engine. It replicates DDL queries across multiple hosts. [#16193](https://github.com/ClickHouse/ClickHouse/pull/16193) ([tavplubix](https://github.com/tavplubix)).
-* Introduce experimental support for window functions, enabled with `allow_experimental_functions = 1`. This is a preliminary, alpha-quality implementation that is not suitable for production use and will change in backward-incompatible ways in future releases. Please see [the documentation](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/sql-reference/window-functions/index.md#experimental-window-functions) for the list of supported features. [#20337](https://github.com/ClickHouse/ClickHouse/pull/20337) ([Alexander Kuzmenkov](https://github.com/akuzm)).
+* Introduce experimental support for window functions, enabled with `allow_experimental_window_functions = 1`. This is a preliminary, alpha-quality implementation that is not suitable for production use and will change in backward-incompatible ways in future releases. Please see [the documentation](https://github.com/ClickHouse/ClickHouse/blob/master/docs/en/sql-reference/window-functions/index.md#experimental-window-functions) for the list of supported features. [#20337](https://github.com/ClickHouse/ClickHouse/pull/20337) ([Alexander Kuzmenkov](https://github.com/akuzm)).
* Add the ability to backup/restore metadata files for DiskS3. [#18377](https://github.com/ClickHouse/ClickHouse/pull/18377) ([Pavel Kovalenko](https://github.com/Jokser)).
#### Performance Improvement
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5d716985c46..1423f3a0bc2 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -68,17 +68,30 @@ endif ()
include (cmake/find/ccache.cmake)
-option(ENABLE_CHECK_HEAVY_BUILDS "Don't allow C++ translation units to compile too long or to take too much memory while compiling" OFF)
+# Take care to add prlimit in command line before ccache, or else ccache thinks that
+# prlimit is compiler, and clang++ is its input file, and refuses to work with
+# multiple inputs, e.g in ccache log:
+# [2021-03-31T18:06:32.655327 36900] Command line: /usr/bin/ccache prlimit --as=10000000000 --data=5000000000 --cpu=600 /usr/bin/clang++-11 - ...... std=gnu++2a -MD -MT src/CMakeFiles/dbms.dir/Storages/MergeTree/IMergeTreeDataPart.cpp.o -MF src/CMakeFiles/dbms.dir/Storages/MergeTree/IMergeTreeDataPart.cpp.o.d -o src/CMakeFiles/dbms.dir/Storages/MergeTree/IMergeTreeDataPart.cpp.o -c ../src/Storages/MergeTree/IMergeTreeDataPart.cpp
+#
+# [2021-03-31T18:06:32.656704 36900] Multiple input files: /usr/bin/clang++-11 and ../src/Storages/MergeTree/IMergeTreeDataPart.cpp
+#
+# Another way would be to use --ccache-skip option before clang++-11 to make
+# ccache ignore it.
+option(ENABLE_CHECK_HEAVY_BUILDS "Don't allow C++ translation units to compile too long or to take too much memory while compiling." OFF)
if (ENABLE_CHECK_HEAVY_BUILDS)
# set DATA (since RSS does not work since 2.6.x+) to 2G
set (RLIMIT_DATA 5000000000)
# set VIRT (RLIMIT_AS) to 10G (DATA*10)
set (RLIMIT_AS 10000000000)
+ # set CPU time limit to 600 seconds
+ set (RLIMIT_CPU 600)
+
# gcc10/gcc10/clang -fsanitize=memory is too heavy
if (SANITIZE STREQUAL "memory" OR COMPILER_GCC)
set (RLIMIT_DATA 10000000000)
endif()
- set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=600)
+
+ set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=${RLIMIT_CPU} ${CMAKE_CXX_COMPILER_LAUNCHER})
endif ()
if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None")
@@ -277,6 +290,12 @@ if (COMPILER_GCC OR COMPILER_CLANG)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsized-deallocation")
endif ()
+# falign-functions=32 prevents from random performance regressions with the code change. Thus, providing more stable
+# benchmarks.
+if (COMPILER_GCC OR COMPILER_CLANG)
+ set(COMPILER_FLAGS "${COMPILER_FLAGS} -falign-functions=32")
+endif ()
+
# Compiler-specific coverage flags e.g. -fcoverage-mapping for gcc
option(WITH_COVERAGE "Profile the resulting binary/binaries" OFF)
@@ -464,6 +483,7 @@ find_contrib_lib(double-conversion) # Must be before parquet
include (cmake/find/ssl.cmake)
include (cmake/find/ldap.cmake) # after ssl
include (cmake/find/icu.cmake)
+include (cmake/find/xz.cmake)
include (cmake/find/zlib.cmake)
include (cmake/find/zstd.cmake)
include (cmake/find/ltdl.cmake) # for odbc
@@ -496,6 +516,7 @@ include (cmake/find/fast_float.cmake)
include (cmake/find/rapidjson.cmake)
include (cmake/find/fastops.cmake)
include (cmake/find/odbc.cmake)
+include (cmake/find/nanodbc.cmake)
include (cmake/find/rocksdb.cmake)
include (cmake/find/libpqxx.cmake)
include (cmake/find/nuraft.cmake)
diff --git a/base/CMakeLists.txt b/base/CMakeLists.txt
index 46bd57eda12..023dcaaccae 100644
--- a/base/CMakeLists.txt
+++ b/base/CMakeLists.txt
@@ -8,6 +8,7 @@ add_subdirectory (loggers)
add_subdirectory (pcg-random)
add_subdirectory (widechar_width)
add_subdirectory (readpassphrase)
+add_subdirectory (bridge)
if (USE_MYSQL)
add_subdirectory (mysqlxx)
diff --git a/base/bridge/CMakeLists.txt b/base/bridge/CMakeLists.txt
new file mode 100644
index 00000000000..20b0b651677
--- /dev/null
+++ b/base/bridge/CMakeLists.txt
@@ -0,0 +1,7 @@
+add_library (bridge
+ IBridge.cpp
+)
+
+target_include_directories (daemon PUBLIC ..)
+target_link_libraries (bridge PRIVATE daemon dbms Poco::Data Poco::Data::ODBC)
+
diff --git a/base/bridge/IBridge.cpp b/base/bridge/IBridge.cpp
new file mode 100644
index 00000000000..b1f71315fef
--- /dev/null
+++ b/base/bridge/IBridge.cpp
@@ -0,0 +1,238 @@
+#include "IBridge.h"
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#if USE_ODBC
+# include
+#endif
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+ extern const int ARGUMENT_OUT_OF_BOUND;
+}
+
+namespace
+{
+ Poco::Net::SocketAddress makeSocketAddress(const std::string & host, UInt16 port, Poco::Logger * log)
+ {
+ Poco::Net::SocketAddress socket_address;
+ try
+ {
+ socket_address = Poco::Net::SocketAddress(host, port);
+ }
+ catch (const Poco::Net::DNSException & e)
+ {
+ const auto code = e.code();
+ if (code == EAI_FAMILY
+#if defined(EAI_ADDRFAMILY)
+ || code == EAI_ADDRFAMILY
+#endif
+ )
+ {
+ LOG_ERROR(log, "Cannot resolve listen_host ({}), error {}: {}. If it is an IPv6 address and your host has disabled IPv6, then consider to specify IPv4 address to listen in element of configuration file. Example: 0.0.0.0", host, e.code(), e.message());
+ }
+
+ throw;
+ }
+ return socket_address;
+ }
+
+ Poco::Net::SocketAddress socketBindListen(Poco::Net::ServerSocket & socket, const std::string & host, UInt16 port, Poco::Logger * log)
+ {
+ auto address = makeSocketAddress(host, port, log);
+#if POCO_VERSION < 0x01080000
+ socket.bind(address, /* reuseAddress = */ true);
+#else
+ socket.bind(address, /* reuseAddress = */ true, /* reusePort = */ false);
+#endif
+
+ socket.listen(/* backlog = */ 64);
+
+ return address;
+ }
+}
+
+
+void IBridge::handleHelp(const std::string &, const std::string &)
+{
+ Poco::Util::HelpFormatter help_formatter(options());
+ help_formatter.setCommand(commandName());
+ help_formatter.setHeader("HTTP-proxy for odbc requests");
+ help_formatter.setUsage("--http-port ");
+ help_formatter.format(std::cerr);
+
+ stopOptionsProcessing();
+}
+
+
+void IBridge::defineOptions(Poco::Util::OptionSet & options)
+{
+ options.addOption(
+ Poco::Util::Option("http-port", "", "port to listen").argument("http-port", true) .binding("http-port"));
+
+ options.addOption(
+ Poco::Util::Option("listen-host", "", "hostname or address to listen, default 127.0.0.1").argument("listen-host").binding("listen-host"));
+
+ options.addOption(
+ Poco::Util::Option("http-timeout", "", "http timeout for socket, default 1800").argument("http-timeout").binding("http-timeout"));
+
+ options.addOption(
+ Poco::Util::Option("max-server-connections", "", "max connections to server, default 1024").argument("max-server-connections").binding("max-server-connections"));
+
+ options.addOption(
+ Poco::Util::Option("keep-alive-timeout", "", "keepalive timeout, default 10").argument("keep-alive-timeout").binding("keep-alive-timeout"));
+
+ options.addOption(
+ Poco::Util::Option("log-level", "", "sets log level, default info") .argument("log-level").binding("logger.level"));
+
+ options.addOption(
+ Poco::Util::Option("log-path", "", "log path for all logs, default console").argument("log-path").binding("logger.log"));
+
+ options.addOption(
+ Poco::Util::Option("err-log-path", "", "err log path for all logs, default no").argument("err-log-path").binding("logger.errorlog"));
+
+ options.addOption(
+ Poco::Util::Option("stdout-path", "", "stdout log path, default console").argument("stdout-path").binding("logger.stdout"));
+
+ options.addOption(
+ Poco::Util::Option("stderr-path", "", "stderr log path, default console").argument("stderr-path").binding("logger.stderr"));
+
+ using Me = std::decay_t;
+
+ options.addOption(
+ Poco::Util::Option("help", "", "produce this help message").binding("help").callback(Poco::Util::OptionCallback(this, &Me::handleHelp)));
+
+ ServerApplication::defineOptions(options); // NOLINT Don't need complex BaseDaemon's .xml config
+}
+
+
+void IBridge::initialize(Application & self)
+{
+ BaseDaemon::closeFDs();
+ is_help = config().has("help");
+
+ if (is_help)
+ return;
+
+ config().setString("logger", bridgeName());
+
+ /// Redirect stdout, stderr to specified files.
+ /// Some libraries and sanitizers write to stderr in case of errors.
+ const auto stdout_path = config().getString("logger.stdout", "");
+ if (!stdout_path.empty())
+ {
+ if (!freopen(stdout_path.c_str(), "a+", stdout))
+ throw Poco::OpenFileException("Cannot attach stdout to " + stdout_path);
+
+ /// Disable buffering for stdout.
+ setbuf(stdout, nullptr);
+ }
+ const auto stderr_path = config().getString("logger.stderr", "");
+ if (!stderr_path.empty())
+ {
+ if (!freopen(stderr_path.c_str(), "a+", stderr))
+ throw Poco::OpenFileException("Cannot attach stderr to " + stderr_path);
+
+ /// Disable buffering for stderr.
+ setbuf(stderr, nullptr);
+ }
+
+ buildLoggers(config(), logger(), self.commandName());
+
+ BaseDaemon::logRevision();
+
+ log = &logger();
+ hostname = config().getString("listen-host", "127.0.0.1");
+ port = config().getUInt("http-port");
+ if (port > 0xFFFF)
+ throw Exception("Out of range 'http-port': " + std::to_string(port), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
+
+ http_timeout = config().getUInt("http-timeout", DEFAULT_HTTP_READ_BUFFER_TIMEOUT);
+ max_server_connections = config().getUInt("max-server-connections", 1024);
+ keep_alive_timeout = config().getUInt("keep-alive-timeout", 10);
+
+ initializeTerminationAndSignalProcessing();
+
+#if USE_ODBC
+ if (bridgeName() == "ODBCBridge")
+ Poco::Data::ODBC::Connector::registerConnector();
+#endif
+
+ ServerApplication::initialize(self); // NOLINT
+}
+
+
+void IBridge::uninitialize()
+{
+ BaseDaemon::uninitialize();
+}
+
+
+int IBridge::main(const std::vector & /*args*/)
+{
+ if (is_help)
+ return Application::EXIT_OK;
+
+ registerFormats();
+ LOG_INFO(log, "Starting up {} on host: {}, port: {}", bridgeName(), hostname, port);
+
+ Poco::Net::ServerSocket socket;
+ auto address = socketBindListen(socket, hostname, port, log);
+ socket.setReceiveTimeout(http_timeout);
+ socket.setSendTimeout(http_timeout);
+
+ Poco::ThreadPool server_pool(3, max_server_connections);
+
+ Poco::Net::HTTPServerParams::Ptr http_params = new Poco::Net::HTTPServerParams;
+ http_params->setTimeout(http_timeout);
+ http_params->setKeepAliveTimeout(keep_alive_timeout);
+
+ auto shared_context = Context::createShared();
+ auto context = Context::createGlobal(shared_context.get());
+ context->makeGlobalContext();
+
+ if (config().has("query_masking_rules"))
+ SensitiveDataMasker::setInstance(std::make_unique(config(), "query_masking_rules"));
+
+ auto server = HTTPServer(
+ context,
+ getHandlerFactoryPtr(context),
+ server_pool,
+ socket,
+ http_params);
+
+ SCOPE_EXIT({
+ LOG_DEBUG(log, "Received termination signal.");
+ LOG_DEBUG(log, "Waiting for current connections to close.");
+
+ server.stop();
+
+ for (size_t count : ext::range(1, 6))
+ {
+ if (server.currentConnections() == 0)
+ break;
+ LOG_DEBUG(log, "Waiting for {} connections, try {}", server.currentConnections(), count);
+ std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+ }
+ });
+
+ server.start();
+ LOG_INFO(log, "Listening http://{}", address.toString());
+
+ waitForTerminationRequest();
+ return Application::EXIT_OK;
+}
+
+}
diff --git a/base/bridge/IBridge.h b/base/bridge/IBridge.h
new file mode 100644
index 00000000000..c64003d9959
--- /dev/null
+++ b/base/bridge/IBridge.h
@@ -0,0 +1,51 @@
+#pragma once
+
+#include
+#include
+#include
+
+#include
+#include
+
+
+namespace DB
+{
+
+/// Class represents base for clickhouse-odbc-bridge and clickhouse-library-bridge servers.
+/// Listens to incoming HTTP POST and GET requests on specified port and host.
+/// Has two handlers '/' for all incoming POST requests and /ping for GET request about service status.
+class IBridge : public BaseDaemon
+{
+
+public:
+ /// Define command line arguments
+ void defineOptions(Poco::Util::OptionSet & options) override;
+
+protected:
+ using HandlerFactoryPtr = std::shared_ptr;
+
+ void initialize(Application & self) override;
+
+ void uninitialize() override;
+
+ int main(const std::vector & args) override;
+
+ virtual std::string bridgeName() const = 0;
+
+ virtual HandlerFactoryPtr getHandlerFactoryPtr(ContextPtr context) const = 0;
+
+ size_t keep_alive_timeout;
+
+private:
+ void handleHelp(const std::string &, const std::string &);
+
+ bool is_help;
+ std::string hostname;
+ size_t port;
+ std::string log_level;
+ size_t max_server_connections;
+ size_t http_timeout;
+
+ Poco::Logger * log;
+};
+}
diff --git a/src/Common/BorrowedObjectPool.h b/base/common/BorrowedObjectPool.h
similarity index 99%
rename from src/Common/BorrowedObjectPool.h
rename to base/common/BorrowedObjectPool.h
index d5263cf92a8..6a90a7e7122 100644
--- a/src/Common/BorrowedObjectPool.h
+++ b/base/common/BorrowedObjectPool.h
@@ -7,8 +7,7 @@
#include
#include
-
-#include
+#include
/** Pool for limited size objects that cannot be used from different threads simultaneously.
* The main use case is to have fixed size of objects that can be reused in difference threads during their lifetime
diff --git a/base/common/DateLUTImpl.h b/base/common/DateLUTImpl.h
index 363f281584e..9e60181e802 100644
--- a/base/common/DateLUTImpl.h
+++ b/base/common/DateLUTImpl.h
@@ -25,7 +25,7 @@
#if defined(__PPC__)
-#if !__clang__
+#if !defined(__clang__)
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
#endif
@@ -1266,7 +1266,7 @@ public:
};
#if defined(__PPC__)
-#if !__clang__
+#if !defined(__clang__)
#pragma GCC diagnostic pop
#endif
#endif
diff --git a/src/Common/MoveOrCopyIfThrow.h b/base/common/MoveOrCopyIfThrow.h
similarity index 100%
rename from src/Common/MoveOrCopyIfThrow.h
rename to base/common/MoveOrCopyIfThrow.h
diff --git a/base/common/wide_integer_impl.h b/base/common/wide_integer_impl.h
index 5b981326e25..456c10a22e4 100644
--- a/base/common/wide_integer_impl.h
+++ b/base/common/wide_integer_impl.h
@@ -271,9 +271,13 @@ struct integer::_impl
/// As to_Integral does a static_cast to int64_t, it may result in UB.
/// The necessary check here is that long double has enough significant (mantissa) bits to store the
/// int64_t max value precisely.
+
+ //TODO Be compatible with Apple aarch64
+#if not (defined(__APPLE__) && defined(__aarch64__))
static_assert(LDBL_MANT_DIG >= 64,
"On your system long double has less than 64 precision bits,"
"which may result in UB when initializing double from int64_t");
+#endif
if ((rhs > 0 && rhs < static_cast(max_int)) || (rhs < 0 && rhs > static_cast(min_int)))
{
diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp
index 29430b65983..1b7d0064b99 100644
--- a/base/daemon/SentryWriter.cpp
+++ b/base/daemon/SentryWriter.cpp
@@ -9,6 +9,7 @@
#include
#include
+#include
#include
#include
#include
diff --git a/base/ext/scope_guard_safe.h b/base/ext/scope_guard_safe.h
new file mode 100644
index 00000000000..55140213572
--- /dev/null
+++ b/base/ext/scope_guard_safe.h
@@ -0,0 +1,68 @@
+#pragma once
+
+#include
+#include
+#include
+
+/// Same as SCOPE_EXIT() but block the MEMORY_LIMIT_EXCEEDED errors.
+///
+/// Typical example of SCOPE_EXIT_MEMORY() usage is when code under it may do
+/// some tiny allocations, that may fail under high memory pressure or/and low
+/// max_memory_usage (and related limits).
+///
+/// NOTE: it should be used with caution.
+#define SCOPE_EXIT_MEMORY(...) SCOPE_EXIT( \
+ MemoryTracker::LockExceptionInThread \
+ lock_memory_tracker(VariableContext::Global); \
+ __VA_ARGS__; \
+)
+
+/// Same as SCOPE_EXIT() but try/catch/tryLogCurrentException any exceptions.
+///
+/// SCOPE_EXIT_SAFE() should be used in case the exception during the code
+/// under SCOPE_EXIT() is not "that fatal" and error message in log is enough.
+///
+/// Good example is calling CurrentThread::detachQueryIfNotDetached().
+///
+/// Anti-pattern is calling WriteBuffer::finalize() under SCOPE_EXIT_SAFE()
+/// (since finalize() can do final write and it is better to fail abnormally
+/// instead of ignoring write error).
+///
+/// NOTE: it should be used with double caution.
+#define SCOPE_EXIT_SAFE(...) SCOPE_EXIT( \
+ try \
+ { \
+ __VA_ARGS__; \
+ } \
+ catch (...) \
+ { \
+ tryLogCurrentException(__PRETTY_FUNCTION__); \
+ } \
+)
+
+/// Same as SCOPE_EXIT() but:
+/// - block the MEMORY_LIMIT_EXCEEDED errors,
+/// - try/catch/tryLogCurrentException any exceptions.
+///
+/// SCOPE_EXIT_MEMORY_SAFE() can be used when the error can be ignored, and in
+/// addition to SCOPE_EXIT_SAFE() it will also lock MEMORY_LIMIT_EXCEEDED to
+/// avoid such exceptions.
+///
+/// It does exists as a separate helper, since you do not need to lock
+/// MEMORY_LIMIT_EXCEEDED always (there are cases when code under SCOPE_EXIT does
+/// not do any allocations, while LockExceptionInThread increment atomic
+/// variable).
+///
+/// NOTE: it should be used with triple caution.
+#define SCOPE_EXIT_MEMORY_SAFE(...) SCOPE_EXIT( \
+ try \
+ { \
+ MemoryTracker::LockExceptionInThread \
+ lock_memory_tracker(VariableContext::Global); \
+ __VA_ARGS__; \
+ } \
+ catch (...) \
+ { \
+ tryLogCurrentException(__PRETTY_FUNCTION__); \
+ } \
+)
diff --git a/base/mysqlxx/Pool.h b/base/mysqlxx/Pool.h
index b6189663f55..530e2c78cf2 100644
--- a/base/mysqlxx/Pool.h
+++ b/base/mysqlxx/Pool.h
@@ -159,9 +159,9 @@ public:
*/
Pool(const std::string & db_,
const std::string & server_,
- const std::string & user_ = "",
- const std::string & password_ = "",
- unsigned port_ = 0,
+ const std::string & user_,
+ const std::string & password_,
+ unsigned port_,
const std::string & socket_ = "",
unsigned connect_timeout_ = MYSQLXX_DEFAULT_TIMEOUT,
unsigned rw_timeout_ = MYSQLXX_DEFAULT_RW_TIMEOUT,
diff --git a/base/mysqlxx/PoolWithFailover.cpp b/base/mysqlxx/PoolWithFailover.cpp
index 5e9f70f4ac1..ea2d060e596 100644
--- a/base/mysqlxx/PoolWithFailover.cpp
+++ b/base/mysqlxx/PoolWithFailover.cpp
@@ -2,7 +2,6 @@
#include
#include
#include
-
#include
@@ -15,9 +14,12 @@ static bool startsWith(const std::string & s, const char * prefix)
using namespace mysqlxx;
-PoolWithFailover::PoolWithFailover(const Poco::Util::AbstractConfiguration & config_,
- const std::string & config_name_, const unsigned default_connections_,
- const unsigned max_connections_, const size_t max_tries_)
+PoolWithFailover::PoolWithFailover(
+ const Poco::Util::AbstractConfiguration & config_,
+ const std::string & config_name_,
+ const unsigned default_connections_,
+ const unsigned max_connections_,
+ const size_t max_tries_)
: max_tries(max_tries_)
{
shareable = config_.getBool(config_name_ + ".share_connection", false);
@@ -59,16 +61,38 @@ PoolWithFailover::PoolWithFailover(const Poco::Util::AbstractConfiguration & con
}
}
-PoolWithFailover::PoolWithFailover(const std::string & config_name_, const unsigned default_connections_,
- const unsigned max_connections_, const size_t max_tries_)
- : PoolWithFailover{
- Poco::Util::Application::instance().config(), config_name_,
- default_connections_, max_connections_, max_tries_}
+
+PoolWithFailover::PoolWithFailover(
+ const std::string & config_name_,
+ const unsigned default_connections_,
+ const unsigned max_connections_,
+ const size_t max_tries_)
+ : PoolWithFailover{Poco::Util::Application::instance().config(),
+ config_name_, default_connections_, max_connections_, max_tries_}
{
}
+
+PoolWithFailover::PoolWithFailover(
+ const std::string & database,
+ const RemoteDescription & addresses,
+ const std::string & user,
+ const std::string & password,
+ size_t max_tries_)
+ : max_tries(max_tries_)
+ , shareable(false)
+{
+ /// Replicas have the same priority, but traversed replicas are moved to the end of the queue.
+ for (const auto & [host, port] : addresses)
+ {
+ replicas_by_priority[0].emplace_back(std::make_shared(database, host, user, password, port));
+ }
+}
+
+
PoolWithFailover::PoolWithFailover(const PoolWithFailover & other)
- : max_tries{other.max_tries}, shareable{other.shareable}
+ : max_tries{other.max_tries}
+ , shareable{other.shareable}
{
if (shareable)
{
diff --git a/base/mysqlxx/PoolWithFailover.h b/base/mysqlxx/PoolWithFailover.h
index 029fc3ebad3..5154fc3e253 100644
--- a/base/mysqlxx/PoolWithFailover.h
+++ b/base/mysqlxx/PoolWithFailover.h
@@ -11,6 +11,8 @@
namespace mysqlxx
{
/** MySQL connection pool with support of failover.
+ *
+ * For dictionary source:
* Have information about replicas and their priorities.
* Tries to connect to replica in an order of priority. When equal priority, choose replica with maximum time without connections.
*
@@ -68,42 +70,58 @@ namespace mysqlxx
using PoolPtr = std::shared_ptr;
using Replicas = std::vector;
- /// [priority][index] -> replica.
+ /// [priority][index] -> replica. Highest priority is 0.
using ReplicasByPriority = std::map;
-
ReplicasByPriority replicas_by_priority;
/// Number of connection tries.
size_t max_tries;
/// Mutex for set of replicas.
std::mutex mutex;
-
/// Can the Pool be shared
bool shareable;
public:
using Entry = Pool::Entry;
+ using RemoteDescription = std::vector>;
/**
- * config_name Name of parameter in configuration file.
+ * * Mysql dictionary sourse related params:
+ * config_name Name of parameter in configuration file for dictionary source.
+ *
+ * * Mysql storage related parameters:
+ * replicas_description
+ *
+ * * Mutual parameters:
* default_connections Number of connection in pool to each replica at start.
* max_connections Maximum number of connections in pool to each replica.
* max_tries_ Max number of connection tries.
*/
- PoolWithFailover(const std::string & config_name_,
+ PoolWithFailover(
+ const std::string & config_name_,
unsigned default_connections_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_START_CONNECTIONS,
unsigned max_connections_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_CONNECTIONS,
size_t max_tries_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES);
- PoolWithFailover(const Poco::Util::AbstractConfiguration & config_,
+ PoolWithFailover(
+ const Poco::Util::AbstractConfiguration & config_,
const std::string & config_name_,
unsigned default_connections_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_START_CONNECTIONS,
unsigned max_connections_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_CONNECTIONS,
size_t max_tries_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES);
+ PoolWithFailover(
+ const std::string & database,
+ const RemoteDescription & addresses,
+ const std::string & user,
+ const std::string & password,
+ size_t max_tries_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES);
+
PoolWithFailover(const PoolWithFailover & other);
/** Allocates a connection to use. */
Entry get();
};
+
+ using PoolWithFailoverPtr = std::shared_ptr;
}
diff --git a/cmake/arch.cmake b/cmake/arch.cmake
index 9604ef62b31..60e0346dbbf 100644
--- a/cmake/arch.cmake
+++ b/cmake/arch.cmake
@@ -1,7 +1,7 @@
if (CMAKE_SYSTEM_PROCESSOR MATCHES "amd64|x86_64")
set (ARCH_AMD64 1)
endif ()
-if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*)")
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)")
set (ARCH_AARCH64 1)
endif ()
if (ARCH_AARCH64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "arm")
diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt
index 9d74179902d..51f4b974161 100644
--- a/cmake/autogenerated_versions.txt
+++ b/cmake/autogenerated_versions.txt
@@ -1,9 +1,9 @@
# This strings autochanged from release_lib.sh:
-SET(VERSION_REVISION 54450)
+SET(VERSION_REVISION 54451)
SET(VERSION_MAJOR 21)
-SET(VERSION_MINOR 5)
+SET(VERSION_MINOR 6)
SET(VERSION_PATCH 1)
-SET(VERSION_GITHASH 3827789b3d8fd2021952e57e5110343d26daa1a1)
-SET(VERSION_DESCRIBE v21.5.1.1-prestable)
-SET(VERSION_STRING 21.5.1.1)
+SET(VERSION_GITHASH 96fced4c3cf432fb0b401d2ab01f0c56e5f74a96)
+SET(VERSION_DESCRIBE v21.6.1.1-prestable)
+SET(VERSION_STRING 21.6.1.1)
# end of autochange
diff --git a/cmake/darwin/default_libs.cmake b/cmake/darwin/default_libs.cmake
index 79ac675f234..a6ee800d59b 100644
--- a/cmake/darwin/default_libs.cmake
+++ b/cmake/darwin/default_libs.cmake
@@ -1,11 +1,14 @@
set (DEFAULT_LIBS "-nodefaultlibs")
-if (NOT COMPILER_CLANG)
- message (FATAL_ERROR "Darwin build is supported only for Clang")
-endif ()
-
set (DEFAULT_LIBS "${DEFAULT_LIBS} ${COVERAGE_OPTION} -lc -lm -lpthread -ldl")
+if (COMPILER_GCC)
+ set (DEFAULT_LIBS "${DEFAULT_LIBS} -lgcc_eh")
+ if (ARCH_AARCH64)
+ set (DEFAULT_LIBS "${DEFAULT_LIBS} -lgcc")
+ endif ()
+endif ()
+
message(STATUS "Default libraries: ${DEFAULT_LIBS}")
set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS})
diff --git a/cmake/darwin/toolchain-aarch64.cmake b/cmake/darwin/toolchain-aarch64.cmake
new file mode 100644
index 00000000000..81398111495
--- /dev/null
+++ b/cmake/darwin/toolchain-aarch64.cmake
@@ -0,0 +1,14 @@
+set (CMAKE_SYSTEM_NAME "Darwin")
+set (CMAKE_SYSTEM_PROCESSOR "aarch64")
+set (CMAKE_C_COMPILER_TARGET "aarch64-apple-darwin")
+set (CMAKE_CXX_COMPILER_TARGET "aarch64-apple-darwin")
+set (CMAKE_ASM_COMPILER_TARGET "aarch64-apple-darwin")
+set (CMAKE_OSX_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/darwin-aarch64")
+
+set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake
+
+set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
+set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
+
+set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
+set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)
diff --git a/cmake/find/amqpcpp.cmake b/cmake/find/amqpcpp.cmake
index 4191dce26bb..e3eaaf33ddb 100644
--- a/cmake/find/amqpcpp.cmake
+++ b/cmake/find/amqpcpp.cmake
@@ -1,3 +1,8 @@
+if (OS_DARWIN AND COMPILER_GCC)
+ # AMQP-CPP requires libuv which cannot be built with GCC in macOS due to a bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=93082
+ set (ENABLE_AMQPCPP OFF CACHE INTERNAL "")
+endif()
+
option(ENABLE_AMQPCPP "Enalbe AMQP-CPP" ${ENABLE_LIBRARIES})
if (NOT ENABLE_AMQPCPP)
diff --git a/cmake/find/cassandra.cmake b/cmake/find/cassandra.cmake
index 037d6c3f131..ded25a5bf41 100644
--- a/cmake/find/cassandra.cmake
+++ b/cmake/find/cassandra.cmake
@@ -1,3 +1,8 @@
+if (OS_DARWIN AND COMPILER_GCC)
+ # Cassandra requires libuv which cannot be built with GCC in macOS due to a bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=93082
+ set (ENABLE_CASSANDRA OFF CACHE INTERNAL "")
+endif()
+
option(ENABLE_CASSANDRA "Enable Cassandra" ${ENABLE_LIBRARIES})
if (NOT ENABLE_CASSANDRA)
diff --git a/cmake/find/ccache.cmake b/cmake/find/ccache.cmake
index fea1f8b4c97..986c9cb5fe2 100644
--- a/cmake/find/ccache.cmake
+++ b/cmake/find/ccache.cmake
@@ -32,7 +32,9 @@ if (CCACHE_FOUND AND NOT COMPILER_MATCHES_CCACHE)
if (CCACHE_VERSION VERSION_GREATER "3.2.0" OR NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
message(STATUS "Using ${CCACHE_FOUND} ${CCACHE_VERSION}")
- set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
+ set (CMAKE_CXX_COMPILER_LAUNCHER ${CCACHE_FOUND} ${CMAKE_CXX_COMPILER_LAUNCHER})
+ set (CMAKE_C_COMPILER_LAUNCHER ${CCACHE_FOUND} ${CMAKE_C_COMPILER_LAUNCHER})
+
set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
# debian (debhelpers) set SOURCE_DATE_EPOCH environment variable, that is
diff --git a/cmake/find/ldap.cmake b/cmake/find/ldap.cmake
index 0dffa334e73..d8baea89429 100644
--- a/cmake/find/ldap.cmake
+++ b/cmake/find/ldap.cmake
@@ -64,7 +64,8 @@ if (NOT OPENLDAP_FOUND AND NOT MISSING_INTERNAL_LDAP_LIBRARY)
( "${_system_name}" STREQUAL "linux" AND "${_system_processor}" STREQUAL "aarch64" ) OR
( "${_system_name}" STREQUAL "linux" AND "${_system_processor}" STREQUAL "ppc64le" ) OR
( "${_system_name}" STREQUAL "freebsd" AND "${_system_processor}" STREQUAL "x86_64" ) OR
- ( "${_system_name}" STREQUAL "darwin" AND "${_system_processor}" STREQUAL "x86_64" )
+ ( "${_system_name}" STREQUAL "darwin" AND "${_system_processor}" STREQUAL "x86_64" ) OR
+ ( "${_system_name}" STREQUAL "darwin" AND "${_system_processor}" STREQUAL "aarch64" )
)
set (_ldap_supported_platform TRUE)
endif ()
diff --git a/cmake/find/nanodbc.cmake b/cmake/find/nanodbc.cmake
new file mode 100644
index 00000000000..894a2a60bad
--- /dev/null
+++ b/cmake/find/nanodbc.cmake
@@ -0,0 +1,16 @@
+if (NOT ENABLE_ODBC)
+ return ()
+endif ()
+
+if (NOT USE_INTERNAL_NANODBC_LIBRARY)
+ message (FATAL_ERROR "Only the bundled nanodbc library can be used")
+endif ()
+
+if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/nanodbc/CMakeLists.txt")
+ message (FATAL_ERROR "submodule contrib/nanodbc is missing. to fix try run: \n git submodule update --init --recursive")
+endif()
+
+set (NANODBC_LIBRARY nanodbc)
+set (NANODBC_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/nanodbc/nanodbc")
+
+message (STATUS "Using nanodbc: ${NANODBC_INCLUDE_DIR} : ${NANODBC_LIBRARY}")
diff --git a/cmake/find/nuraft.cmake b/cmake/find/nuraft.cmake
index 7fa5251946e..4e5258e132f 100644
--- a/cmake/find/nuraft.cmake
+++ b/cmake/find/nuraft.cmake
@@ -11,7 +11,7 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/NuRaft/CMakeLists.txt")
return()
endif ()
-if (NOT OS_FREEBSD AND NOT OS_DARWIN)
+if (NOT OS_FREEBSD)
set (USE_NURAFT 1)
set (NURAFT_LIBRARY nuraft)
diff --git a/cmake/find/odbc.cmake b/cmake/find/odbc.cmake
index a23f0c831e9..c475e600c0d 100644
--- a/cmake/find/odbc.cmake
+++ b/cmake/find/odbc.cmake
@@ -50,4 +50,6 @@ if (NOT EXTERNAL_ODBC_LIBRARY_FOUND)
set (USE_INTERNAL_ODBC_LIBRARY 1)
endif ()
+set (USE_INTERNAL_NANODBC_LIBRARY 1)
+
message (STATUS "Using unixodbc")
diff --git a/cmake/find/rocksdb.cmake b/cmake/find/rocksdb.cmake
index 968cdb52407..94278a603d7 100644
--- a/cmake/find/rocksdb.cmake
+++ b/cmake/find/rocksdb.cmake
@@ -1,3 +1,7 @@
+if (OS_DARWIN AND ARCH_AARCH64)
+ set (ENABLE_ROCKSDB OFF CACHE INTERNAL "")
+endif()
+
option(ENABLE_ROCKSDB "Enable ROCKSDB" ${ENABLE_LIBRARIES})
if (NOT ENABLE_ROCKSDB)
diff --git a/cmake/find/xz.cmake b/cmake/find/xz.cmake
new file mode 100644
index 00000000000..0d19859c6b1
--- /dev/null
+++ b/cmake/find/xz.cmake
@@ -0,0 +1,27 @@
+option (USE_INTERNAL_XZ_LIBRARY "Set to OFF to use system xz (lzma) library instead of bundled" ${NOT_UNBUNDLED})
+
+if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/xz/src/liblzma/api/lzma.h")
+ if(USE_INTERNAL_XZ_LIBRARY)
+ message(WARNING "submodule contrib/xz is missing. to fix try run: \n git submodule update --init --recursive")
+ message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal xz (lzma) library")
+ set(USE_INTERNAL_XZ_LIBRARY 0)
+ endif()
+ set(MISSING_INTERNAL_XZ_LIBRARY 1)
+endif()
+
+if (NOT USE_INTERNAL_XZ_LIBRARY)
+ find_library (XZ_LIBRARY lzma)
+ find_path (XZ_INCLUDE_DIR NAMES lzma.h PATHS ${XZ_INCLUDE_PATHS})
+ if (NOT XZ_LIBRARY OR NOT XZ_INCLUDE_DIR)
+ message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system xz (lzma) library")
+ endif ()
+endif ()
+
+if (XZ_LIBRARY AND XZ_INCLUDE_DIR)
+elseif (NOT MISSING_INTERNAL_XZ_LIBRARY)
+ set (USE_INTERNAL_XZ_LIBRARY 1)
+ set (XZ_LIBRARY liblzma)
+ set (XZ_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/xz/src/liblzma/api)
+endif ()
+
+message (STATUS "Using xz (lzma): ${XZ_INCLUDE_DIR} : ${XZ_LIBRARY}")
diff --git a/cmake/warnings.cmake b/cmake/warnings.cmake
index a398c59e981..a85fe8963c7 100644
--- a/cmake/warnings.cmake
+++ b/cmake/warnings.cmake
@@ -171,6 +171,7 @@ elseif (COMPILER_GCC)
add_cxx_compile_options(-Wtrampolines)
# Obvious
add_cxx_compile_options(-Wunused)
+ add_cxx_compile_options(-Wundef)
# Warn if vector operation is not implemented via SIMD capabilities of the architecture
add_cxx_compile_options(-Wvector-operation-performance)
# XXX: libstdc++ has some of these for 3way compare
diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 4aeb67a5085..087212ad3b0 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -47,7 +47,11 @@ add_subdirectory (lz4-cmake)
add_subdirectory (murmurhash)
add_subdirectory (replxx-cmake)
add_subdirectory (unixodbc-cmake)
-add_subdirectory (xz)
+add_subdirectory (nanodbc-cmake)
+
+if (USE_INTERNAL_XZ_LIBRARY)
+ add_subdirectory (xz)
+endif()
add_subdirectory (poco-cmake)
add_subdirectory (croaring-cmake)
@@ -93,14 +97,8 @@ if (USE_INTERNAL_ZLIB_LIBRARY)
add_subdirectory (${INTERNAL_ZLIB_NAME})
# We should use same defines when including zlib.h as used when zlib compiled
target_compile_definitions (zlib PUBLIC ZLIB_COMPAT WITH_GZFILEOP)
- if (TARGET zlibstatic)
- target_compile_definitions (zlibstatic PUBLIC ZLIB_COMPAT WITH_GZFILEOP)
- endif ()
if (ARCH_AMD64 OR ARCH_AARCH64)
target_compile_definitions (zlib PUBLIC X86_64 UNALIGNED_OK)
- if (TARGET zlibstatic)
- target_compile_definitions (zlibstatic PUBLIC X86_64 UNALIGNED_OK)
- endif ()
endif ()
endif ()
diff --git a/contrib/NuRaft b/contrib/NuRaft
index 70468326ad5..377f8e77491 160000
--- a/contrib/NuRaft
+++ b/contrib/NuRaft
@@ -1 +1 @@
-Subproject commit 70468326ad5d72e9497944838484c591dae054ea
+Subproject commit 377f8e77491d9f66ce8e32e88aae19dffe8dc4d7
diff --git a/contrib/antlr4-runtime b/contrib/antlr4-runtime
index a2fa7b76e2e..672643e9a42 160000
--- a/contrib/antlr4-runtime
+++ b/contrib/antlr4-runtime
@@ -1 +1 @@
-Subproject commit a2fa7b76e2ee16d2ad955e9214a90bbf79da66fc
+Subproject commit 672643e9a427ef803abf13bc8cb4989606553d64
diff --git a/contrib/boost b/contrib/boost
index ee24fa55bc4..a8d43d3142c 160000
--- a/contrib/boost
+++ b/contrib/boost
@@ -1 +1 @@
-Subproject commit ee24fa55bc46e4d2ce7d0d052cc5a0d9b1be8c36
+Subproject commit a8d43d3142cc6b26fc55bec33f7f6edb1156ab7a
diff --git a/contrib/boringssl b/contrib/boringssl
index fd9ce1a0406..83c1cda8a02 160000
--- a/contrib/boringssl
+++ b/contrib/boringssl
@@ -1 +1 @@
-Subproject commit fd9ce1a0406f571507068b9555d0b545b8a18332
+Subproject commit 83c1cda8a0224dc817cbad2966c7ed4acc35f02a
diff --git a/contrib/boringssl-cmake/CMakeLists.txt b/contrib/boringssl-cmake/CMakeLists.txt
index 017a8a64c0e..adfee82dda4 100644
--- a/contrib/boringssl-cmake/CMakeLists.txt
+++ b/contrib/boringssl-cmake/CMakeLists.txt
@@ -16,7 +16,7 @@ endif()
if(CMAKE_COMPILER_IS_GNUCXX OR CLANG)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -fvisibility=hidden -fno-common -fno-exceptions -fno-rtti")
- if(APPLE)
+ if(APPLE AND CLANG)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
endif()
diff --git a/contrib/flatbuffers b/contrib/flatbuffers
index 6df40a24717..22e3ffc66d2 160000
--- a/contrib/flatbuffers
+++ b/contrib/flatbuffers
@@ -1 +1 @@
-Subproject commit 6df40a2471737b27271bdd9b900ab5f3aec746c7
+Subproject commit 22e3ffc66d2d7d72d1414390aa0f04ffd114a5a1
diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt
index 73afa99f1d8..f8cab3e548c 100644
--- a/contrib/jemalloc-cmake/CMakeLists.txt
+++ b/contrib/jemalloc-cmake/CMakeLists.txt
@@ -1,10 +1,13 @@
-if (SANITIZE OR NOT (ARCH_AMD64 OR ARCH_ARM OR ARCH_PPC64LE) OR NOT (OS_LINUX OR OS_FREEBSD OR OS_DARWIN))
+if (SANITIZE OR NOT (
+ ((OS_LINUX OR OS_FREEBSD) AND (ARCH_AMD64 OR ARCH_ARM OR ARCH_PPC64LE)) OR
+ (OS_DARWIN AND CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
+))
if (ENABLE_JEMALLOC)
message (${RECONFIGURE_MESSAGE_LEVEL}
- "jemalloc is disabled implicitly: it doesn't work with sanitizers and can only be used with x86_64, aarch64 or ppc64le on linux or freebsd.")
- endif()
+ "jemalloc is disabled implicitly: it doesn't work with sanitizers and can only be used with x86_64, aarch64, or ppc64le Linux or FreeBSD builds and RelWithDebInfo macOS builds.")
+ endif ()
set (ENABLE_JEMALLOC OFF)
-else()
+else ()
option (ENABLE_JEMALLOC "Enable jemalloc allocator" ${ENABLE_LIBRARIES})
endif ()
@@ -34,9 +37,9 @@ if (OS_LINUX)
# avoid spurious latencies and additional work associated with
# MADV_DONTNEED. See
# https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
- set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:10000")
+ set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000")
else()
- set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:10000")
+ set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000")
endif()
# CACHE variable is empty, to allow changing defaults without necessity
# to purge cache
@@ -121,12 +124,14 @@ target_include_directories(jemalloc SYSTEM PRIVATE
target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE)
if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
- target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_DEBUG=1 -DJEMALLOC_PROF=1)
+ target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_DEBUG=1)
+endif ()
- if (USE_UNWIND)
- target_compile_definitions (jemalloc PRIVATE -DJEMALLOC_PROF_LIBUNWIND=1)
- target_link_libraries (jemalloc PRIVATE unwind)
- endif ()
+target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_PROF=1)
+
+if (USE_UNWIND)
+ target_compile_definitions (jemalloc PRIVATE -DJEMALLOC_PROF_LIBUNWIND=1)
+ target_link_libraries (jemalloc PRIVATE unwind)
endif ()
target_compile_options(jemalloc PRIVATE -Wno-redundant-decls)
diff --git a/contrib/jemalloc-cmake/include_darwin_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_darwin_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in
index c7c884d0eaa..5c0407db24a 100644
--- a/contrib/jemalloc-cmake/include_darwin_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in
+++ b/contrib/jemalloc-cmake/include_darwin_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in
@@ -42,7 +42,7 @@
* total number of bits in a pointer, e.g. on x64, for which the uppermost 16
* bits are the same as bit 47.
*/
-#define LG_VADDR 48
+#define LG_VADDR 64
/* Defined if C11 atomics are available. */
#define JEMALLOC_C11_ATOMICS 1
@@ -101,11 +101,6 @@
*/
#define JEMALLOC_HAVE_MACH_ABSOLUTE_TIME 1
-/*
- * Defined if clock_gettime(CLOCK_REALTIME, ...) is available.
- */
-#define JEMALLOC_HAVE_CLOCK_REALTIME 1
-
/*
* Defined if _malloc_thread_cleanup() exists. At least in the case of
* FreeBSD, pthread_key_create() allocates, which if used during malloc
@@ -181,14 +176,14 @@
/* #undef LG_QUANTUM */
/* One page is 2^LG_PAGE bytes. */
-#define LG_PAGE 16
+#define LG_PAGE 14
/*
* One huge page is 2^LG_HUGEPAGE bytes. Note that this is defined even if the
* system does not explicitly support huge pages; system calls that require
* explicit huge page support are separately configured.
*/
-#define LG_HUGEPAGE 29
+#define LG_HUGEPAGE 21
/*
* If defined, adjacent virtual memory mappings with identical attributes
@@ -356,7 +351,7 @@
/* #undef JEMALLOC_EXPORT */
/* config.malloc_conf options string. */
-#define JEMALLOC_CONFIG_MALLOC_CONF "@JEMALLOC_CONFIG_MALLOC_CONF@"
+#define JEMALLOC_CONFIG_MALLOC_CONF ""
/* If defined, jemalloc takes the malloc/free/etc. symbol names. */
/* #undef JEMALLOC_IS_MALLOC */
diff --git a/contrib/libcxx b/contrib/libcxx
index 8b80a151d12..2fa892f69ac 160000
--- a/contrib/libcxx
+++ b/contrib/libcxx
@@ -1 +1 @@
-Subproject commit 8b80a151d12b98ffe2d0c22f7cec12c3b9ff88d7
+Subproject commit 2fa892f69acbaa40f8a18c6484854a6183a34482
diff --git a/contrib/libcxx-cmake/CMakeLists.txt b/contrib/libcxx-cmake/CMakeLists.txt
index 3b5d53cd1c0..59d23b2cd9e 100644
--- a/contrib/libcxx-cmake/CMakeLists.txt
+++ b/contrib/libcxx-cmake/CMakeLists.txt
@@ -56,6 +56,11 @@ if (USE_UNWIND)
target_compile_definitions(cxx PUBLIC -DSTD_EXCEPTION_HAS_STACK_TRACE=1)
endif ()
+# Override the deduced attribute support that causes error.
+if (OS_DARWIN AND COMPILER_GCC)
+ add_compile_definitions(_LIBCPP_INIT_PRIORITY_MAX)
+endif ()
+
target_compile_options(cxx PUBLIC $<$:-nostdinc++>)
# Third party library may have substandard code.
diff --git a/contrib/librdkafka-cmake/config.h.in b/contrib/librdkafka-cmake/config.h.in
index 80b6ea61b6e..9fecb45e42d 100644
--- a/contrib/librdkafka-cmake/config.h.in
+++ b/contrib/librdkafka-cmake/config.h.in
@@ -66,7 +66,7 @@
#cmakedefine WITH_SASL_OAUTHBEARER 1
#cmakedefine WITH_SASL_CYRUS 1
// crc32chw
-#if !defined(__PPC__) && (!defined(__aarch64__) || defined(__ARM_FEATURE_CRC32))
+#if !defined(__PPC__) && (!defined(__aarch64__) || defined(__ARM_FEATURE_CRC32)) && !(defined(__aarch64__) && defined(__APPLE__))
#define WITH_CRC32C_HW 1
#endif
// regex
@@ -75,6 +75,8 @@
#define HAVE_STRNDUP 1
// strerror_r
#define HAVE_STRERROR_R 1
+// rand_r
+#define HAVE_RAND_R 1
#ifdef __APPLE__
// pthread_setname_np
diff --git a/contrib/mariadb-connector-c b/contrib/mariadb-connector-c
index f4476ee7311..5f4034a3a63 160000
--- a/contrib/mariadb-connector-c
+++ b/contrib/mariadb-connector-c
@@ -1 +1 @@
-Subproject commit f4476ee7311b35b593750f6ae2cbdb62a4006374
+Subproject commit 5f4034a3a6376416504f17186c55fe401c6d8e5e
diff --git a/contrib/nanodbc b/contrib/nanodbc
new file mode 160000
index 00000000000..9fc45967551
--- /dev/null
+++ b/contrib/nanodbc
@@ -0,0 +1 @@
+Subproject commit 9fc459675515d491401727ec67fca38db721f28c
diff --git a/contrib/nanodbc-cmake/CMakeLists.txt b/contrib/nanodbc-cmake/CMakeLists.txt
new file mode 100644
index 00000000000..1673b311c49
--- /dev/null
+++ b/contrib/nanodbc-cmake/CMakeLists.txt
@@ -0,0 +1,18 @@
+if (NOT USE_INTERNAL_NANODBC_LIBRARY)
+ return ()
+endif ()
+
+set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/nanodbc)
+
+if (NOT TARGET unixodbc)
+ message(FATAL_ERROR "Configuration error: unixodbc is not a target")
+endif()
+
+set (SRCS
+ ${LIBRARY_DIR}/nanodbc/nanodbc.cpp
+)
+
+add_library(nanodbc ${SRCS})
+
+target_link_libraries (nanodbc PUBLIC unixodbc)
+target_include_directories (nanodbc SYSTEM PUBLIC ${LIBRARY_DIR}/)
diff --git a/contrib/openldap-cmake/darwin_aarch64/include/lber_types.h b/contrib/openldap-cmake/darwin_aarch64/include/lber_types.h
new file mode 100644
index 00000000000..dbd59430527
--- /dev/null
+++ b/contrib/openldap-cmake/darwin_aarch64/include/lber_types.h
@@ -0,0 +1,63 @@
+/* include/lber_types.h. Generated from lber_types.hin by configure. */
+/* $OpenLDAP$ */
+/* This work is part of OpenLDAP Software .
+ *
+ * Copyright 1998-2020 The OpenLDAP Foundation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * .
+ */
+
+/*
+ * LBER types
+ */
+
+#ifndef _LBER_TYPES_H
+#define _LBER_TYPES_H
+
+#include
+
+LDAP_BEGIN_DECL
+
+/* LBER boolean, enum, integers (32 bits or larger) */
+#define LBER_INT_T int
+
+/* LBER tags (32 bits or larger) */
+#define LBER_TAG_T long
+
+/* LBER socket descriptor */
+#define LBER_SOCKET_T int
+
+/* LBER lengths (32 bits or larger) */
+#define LBER_LEN_T long
+
+/* ------------------------------------------------------------ */
+
+/* booleans, enumerations, and integers */
+typedef LBER_INT_T ber_int_t;
+
+/* signed and unsigned versions */
+typedef signed LBER_INT_T ber_sint_t;
+typedef unsigned LBER_INT_T ber_uint_t;
+
+/* tags */
+typedef unsigned LBER_TAG_T ber_tag_t;
+
+/* "socket" descriptors */
+typedef LBER_SOCKET_T ber_socket_t;
+
+/* lengths */
+typedef unsigned LBER_LEN_T ber_len_t;
+
+/* signed lengths */
+typedef signed LBER_LEN_T ber_slen_t;
+
+LDAP_END_DECL
+
+#endif /* _LBER_TYPES_H */
diff --git a/contrib/openldap-cmake/darwin_aarch64/include/ldap_config.h b/contrib/openldap-cmake/darwin_aarch64/include/ldap_config.h
new file mode 100644
index 00000000000..89f7b40b884
--- /dev/null
+++ b/contrib/openldap-cmake/darwin_aarch64/include/ldap_config.h
@@ -0,0 +1,74 @@
+/* include/ldap_config.h. Generated from ldap_config.hin by configure. */
+/* $OpenLDAP$ */
+/* This work is part of OpenLDAP Software .
+ *
+ * Copyright 1998-2020 The OpenLDAP Foundation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * .
+ */
+
+/*
+ * This file works in conjunction with OpenLDAP configure system.
+ * If you do no like the values below, adjust your configure options.
+ */
+
+#ifndef _LDAP_CONFIG_H
+#define _LDAP_CONFIG_H
+
+/* directory separator */
+#ifndef LDAP_DIRSEP
+#ifndef _WIN32
+#define LDAP_DIRSEP "/"
+#else
+#define LDAP_DIRSEP "\\"
+#endif
+#endif
+
+/* directory for temporary files */
+#if defined(_WIN32)
+# define LDAP_TMPDIR "C:\\." /* we don't have much of a choice */
+#elif defined( _P_tmpdir )
+# define LDAP_TMPDIR _P_tmpdir
+#elif defined( P_tmpdir )
+# define LDAP_TMPDIR P_tmpdir
+#elif defined( _PATH_TMPDIR )
+# define LDAP_TMPDIR _PATH_TMPDIR
+#else
+# define LDAP_TMPDIR LDAP_DIRSEP "tmp"
+#endif
+
+/* directories */
+#ifndef LDAP_BINDIR
+#define LDAP_BINDIR "/tmp/ldap-prefix/bin"
+#endif
+#ifndef LDAP_SBINDIR
+#define LDAP_SBINDIR "/tmp/ldap-prefix/sbin"
+#endif
+#ifndef LDAP_DATADIR
+#define LDAP_DATADIR "/tmp/ldap-prefix/share/openldap"
+#endif
+#ifndef LDAP_SYSCONFDIR
+#define LDAP_SYSCONFDIR "/tmp/ldap-prefix/etc/openldap"
+#endif
+#ifndef LDAP_LIBEXECDIR
+#define LDAP_LIBEXECDIR "/tmp/ldap-prefix/libexec"
+#endif
+#ifndef LDAP_MODULEDIR
+#define LDAP_MODULEDIR "/tmp/ldap-prefix/libexec/openldap"
+#endif
+#ifndef LDAP_RUNDIR
+#define LDAP_RUNDIR "/tmp/ldap-prefix/var"
+#endif
+#ifndef LDAP_LOCALEDIR
+#define LDAP_LOCALEDIR ""
+#endif
+
+
+#endif /* _LDAP_CONFIG_H */
diff --git a/contrib/openldap-cmake/darwin_aarch64/include/ldap_features.h b/contrib/openldap-cmake/darwin_aarch64/include/ldap_features.h
new file mode 100644
index 00000000000..f0cc7c3626f
--- /dev/null
+++ b/contrib/openldap-cmake/darwin_aarch64/include/ldap_features.h
@@ -0,0 +1,61 @@
+/* include/ldap_features.h. Generated from ldap_features.hin by configure. */
+/* $OpenLDAP$ */
+/* This work is part of OpenLDAP Software .
+ *
+ * Copyright 1998-2020 The OpenLDAP Foundation.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * .
+ */
+
+/*
+ * LDAP Features
+ */
+
+#ifndef _LDAP_FEATURES_H
+#define _LDAP_FEATURES_H 1
+
+/* OpenLDAP API version macros */
+#define LDAP_VENDOR_VERSION 20501
+#define LDAP_VENDOR_VERSION_MAJOR 2
+#define LDAP_VENDOR_VERSION_MINOR 5
+#define LDAP_VENDOR_VERSION_PATCH X
+
+/*
+** WORK IN PROGRESS!
+**
+** OpenLDAP reentrancy/thread-safeness should be dynamically
+** checked using ldap_get_option().
+**
+** The -lldap implementation is not thread-safe.
+**
+** The -lldap_r implementation is:
+** LDAP_API_FEATURE_THREAD_SAFE (basic thread safety)
+** but also be:
+** LDAP_API_FEATURE_SESSION_THREAD_SAFE
+** LDAP_API_FEATURE_OPERATION_THREAD_SAFE
+**
+** The preprocessor flag LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE
+** can be used to determine if -lldap_r is available at compile
+** time. You must define LDAP_THREAD_SAFE if and only if you
+** link with -lldap_r.
+**
+** If you fail to define LDAP_THREAD_SAFE when linking with
+** -lldap_r or define LDAP_THREAD_SAFE when linking with -lldap,
+** provided header definitions and declarations may be incorrect.
+**
+*/
+
+/* is -lldap_r available or not */
+#define LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE 1
+
+/* LDAP v2 Referrals */
+/* #undef LDAP_API_FEATURE_X_OPENLDAP_V2_REFERRALS */
+
+#endif /* LDAP_FEATURES */
diff --git a/contrib/openldap-cmake/darwin_aarch64/include/portable.h b/contrib/openldap-cmake/darwin_aarch64/include/portable.h
new file mode 100644
index 00000000000..fdf4e89017e
--- /dev/null
+++ b/contrib/openldap-cmake/darwin_aarch64/include/portable.h
@@ -0,0 +1,1169 @@
+/* include/portable.h. Generated from portable.hin by configure. */
+/* include/portable.hin. Generated from configure.in by autoheader. */
+
+
+/* begin of portable.h.pre */
+/* This work is part of OpenLDAP Software .
+ *
+ * Copyright 1998-2020 The OpenLDAP Foundation
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the OpenLDAP
+ * Public License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * top-level directory of the distribution or, alternatively, at
+ * .
+ */
+
+#ifndef _LDAP_PORTABLE_H
+#define _LDAP_PORTABLE_H
+
+/* define this if needed to get reentrant functions */
+#ifndef REENTRANT
+#define REENTRANT 1
+#endif
+#ifndef _REENTRANT
+#define _REENTRANT 1
+#endif
+
+/* define this if needed to get threadsafe functions */
+#ifndef THREADSAFE
+#define THREADSAFE 1
+#endif
+#ifndef _THREADSAFE
+#define _THREADSAFE 1
+#endif
+#ifndef THREAD_SAFE
+#define THREAD_SAFE 1
+#endif
+#ifndef _THREAD_SAFE
+#define _THREAD_SAFE 1
+#endif
+
+#ifndef _SGI_MP_SOURCE
+#define _SGI_MP_SOURCE 1
+#endif
+
+/* end of portable.h.pre */
+
+
+/* Define if building universal (internal helper macro) */
+/* #undef AC_APPLE_UNIVERSAL_BUILD */
+
+/* define to use both and */
+/* #undef BOTH_STRINGS_H */
+
+/* define if cross compiling */
+/* #undef CROSS_COMPILING */
+
+/* set to the number of arguments ctime_r() expects */
+#define CTIME_R_NARGS 2
+
+/* define if toupper() requires islower() */
+/* #undef C_UPPER_LOWER */
+
+/* define if sys_errlist is not declared in stdio.h or errno.h */
+/* #undef DECL_SYS_ERRLIST */
+
+/* define to enable slapi library */
+/* #undef ENABLE_SLAPI */
+
+/* defined to be the EXE extension */
+#define EXEEXT ""
+
+/* set to the number of arguments gethostbyaddr_r() expects */
+/* #undef GETHOSTBYADDR_R_NARGS */
+
+/* set to the number of arguments gethostbyname_r() expects */
+/* #undef GETHOSTBYNAME_R_NARGS */
+
+/* Define to 1 if `TIOCGWINSZ' requires . */
+/* #undef GWINSZ_IN_SYS_IOCTL */
+
+/* define if you have AIX security lib */
+/* #undef HAVE_AIX_SECURITY */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_ARPA_INET_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_ARPA_NAMESER_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_ASSERT_H 1
+
+/* Define to 1 if you have the `bcopy' function. */
+#define HAVE_BCOPY 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_BITS_TYPES_H */
+
+/* Define to 1 if you have the `chroot' function. */
+#define HAVE_CHROOT 1
+
+/* Define to 1 if you have the `closesocket' function. */
+/* #undef HAVE_CLOSESOCKET */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_CONIO_H */
+
+/* define if crypt(3) is available */
+/* #undef HAVE_CRYPT */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_CRYPT_H */
+
+/* define if crypt_r() is also available */
+/* #undef HAVE_CRYPT_R */
+
+/* Define to 1 if you have the `ctime_r' function. */
+#define HAVE_CTIME_R 1
+
+/* define if you have Cyrus SASL */
+/* #undef HAVE_CYRUS_SASL */
+
+/* define if your system supports /dev/poll */
+/* #undef HAVE_DEVPOLL */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_DIRECT_H */
+
+/* Define to 1 if you have the header file, and it defines `DIR'.
+ */
+#define HAVE_DIRENT_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_DLFCN_H 1
+
+/* Define to 1 if you don't have `vprintf' but do have `_doprnt.' */
+/* #undef HAVE_DOPRNT */
+
+/* define if system uses EBCDIC instead of ASCII */
+/* #undef HAVE_EBCDIC */
+
+/* Define to 1 if you have the `endgrent' function. */
+#define HAVE_ENDGRENT 1
+
+/* Define to 1 if you have the `endpwent' function. */
+#define HAVE_ENDPWENT 1
+
+/* define if your system supports epoll */
+/* #undef HAVE_EPOLL */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_ERRNO_H 1
+
+/* Define to 1 if you have the `fcntl' function. */
+#define HAVE_FCNTL 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_FCNTL_H 1
+
+/* define if you actually have FreeBSD fetch(3) */
+/* #undef HAVE_FETCH */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_FILIO_H */
+
+/* Define to 1 if you have the `flock' function. */
+#define HAVE_FLOCK 1
+
+/* Define to 1 if you have the `fstat' function. */
+#define HAVE_FSTAT 1
+
+/* Define to 1 if you have the `gai_strerror' function. */
+#define HAVE_GAI_STRERROR 1
+
+/* Define to 1 if you have the `getaddrinfo' function. */
+#define HAVE_GETADDRINFO 1
+
+/* Define to 1 if you have the `getdtablesize' function. */
+#define HAVE_GETDTABLESIZE 1
+
+/* Define to 1 if you have the `geteuid' function. */
+#define HAVE_GETEUID 1
+
+/* Define to 1 if you have the `getgrgid' function. */
+#define HAVE_GETGRGID 1
+
+/* Define to 1 if you have the `gethostbyaddr_r' function. */
+/* #undef HAVE_GETHOSTBYADDR_R */
+
+/* Define to 1 if you have the `gethostbyname_r' function. */
+/* #undef HAVE_GETHOSTBYNAME_R */
+
+/* Define to 1 if you have the `gethostname' function. */
+#define HAVE_GETHOSTNAME 1
+
+/* Define to 1 if you have the `getnameinfo' function. */
+#define HAVE_GETNAMEINFO 1
+
+/* Define to 1 if you have the `getopt' function. */
+#define HAVE_GETOPT 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_GETOPT_H 1
+
+/* Define to 1 if you have the `getpassphrase' function. */
+/* #undef HAVE_GETPASSPHRASE */
+
+/* Define to 1 if you have the `getpeereid' function. */
+#define HAVE_GETPEEREID 1
+
+/* Define to 1 if you have the `getpeerucred' function. */
+/* #undef HAVE_GETPEERUCRED */
+
+/* Define to 1 if you have the `getpwnam' function. */
+#define HAVE_GETPWNAM 1
+
+/* Define to 1 if you have the `getpwuid' function. */
+#define HAVE_GETPWUID 1
+
+/* Define to 1 if you have the `getspnam' function. */
+/* #undef HAVE_GETSPNAM */
+
+/* Define to 1 if you have the `gettimeofday' function. */
+#define HAVE_GETTIMEOFDAY 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_GMP_H */
+
+/* Define to 1 if you have the `gmtime_r' function. */
+#define HAVE_GMTIME_R 1
+
+/* define if you have GNUtls */
+/* #undef HAVE_GNUTLS */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_GNUTLS_GNUTLS_H */
+
+/* if you have GNU Pth */
+/* #undef HAVE_GNU_PTH */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_GRP_H 1
+
+/* Define to 1 if you have the `hstrerror' function. */
+#define HAVE_HSTRERROR 1
+
+/* define to you inet_aton(3) is available */
+#define HAVE_INET_ATON 1
+
+/* Define to 1 if you have the `inet_ntoa_b' function. */
+/* #undef HAVE_INET_NTOA_B */
+
+/* Define to 1 if you have the `inet_ntop' function. */
+#define HAVE_INET_NTOP 1
+
+/* Define to 1 if you have the `initgroups' function. */
+#define HAVE_INITGROUPS 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_INTTYPES_H 1
+
+/* Define to 1 if you have the `ioctl' function. */
+#define HAVE_IOCTL 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_IO_H */
+
+/* define if your system supports kqueue */
+#define HAVE_KQUEUE 1
+
+/* Define to 1 if you have the `gen' library (-lgen). */
+/* #undef HAVE_LIBGEN */
+
+/* Define to 1 if you have the `gmp' library (-lgmp). */
+/* #undef HAVE_LIBGMP */
+
+/* Define to 1 if you have the `inet' library (-linet). */
+/* #undef HAVE_LIBINET */
+
+/* define if you have libtool -ltdl */
+/* #undef HAVE_LIBLTDL */
+
+/* Define to 1 if you have the `net' library (-lnet). */
+/* #undef HAVE_LIBNET */
+
+/* Define to 1 if you have the `nsl' library (-lnsl). */
+/* #undef HAVE_LIBNSL */
+
+/* Define to 1 if you have the `nsl_s' library (-lnsl_s). */
+/* #undef HAVE_LIBNSL_S */
+
+/* Define to 1 if you have the `socket' library (-lsocket). */
+/* #undef HAVE_LIBSOCKET */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_LIBUTIL_H */
+
+/* Define to 1 if you have the `V3' library (-lV3). */
+/* #undef HAVE_LIBV3 */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_LIMITS_H 1
+
+/* if you have LinuxThreads */
+/* #undef HAVE_LINUX_THREADS */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_LOCALE_H 1
+
+/* Define to 1 if you have the `localtime_r' function. */
+#define HAVE_LOCALTIME_R 1
+
+/* Define to 1 if you have the `lockf' function. */
+#define HAVE_LOCKF 1
+
+/* Define to 1 if the system has the type `long long'. */
+#define HAVE_LONG_LONG 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_LTDL_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_MALLOC_H */
+
+/* Define to 1 if you have the `memcpy' function. */
+#define HAVE_MEMCPY 1
+
+/* Define to 1 if you have the `memmove' function. */
+#define HAVE_MEMMOVE 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_MEMORY_H 1
+
+/* Define to 1 if you have the `memrchr' function. */
+/* #undef HAVE_MEMRCHR */
+
+/* Define to 1 if you have the `mkstemp' function. */
+#define HAVE_MKSTEMP 1
+
+/* Define to 1 if you have the `mktemp' function. */
+#define HAVE_MKTEMP 1
+
+/* define this if you have mkversion */
+#define HAVE_MKVERSION 1
+
+/* Define to 1 if you have the header file, and it defines `DIR'. */
+/* #undef HAVE_NDIR_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_NETINET_TCP_H 1
+
+/* define if strerror_r returns char* instead of int */
+/* #undef HAVE_NONPOSIX_STRERROR_R */
+
+/* if you have NT Event Log */
+/* #undef HAVE_NT_EVENT_LOG */
+
+/* if you have NT Service Manager */
+/* #undef HAVE_NT_SERVICE_MANAGER */
+
+/* if you have NT Threads */
+/* #undef HAVE_NT_THREADS */
+
+/* define if you have OpenSSL */
+#define HAVE_OPENSSL 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_BN_H 1
+
+/* define if you have OpenSSL with CRL checking capability */
+#define HAVE_OPENSSL_CRL 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_CRYPTO_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_OPENSSL_SSL_H 1
+
+/* Define to 1 if you have the `pipe' function. */
+#define HAVE_PIPE 1
+
+/* Define to 1 if you have the `poll' function. */
+#define HAVE_POLL 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_POLL_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_PROCESS_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_PSAP_H */
+
+/* define to pthreads API spec revision */
+#define HAVE_PTHREADS 10
+
+/* define if you have pthread_detach function */
+#define HAVE_PTHREAD_DETACH 1
+
+/* Define to 1 if you have the `pthread_getconcurrency' function. */
+#define HAVE_PTHREAD_GETCONCURRENCY 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_PTHREAD_H 1
+
+/* Define to 1 if you have the `pthread_kill' function. */
+#define HAVE_PTHREAD_KILL 1
+
+/* Define to 1 if you have the `pthread_kill_other_threads_np' function. */
+/* #undef HAVE_PTHREAD_KILL_OTHER_THREADS_NP */
+
+/* define if you have pthread_rwlock_destroy function */
+#define HAVE_PTHREAD_RWLOCK_DESTROY 1
+
+/* Define to 1 if you have the `pthread_setconcurrency' function. */
+#define HAVE_PTHREAD_SETCONCURRENCY 1
+
+/* Define to 1 if you have the `pthread_yield' function. */
+/* #undef HAVE_PTHREAD_YIELD */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_PTH_H */
+
+/* Define to 1 if the system has the type `ptrdiff_t'. */
+#define HAVE_PTRDIFF_T 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_PWD_H 1
+
+/* Define to 1 if you have the `read' function. */
+#define HAVE_READ 1
+
+/* Define to 1 if you have the `recv' function. */
+#define HAVE_RECV 1
+
+/* Define to 1 if you have the `recvfrom' function. */
+#define HAVE_RECVFROM 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_REGEX_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_RESOLV_H */
+
+/* define if you have res_query() */
+/* #undef HAVE_RES_QUERY */
+
+/* define if OpenSSL needs RSAref */
+/* #undef HAVE_RSAREF */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_SASL_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_SASL_SASL_H */
+
+/* define if your SASL library has sasl_version() */
+/* #undef HAVE_SASL_VERSION */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SCHED_H 1
+
+/* Define to 1 if you have the `sched_yield' function. */
+#define HAVE_SCHED_YIELD 1
+
+/* Define to 1 if you have the `send' function. */
+#define HAVE_SEND 1
+
+/* Define to 1 if you have the `sendmsg' function. */
+#define HAVE_SENDMSG 1
+
+/* Define to 1 if you have the `sendto' function. */
+#define HAVE_SENDTO 1
+
+/* Define to 1 if you have the `setegid' function. */
+#define HAVE_SETEGID 1
+
+/* Define to 1 if you have the `seteuid' function. */
+#define HAVE_SETEUID 1
+
+/* Define to 1 if you have the `setgid' function. */
+#define HAVE_SETGID 1
+
+/* Define to 1 if you have the `setpwfile' function. */
+/* #undef HAVE_SETPWFILE */
+
+/* Define to 1 if you have the `setsid' function. */
+#define HAVE_SETSID 1
+
+/* Define to 1 if you have the `setuid' function. */
+#define HAVE_SETUID 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SGTTY_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_SHADOW_H */
+
+/* Define to 1 if you have the `sigaction' function. */
+#define HAVE_SIGACTION 1
+
+/* Define to 1 if you have the `signal' function. */
+#define HAVE_SIGNAL 1
+
+/* Define to 1 if you have the `sigset' function. */
+#define HAVE_SIGSET 1
+
+/* define if you have -lslp */
+/* #undef HAVE_SLP */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_SLP_H */
+
+/* Define to 1 if you have the `snprintf' function. */
+#define HAVE_SNPRINTF 1
+
+/* if you have spawnlp() */
+/* #undef HAVE_SPAWNLP */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_SQLEXT_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_SQL_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STDDEF_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STDINT_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STDLIB_H 1
+
+/* Define to 1 if you have the `strdup' function. */
+#define HAVE_STRDUP 1
+
+/* Define to 1 if you have the `strerror' function. */
+#define HAVE_STRERROR 1
+
+/* Define to 1 if you have the `strerror_r' function. */
+#define HAVE_STRERROR_R 1
+
+/* Define to 1 if you have the `strftime' function. */
+#define HAVE_STRFTIME 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STRINGS_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_STRING_H 1
+
+/* Define to 1 if you have the `strpbrk' function. */
+#define HAVE_STRPBRK 1
+
+/* Define to 1 if you have the `strrchr' function. */
+#define HAVE_STRRCHR 1
+
+/* Define to 1 if you have the `strsep' function. */
+#define HAVE_STRSEP 1
+
+/* Define to 1 if you have the `strspn' function. */
+#define HAVE_STRSPN 1
+
+/* Define to 1 if you have the `strstr' function. */
+#define HAVE_STRSTR 1
+
+/* Define to 1 if you have the `strtol' function. */
+#define HAVE_STRTOL 1
+
+/* Define to 1 if you have the `strtoll' function. */
+#define HAVE_STRTOLL 1
+
+/* Define to 1 if you have the `strtoq' function. */
+#define HAVE_STRTOQ 1
+
+/* Define to 1 if you have the `strtoul' function. */
+#define HAVE_STRTOUL 1
+
+/* Define to 1 if you have the `strtoull' function. */
+#define HAVE_STRTOULL 1
+
+/* Define to 1 if you have the `strtouq' function. */
+#define HAVE_STRTOUQ 1
+
+/* Define to 1 if `msg_accrightslen' is a member of `struct msghdr'. */
+/* #undef HAVE_STRUCT_MSGHDR_MSG_ACCRIGHTSLEN */
+
+/* Define to 1 if `msg_control' is a member of `struct msghdr'. */
+/* #undef HAVE_STRUCT_MSGHDR_MSG_CONTROL */
+
+/* Define to 1 if `pw_gecos' is a member of `struct passwd'. */
+#define HAVE_STRUCT_PASSWD_PW_GECOS 1
+
+/* Define to 1 if `pw_passwd' is a member of `struct passwd'. */
+#define HAVE_STRUCT_PASSWD_PW_PASSWD 1
+
+/* Define to 1 if `st_blksize' is a member of `struct stat'. */
+#define HAVE_STRUCT_STAT_ST_BLKSIZE 1
+
+/* Define to 1 if `st_fstype' is a member of `struct stat'. */
+/* #undef HAVE_STRUCT_STAT_ST_FSTYPE */
+
+/* define to 1 if st_fstype is char * */
+/* #undef HAVE_STRUCT_STAT_ST_FSTYPE_CHAR */
+
+/* define to 1 if st_fstype is int */
+/* #undef HAVE_STRUCT_STAT_ST_FSTYPE_INT */
+
+/* Define to 1 if `st_vfstype' is a member of `struct stat'. */
+/* #undef HAVE_STRUCT_STAT_ST_VFSTYPE */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_SYNCH_H */
+
+/* Define to 1 if you have the `sysconf' function. */
+#define HAVE_SYSCONF 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYSEXITS_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYSLOG_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_SYS_DEVPOLL_H */
+
+/* Define to 1 if you have the header file, and it defines `DIR'.
+ */
+/* #undef HAVE_SYS_DIR_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_SYS_EPOLL_H */
+
+/* define if you actually have sys_errlist in your libs */
+#define HAVE_SYS_ERRLIST 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_ERRNO_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_EVENT_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_FILE_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_FILIO_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_SYS_FSTYP_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_IOCTL_H 1
+
+/* Define to 1 if you have the header file, and it defines `DIR'.
+ */
+/* #undef HAVE_SYS_NDIR_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_PARAM_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_POLL_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_SYS_PRIVGRP_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_RESOURCE_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_SELECT_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_SOCKET_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_STAT_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_SYSLOG_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_TIME_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_TYPES_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_UCRED_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_UIO_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_SYS_UN_H 1
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_SYS_UUID_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_SYS_VMOUNT_H */
+
+/* Define to 1 if you have that is POSIX.1 compatible. */
+#define HAVE_SYS_WAIT_H 1
+
+/* define if you have -lwrap */
+/* #undef HAVE_TCPD */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_TCPD_H */
+
+/* Define to 1 if you have the header file. */
+#define HAVE_TERMIOS_H 1
+
+/* if you have Solaris LWP (thr) package */
+/* #undef HAVE_THR */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_THREAD_H */
+
+/* Define to 1 if you have the `thr_getconcurrency' function. */
+/* #undef HAVE_THR_GETCONCURRENCY */
+
+/* Define to 1 if you have the `thr_setconcurrency' function. */
+/* #undef HAVE_THR_SETCONCURRENCY */
+
+/* Define to 1 if you have the `thr_yield' function. */
+/* #undef HAVE_THR_YIELD */
+
+/* define if you have TLS */
+#define HAVE_TLS 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_UNISTD_H 1
+
+/* Define to 1 if you have the header file. */
+#define HAVE_UTIME_H 1
+
+/* define if you have uuid_generate() */
+/* #undef HAVE_UUID_GENERATE */
+
+/* define if you have uuid_to_str() */
+/* #undef HAVE_UUID_TO_STR */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_UUID_UUID_H */
+
+/* Define to 1 if you have the `vprintf' function. */
+#define HAVE_VPRINTF 1
+
+/* Define to 1 if you have the `vsnprintf' function. */
+#define HAVE_VSNPRINTF 1
+
+/* Define to 1 if you have the `wait4' function. */
+#define HAVE_WAIT4 1
+
+/* Define to 1 if you have the `waitpid' function. */
+#define HAVE_WAITPID 1
+
+/* define if you have winsock */
+/* #undef HAVE_WINSOCK */
+
+/* define if you have winsock2 */
+/* #undef HAVE_WINSOCK2 */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_WINSOCK2_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_WINSOCK_H */
+
+/* Define to 1 if you have the header file. */
+/* #undef HAVE_WIREDTIGER_H */
+
+/* Define to 1 if you have the `write' function. */
+#define HAVE_WRITE 1
+
+/* define if select implicitly yields */
+#define HAVE_YIELDING_SELECT 1
+
+/* Define to 1 if you have the `_vsnprintf' function. */
+/* #undef HAVE__VSNPRINTF */
+
+/* define to 32-bit or greater integer type */
+#define LBER_INT_T int
+
+/* define to large integer type */
+#define LBER_LEN_T long
+
+/* define to socket descriptor type */
+#define LBER_SOCKET_T int
+
+/* define to large integer type */
+#define LBER_TAG_T long
+
+/* define to 1 if library is thread safe */
+#define LDAP_API_FEATURE_X_OPENLDAP_THREAD_SAFE 1
+
+/* define to LDAP VENDOR VERSION */
+/* #undef LDAP_API_FEATURE_X_OPENLDAP_V2_REFERRALS */
+
+/* define this to add debugging code */
+/* #undef LDAP_DEBUG */
+
+/* define if LDAP libs are dynamic */
+/* #undef LDAP_LIBS_DYNAMIC */
+
+/* define to support PF_INET6 */
+#define LDAP_PF_INET6 1
+
+/* define to support PF_LOCAL */
+#define LDAP_PF_LOCAL 1
+
+/* define this to add SLAPI code */
+/* #undef LDAP_SLAPI */
+
+/* define this to add syslog code */
+/* #undef LDAP_SYSLOG */
+
+/* Version */
+#define LDAP_VENDOR_VERSION 20501
+
+/* Major */
+#define LDAP_VENDOR_VERSION_MAJOR 2
+
+/* Minor */
+#define LDAP_VENDOR_VERSION_MINOR 5
+
+/* Patch */
+#define LDAP_VENDOR_VERSION_PATCH X
+
+/* Define to the sub-directory where libtool stores uninstalled libraries. */
+#define LT_OBJDIR ".libs/"
+
+/* define if memcmp is not 8-bit clean or is otherwise broken */
+/* #undef NEED_MEMCMP_REPLACEMENT */
+
+/* define if you have (or want) no threads */
+/* #undef NO_THREADS */
+
+/* define to use the original debug style */
+/* #undef OLD_DEBUG */
+
+/* Package */
+#define OPENLDAP_PACKAGE "OpenLDAP"
+
+/* Version */
+#define OPENLDAP_VERSION "2.5.X"
+
+/* Define to the address where bug reports for this package should be sent. */
+#define PACKAGE_BUGREPORT ""
+
+/* Define to the full name of this package. */
+#define PACKAGE_NAME ""
+
+/* Define to the full name and version of this package. */
+#define PACKAGE_STRING ""
+
+/* Define to the one symbol short name of this package. */
+#define PACKAGE_TARNAME ""
+
+/* Define to the home page for this package. */
+#define PACKAGE_URL ""
+
+/* Define to the version of this package. */
+#define PACKAGE_VERSION ""
+
+/* define if sched_yield yields the entire process */
+/* #undef REPLACE_BROKEN_YIELD */
+
+/* Define as the return type of signal handlers (`int' or `void'). */
+#define RETSIGTYPE void
+
+/* Define to the type of arg 1 for `select'. */
+#define SELECT_TYPE_ARG1 int
+
+/* Define to the type of args 2, 3 and 4 for `select'. */
+#define SELECT_TYPE_ARG234 (fd_set *)
+
+/* Define to the type of arg 5 for `select'. */
+#define SELECT_TYPE_ARG5 (struct timeval *)
+
+/* The size of `int', as computed by sizeof. */
+#define SIZEOF_INT 4
+
+/* The size of `long', as computed by sizeof. */
+#define SIZEOF_LONG 8
+
+/* The size of `long long', as computed by sizeof. */
+#define SIZEOF_LONG_LONG 8
+
+/* The size of `short', as computed by sizeof. */
+#define SIZEOF_SHORT 2
+
+/* The size of `wchar_t', as computed by sizeof. */
+#define SIZEOF_WCHAR_T 4
+
+/* define to support per-object ACIs */
+/* #undef SLAPD_ACI_ENABLED */
+
+/* define to support LDAP Async Metadirectory backend */
+/* #undef SLAPD_ASYNCMETA */
+
+/* define to support cleartext passwords */
+/* #undef SLAPD_CLEARTEXT */
+
+/* define to support crypt(3) passwords */
+/* #undef SLAPD_CRYPT */
+
+/* define to support DNS SRV backend */
+/* #undef SLAPD_DNSSRV */
+
+/* define to support LDAP backend */
+/* #undef SLAPD_LDAP */
+
+/* define to support MDB backend */
+/* #undef SLAPD_MDB */
+
+/* define to support LDAP Metadirectory backend */
+/* #undef SLAPD_META */
+
+/* define to support modules */
+/* #undef SLAPD_MODULES */
+
+/* dynamically linked module */
+#define SLAPD_MOD_DYNAMIC 2
+
+/* statically linked module */
+#define SLAPD_MOD_STATIC 1
+
+/* define to support cn=Monitor backend */
+/* #undef SLAPD_MONITOR */
+
+/* define to support NDB backend */
+/* #undef SLAPD_NDB */
+
+/* define to support NULL backend */
+/* #undef SLAPD_NULL */
+
+/* define for In-Directory Access Logging overlay */
+/* #undef SLAPD_OVER_ACCESSLOG */
+
+/* define for Audit Logging overlay */
+/* #undef SLAPD_OVER_AUDITLOG */
+
+/* define for Automatic Certificate Authority overlay */
+/* #undef SLAPD_OVER_AUTOCA */
+
+/* define for Collect overlay */
+/* #undef SLAPD_OVER_COLLECT */
+
+/* define for Attribute Constraint overlay */
+/* #undef SLAPD_OVER_CONSTRAINT */
+
+/* define for Dynamic Directory Services overlay */
+/* #undef SLAPD_OVER_DDS */
+
+/* define for Dynamic Directory Services overlay */
+/* #undef SLAPD_OVER_DEREF */
+
+/* define for Dynamic Group overlay */
+/* #undef SLAPD_OVER_DYNGROUP */
+
+/* define for Dynamic List overlay */
+/* #undef SLAPD_OVER_DYNLIST */
+
+/* define for Reverse Group Membership overlay */
+/* #undef SLAPD_OVER_MEMBEROF */
+
+/* define for Password Policy overlay */
+/* #undef SLAPD_OVER_PPOLICY */
+
+/* define for Proxy Cache overlay */
+/* #undef SLAPD_OVER_PROXYCACHE */
+
+/* define for Referential Integrity overlay */
+/* #undef SLAPD_OVER_REFINT */
+
+/* define for Return Code overlay */
+/* #undef SLAPD_OVER_RETCODE */
+
+/* define for Rewrite/Remap overlay */
+/* #undef SLAPD_OVER_RWM */
+
+/* define for Sequential Modify overlay */
+/* #undef SLAPD_OVER_SEQMOD */
+
+/* define for ServerSideSort/VLV overlay */
+/* #undef SLAPD_OVER_SSSVLV */
+
+/* define for Syncrepl Provider overlay */
+/* #undef SLAPD_OVER_SYNCPROV */
+
+/* define for Translucent Proxy overlay */
+/* #undef SLAPD_OVER_TRANSLUCENT */
+
+/* define for Attribute Uniqueness overlay */
+/* #undef SLAPD_OVER_UNIQUE */
+
+/* define for Value Sorting overlay */
+/* #undef SLAPD_OVER_VALSORT */
+
+/* define to support PASSWD backend */
+/* #undef SLAPD_PASSWD */
+
+/* define to support PERL backend */
+/* #undef SLAPD_PERL */
+
+/* define to support relay backend */
+/* #undef SLAPD_RELAY */
+
+/* define to support reverse lookups */
+/* #undef SLAPD_RLOOKUPS */
+
+/* define to support SHELL backend */
+/* #undef SLAPD_SHELL */
+
+/* define to support SOCK backend */
+/* #undef SLAPD_SOCK */
+
+/* define to support SASL passwords */
+/* #undef SLAPD_SPASSWD */
+
+/* define to support SQL backend */
+/* #undef SLAPD_SQL */
+
+/* define to support WiredTiger backend */
+/* #undef SLAPD_WT */
+
+/* define to support run-time loadable ACL */
+/* #undef SLAP_DYNACL */
+
+/* Define to 1 if you have the ANSI C header files. */
+#define STDC_HEADERS 1
+
+/* Define to 1 if you can safely include both and . */
+#define TIME_WITH_SYS_TIME 1
+
+/* Define to 1 if your declares `struct tm'. */
+/* #undef TM_IN_SYS_TIME */
+
+/* set to urandom device */
+#define URANDOM_DEVICE "/dev/urandom"
+
+/* define to use OpenSSL BIGNUM for MP */
+/* #undef USE_MP_BIGNUM */
+
+/* define to use GMP for MP */
+/* #undef USE_MP_GMP */
+
+/* define to use 'long' for MP */
+/* #undef USE_MP_LONG */
+
+/* define to use 'long long' for MP */
+/* #undef USE_MP_LONG_LONG */
+
+/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
+ significant byte first (like Motorola and SPARC, unlike Intel). */
+#if defined AC_APPLE_UNIVERSAL_BUILD
+# if defined __BIG_ENDIAN__
+# define WORDS_BIGENDIAN 1
+# endif
+#else
+# ifndef WORDS_BIGENDIAN
+/* # undef WORDS_BIGENDIAN */
+# endif
+#endif
+
+/* Define to the type of arg 3 for `accept'. */
+#define ber_socklen_t socklen_t
+
+/* Define to `char *' if does not define. */
+/* #undef caddr_t */
+
+/* Define to empty if `const' does not conform to ANSI C. */
+/* #undef const */
+
+/* Define to `int' if doesn't define. */
+/* #undef gid_t */
+
+/* Define to `int' if does not define. */
+/* #undef mode_t */
+
+/* Define to `long' if does not define. */
+/* #undef off_t */
+
+/* Define to `int' if does not define. */
+/* #undef pid_t */
+
+/* Define to `int' if does not define. */
+/* #undef sig_atomic_t */
+
+/* Define to `unsigned' if does not define. */
+/* #undef size_t */
+
+/* define to snprintf routine */
+/* #undef snprintf */
+
+/* Define like ber_socklen_t if does not define. */
+/* #undef socklen_t */
+
+/* Define to `signed int' if does not define. */
+/* #undef ssize_t */
+
+/* Define to `int' if doesn't define. */
+/* #undef uid_t */
+
+/* define as empty if volatile is not supported */
+/* #undef volatile */
+
+/* define to snprintf routine */
+/* #undef vsnprintf */
+
+
+/* begin of portable.h.post */
+
+#ifdef _WIN32
+/* don't suck in all of the win32 api */
+# define WIN32_LEAN_AND_MEAN 1
+#endif
+
+#ifndef LDAP_NEEDS_PROTOTYPES
+/* force LDAP_P to always include prototypes */
+#define LDAP_NEEDS_PROTOTYPES 1
+#endif
+
+#ifndef LDAP_REL_ENG
+#if (LDAP_VENDOR_VERSION == 000000) && !defined(LDAP_DEVEL)
+#define LDAP_DEVEL
+#endif
+#if defined(LDAP_DEVEL) && !defined(LDAP_TEST)
+#define LDAP_TEST
+#endif
+#endif
+
+#ifdef HAVE_STDDEF_H
+# include
+#endif
+
+#ifdef HAVE_EBCDIC
+/* ASCII/EBCDIC converting replacements for stdio funcs
+ * vsnprintf and snprintf are used too, but they are already
+ * checked by the configure script
+ */
+#define fputs ber_pvt_fputs
+#define fgets ber_pvt_fgets
+#define printf ber_pvt_printf
+#define fprintf ber_pvt_fprintf
+#define vfprintf ber_pvt_vfprintf
+#define vsprintf ber_pvt_vsprintf
+#endif
+
+#include "ac/fdset.h"
+
+#include "ldap_cdefs.h"
+#include "ldap_features.h"
+
+#include "ac/assert.h"
+#include "ac/localize.h"
+
+#endif /* _LDAP_PORTABLE_H */
+/* end of portable.h.post */
+
diff --git a/contrib/poco b/contrib/poco
index 83beecccb09..b7d9ec16ee3 160000
--- a/contrib/poco
+++ b/contrib/poco
@@ -1 +1 @@
-Subproject commit 83beecccb09eec0c9fd2669cacea03ede1d9f138
+Subproject commit b7d9ec16ee33ca76643d5fcd907ea9a33285640a
diff --git a/contrib/poco-cmake/Foundation/CMakeLists.txt b/contrib/poco-cmake/Foundation/CMakeLists.txt
index f4647461ec0..6476845b4e3 100644
--- a/contrib/poco-cmake/Foundation/CMakeLists.txt
+++ b/contrib/poco-cmake/Foundation/CMakeLists.txt
@@ -233,3 +233,10 @@ else ()
message (STATUS "Using Poco::Foundation: ${LIBRARY_POCO_FOUNDATION} ${INCLUDE_POCO_FOUNDATION}")
endif ()
+
+if(OS_DARWIN AND ARCH_AARCH64)
+ target_compile_definitions (_poco_foundation
+ PRIVATE
+ POCO_NO_STAT64
+ )
+endif()
diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt
index 77a30776a4a..117015ef5c2 100644
--- a/contrib/rocksdb-cmake/CMakeLists.txt
+++ b/contrib/rocksdb-cmake/CMakeLists.txt
@@ -142,14 +142,14 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64")
endif(HAS_ALTIVEC)
endif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64")
-if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64")
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64|arm64|ARM64")
CHECK_C_COMPILER_FLAG("-march=armv8-a+crc+crypto" HAS_ARMV8_CRC)
if(HAS_ARMV8_CRC)
message(STATUS " HAS_ARMV8_CRC yes")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=armv8-a+crc+crypto -Wno-unused-function")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=armv8-a+crc+crypto -Wno-unused-function")
endif(HAS_ARMV8_CRC)
-endif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64")
+endif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64|arm64|ARM64")
include(CheckCXXSourceCompiles)
diff --git a/contrib/zlib-ng b/contrib/zlib-ng
index 6fd1846c8b8..5cc4d232020 160000
--- a/contrib/zlib-ng
+++ b/contrib/zlib-ng
@@ -1 +1 @@
-Subproject commit 6fd1846c8b8f59436fe2dd752d0f316ddbb64df6
+Subproject commit 5cc4d232020dc66d1d6c5438834457e2a2f6127b
diff --git a/debian/changelog b/debian/changelog
index be77dfdefe9..8b6626416a9 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,5 +1,5 @@
-clickhouse (21.5.1.1) unstable; urgency=low
+clickhouse (21.6.1.1) unstable; urgency=low
* Modified source code
- -- clickhouse-release Fri, 02 Apr 2021 18:34:26 +0300
+ -- clickhouse-release Tue, 20 Apr 2021 01:48:16 +0300
diff --git a/debian/clickhouse-common-static.install b/debian/clickhouse-common-static.install
index 17c955a12a9..087a6dbba8f 100644
--- a/debian/clickhouse-common-static.install
+++ b/debian/clickhouse-common-static.install
@@ -1,5 +1,5 @@
usr/bin/clickhouse
usr/bin/clickhouse-odbc-bridge
+usr/bin/clickhouse-library-bridge
usr/bin/clickhouse-extract-from-config
usr/share/bash-completion/completions
-etc/security/limits.d/clickhouse.conf
diff --git a/debian/clickhouse-server.config b/debian/clickhouse-server.config
deleted file mode 100644
index 636ff7f4da7..00000000000
--- a/debian/clickhouse-server.config
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/sh -e
-
-test -f /usr/share/debconf/confmodule && . /usr/share/debconf/confmodule
-
-db_fget clickhouse-server/default-password seen || true
-password_seen="$RET"
-
-if [ "$1" = "reconfigure" ]; then
- password_seen=false
-fi
-
-if [ "$password_seen" != "true" ]; then
- db_input high clickhouse-server/default-password || true
- db_go || true
-fi
-db_go || true
diff --git a/debian/clickhouse-server.postinst b/debian/clickhouse-server.postinst
index dc876f45954..419c13e3daf 100644
--- a/debian/clickhouse-server.postinst
+++ b/debian/clickhouse-server.postinst
@@ -23,11 +23,13 @@ if [ ! -f "/etc/debian_version" ]; then
fi
if [ "$1" = configure ] || [ -n "$not_deb_os" ]; then
+
+ ${CLICKHOUSE_GENERIC_PROGRAM} install --user "${CLICKHOUSE_USER}" --group "${CLICKHOUSE_GROUP}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" --log-path "${CLICKHOUSE_LOGDIR}" --data-path "${CLICKHOUSE_DATADIR}"
+
if [ -x "/bin/systemctl" ] && [ -f /etc/systemd/system/clickhouse-server.service ] && [ -d /run/systemd/system ]; then
# if old rc.d service present - remove it
if [ -x "/etc/init.d/clickhouse-server" ] && [ -x "/usr/sbin/update-rc.d" ]; then
/usr/sbin/update-rc.d clickhouse-server remove
- echo "ClickHouse init script has migrated to systemd. Please manually stop old server and restart the service: sudo killall clickhouse-server && sleep 5 && sudo service clickhouse-server restart"
fi
/bin/systemctl daemon-reload
@@ -38,10 +40,8 @@ if [ "$1" = configure ] || [ -n "$not_deb_os" ]; then
if [ -x "/usr/sbin/update-rc.d" ]; then
/usr/sbin/update-rc.d clickhouse-server defaults 19 19 >/dev/null || exit $?
else
- echo # TODO [ "$OS" = "rhel" ] || [ "$OS" = "centos" ] || [ "$OS" = "fedora" ]
+ echo # Other OS
fi
fi
fi
-
- ${CLICKHOUSE_GENERIC_PROGRAM} install --user "${CLICKHOUSE_USER}" --group "${CLICKHOUSE_GROUP}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" --log-path "${CLICKHOUSE_LOGDIR}" --data-path "${CLICKHOUSE_DATADIR}"
fi
diff --git a/debian/clickhouse-server.preinst b/debian/clickhouse-server.preinst
deleted file mode 100644
index 3529aefa7da..00000000000
--- a/debian/clickhouse-server.preinst
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh
-
-if [ "$1" = "upgrade" ]; then
- # Return etc/cron.d/clickhouse-server to original state
- service clickhouse-server disable_cron ||:
-fi
-
-#DEBHELPER#
diff --git a/debian/clickhouse-server.prerm b/debian/clickhouse-server.prerm
deleted file mode 100644
index 02e855a7125..00000000000
--- a/debian/clickhouse-server.prerm
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/sh
-
-if [ "$1" = "upgrade" ] || [ "$1" = "remove" ]; then
- # Return etc/cron.d/clickhouse-server to original state
- service clickhouse-server disable_cron ||:
-fi
diff --git a/debian/clickhouse-server.templates b/debian/clickhouse-server.templates
deleted file mode 100644
index dd55824e15c..00000000000
--- a/debian/clickhouse-server.templates
+++ /dev/null
@@ -1,3 +0,0 @@
-Template: clickhouse-server/default-password
-Type: password
-Description: Enter password for default user:
diff --git a/debian/clickhouse.limits b/debian/clickhouse.limits
deleted file mode 100644
index aca44082c4e..00000000000
--- a/debian/clickhouse.limits
+++ /dev/null
@@ -1,2 +0,0 @@
-clickhouse soft nofile 262144
-clickhouse hard nofile 262144
diff --git a/debian/rules b/debian/rules
index 8eb47e95389..73d1f3d3b34 100755
--- a/debian/rules
+++ b/debian/rules
@@ -113,9 +113,6 @@ override_dh_install:
ln -sf clickhouse-server.docs debian/clickhouse-client.docs
ln -sf clickhouse-server.docs debian/clickhouse-common-static.docs
- mkdir -p $(DESTDIR)/etc/security/limits.d
- cp debian/clickhouse.limits $(DESTDIR)/etc/security/limits.d/clickhouse.conf
-
# systemd compatibility
mkdir -p $(DESTDIR)/etc/systemd/system/
cp debian/clickhouse-server.service $(DESTDIR)/etc/systemd/system/
diff --git a/debian/watch b/debian/watch
index 7ad4cedf713..ed3cab97ade 100644
--- a/debian/watch
+++ b/debian/watch
@@ -1,6 +1,6 @@
version=4
opts="filenamemangle=s%(?:.*?)?v?(\d[\d.]*)-stable\.tar\.gz%clickhouse-$1.tar.gz%" \
- https://github.com/yandex/clickhouse/tags \
+ https://github.com/ClickHouse/ClickHouse/tags \
(?:.*?/)?v?(\d[\d.]*)-stable\.tar\.gz debian uupdate
diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile
index 2efba9735ae..569025dec1c 100644
--- a/docker/client/Dockerfile
+++ b/docker/client/Dockerfile
@@ -1,7 +1,7 @@
FROM ubuntu:18.04
ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
-ARG version=21.5.1.*
+ARG version=21.6.1.*
RUN apt-get update \
&& apt-get install --yes --no-install-recommends \
diff --git a/docker/images.json b/docker/images.json
index 303bd159ce4..e2e22468596 100644
--- a/docker/images.json
+++ b/docker/images.json
@@ -138,7 +138,8 @@
"docker/test/stateless_unbundled",
"docker/test/stateless_pytest",
"docker/test/integration/base",
- "docker/test/fuzzer"
+ "docker/test/fuzzer",
+ "docker/test/keeper-jepsen"
]
},
"docker/packager/unbundled": {
@@ -159,5 +160,9 @@
"docker/test/sqlancer": {
"name": "yandex/clickhouse-sqlancer-test",
"dependent": []
+ },
+ "docker/test/keeper-jepsen": {
+ "name": "yandex/clickhouse-keeper-jepsen-test",
+ "dependent": []
}
}
diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile
index 94c7f934f6e..fccae66b66b 100644
--- a/docker/packager/binary/Dockerfile
+++ b/docker/packager/binary/Dockerfile
@@ -35,35 +35,32 @@ RUN apt-get update \
RUN apt-get update \
&& apt-get install \
bash \
- cmake \
+ build-essential \
ccache \
- curl \
- gcc-9 \
- g++-9 \
clang-10 \
- clang-tidy-10 \
- lld-10 \
- llvm-10 \
- llvm-10-dev \
clang-11 \
+ clang-tidy-10 \
clang-tidy-11 \
- lld-11 \
- llvm-11 \
- llvm-11-dev \
+ cmake \
+ curl \
+ g++-9 \
+ gcc-9 \
+ gdb \
+ git \
+ gperf \
libicu-dev \
libreadline-dev \
+ lld-10 \
+ lld-11 \
+ llvm-10 \
+ llvm-10-dev \
+ llvm-11 \
+ llvm-11-dev \
+ moreutils \
ninja-build \
- gperf \
- git \
- opencl-headers \
- ocl-icd-libopencl1 \
- intel-opencl-icd \
- tzdata \
- gperf \
- cmake \
- gdb \
+ pigz \
rename \
- build-essential \
+ tzdata \
--yes --no-install-recommends
# This symlink required by gcc to find lld compiler
@@ -111,4 +108,4 @@ RUN rm /etc/apt/sources.list.d/proposed-repositories.list && apt-get update
COPY build.sh /
-CMD ["/bin/bash", "/build.sh"]
+CMD ["bash", "-c", "/build.sh 2>&1 | ts"]
diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh
index a42789c6186..cf74105fbbb 100755
--- a/docker/packager/binary/build.sh
+++ b/docker/packager/binary/build.sh
@@ -11,17 +11,28 @@ tar xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -C build/cmake/toolc
mkdir -p build/cmake/toolchain/freebsd-x86_64
tar xJf freebsd-11.3-toolchain.tar.xz -C build/cmake/toolchain/freebsd-x86_64 --strip-components=1
+# Uncomment to debug ccache. Don't put ccache log in /output right away, or it
+# will be confusingly packed into the "performance" package.
+# export CCACHE_LOGFILE=/build/ccache.log
+# export CCACHE_DEBUG=1
+
mkdir -p build/build_docker
cd build/build_docker
-ccache --show-stats ||:
-ccache --zero-stats ||:
-ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||:
rm -f CMakeCache.txt
# Read cmake arguments into array (possibly empty)
read -ra CMAKE_FLAGS <<< "${CMAKE_FLAGS:-}"
cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
+
+ccache --show-config ||:
+ccache --show-stats ||:
+ccache --zero-stats ||:
+
# shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty.
ninja $NINJA_FLAGS clickhouse-bundle
+
+ccache --show-config ||:
+ccache --show-stats ||:
+
mv ./programs/clickhouse* /output
mv ./src/unit_tests_dbms /output ||: # may not exist for some binary builds
find . -name '*.so' -print -exec mv '{}' /output \;
@@ -65,8 +76,21 @@ then
cp ../programs/server/config.xml /output/config
cp ../programs/server/users.xml /output/config
cp -r --dereference ../programs/server/config.d /output/config
- tar -czvf "$COMBINED_OUTPUT.tgz" /output
+ tar -cv -I pigz -f "$COMBINED_OUTPUT.tgz" /output
rm -r /output/*
mv "$COMBINED_OUTPUT.tgz" /output
fi
-ccache --show-stats ||:
+
+if [ "${CCACHE_DEBUG:-}" == "1" ]
+then
+ find . -name '*.ccache-*' -print0 \
+ | tar -c -I pixz -f /output/ccache-debug.txz --null -T -
+fi
+
+if [ -n "$CCACHE_LOGFILE" ]
+then
+ # Compress the log as well, or else the CI will try to compress all log
+ # files in place, and will fail because this directory is not writable.
+ tar -cv -I pixz -f /output/ccache.log.txz "$CCACHE_LOGFILE"
+fi
+
diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile
index 8fd89d60f85..902929a2644 100644
--- a/docker/packager/deb/Dockerfile
+++ b/docker/packager/deb/Dockerfile
@@ -34,31 +34,32 @@ RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb \
# Libraries from OS are only needed to test the "unbundled" build (this is not used in production).
RUN apt-get update \
&& apt-get install \
- gcc-9 \
- g++-9 \
- clang-11 \
- clang-tidy-11 \
- lld-11 \
- llvm-11 \
- llvm-11-dev \
+ alien \
clang-10 \
+ clang-11 \
clang-tidy-10 \
+ clang-tidy-11 \
+ cmake \
+ debhelper \
+ devscripts \
+ g++-9 \
+ gcc-9 \
+ gdb \
+ git \
+ gperf \
lld-10 \
+ lld-11 \
llvm-10 \
llvm-10-dev \
+ llvm-11 \
+ llvm-11-dev \
+ moreutils \
ninja-build \
perl \
- pkg-config \
- devscripts \
- debhelper \
- git \
- tzdata \
- gperf \
- alien \
- cmake \
- gdb \
- moreutils \
pigz \
+ pixz \
+ pkg-config \
+ tzdata \
--yes --no-install-recommends
# NOTE: For some reason we have outdated version of gcc-10 in ubuntu 20.04 stable.
diff --git a/docker/packager/deb/build.sh b/docker/packager/deb/build.sh
index 6450e21d289..c1a0b27db5d 100755
--- a/docker/packager/deb/build.sh
+++ b/docker/packager/deb/build.sh
@@ -2,8 +2,14 @@
set -x -e
+# Uncomment to debug ccache.
+# export CCACHE_LOGFILE=/build/ccache.log
+# export CCACHE_DEBUG=1
+
+ccache --show-config ||:
ccache --show-stats ||:
ccache --zero-stats ||:
+
read -ra ALIEN_PKGS <<< "${ALIEN_PKGS:-}"
build/release --no-pbuilder "${ALIEN_PKGS[@]}" | ts '%Y-%m-%d %H:%M:%S'
mv /*.deb /output
@@ -22,5 +28,19 @@ then
mv /build/obj-*/src/unit_tests_dbms /output/binary
fi
fi
+
+ccache --show-config ||:
ccache --show-stats ||:
-ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||:
+
+if [ "${CCACHE_DEBUG:-}" == "1" ]
+then
+ find /build -name '*.ccache-*' -print0 \
+ | tar -c -I pixz -f /output/ccache-debug.txz --null -T -
+fi
+
+if [ -n "$CCACHE_LOGFILE" ]
+then
+ # Compress the log as well, or else the CI will try to compress all log
+ # files in place, and will fail because this directory is not writable.
+ tar -cv -I pixz -f /output/ccache.log.txz "$CCACHE_LOGFILE"
+fi
diff --git a/docker/packager/unbundled/Dockerfile b/docker/packager/unbundled/Dockerfile
index f640c595f14..4dd6dbc61d8 100644
--- a/docker/packager/unbundled/Dockerfile
+++ b/docker/packager/unbundled/Dockerfile
@@ -35,9 +35,6 @@ RUN apt-get update \
libjemalloc-dev \
libmsgpack-dev \
libcurl4-openssl-dev \
- opencl-headers \
- ocl-icd-libopencl1 \
- intel-opencl-icd \
unixodbc-dev \
odbcinst \
tzdata \
diff --git a/docker/packager/unbundled/build.sh b/docker/packager/unbundled/build.sh
index 54575ab977c..99fc34fd9f3 100755
--- a/docker/packager/unbundled/build.sh
+++ b/docker/packager/unbundled/build.sh
@@ -13,4 +13,3 @@ mv /*.rpm /output ||: # if exists
mv /*.tgz /output ||: # if exists
ccache --show-stats ||:
-ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so ||:
diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile
index 05ca29f22d4..48c978366c6 100644
--- a/docker/server/Dockerfile
+++ b/docker/server/Dockerfile
@@ -1,7 +1,7 @@
FROM ubuntu:20.04
ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
-ARG version=21.5.1.*
+ARG version=21.6.1.*
ARG gosu_ver=1.10
# set non-empty deb_location_url url to create a docker image
diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile
index 976c46ebe27..0e4646386ce 100644
--- a/docker/test/Dockerfile
+++ b/docker/test/Dockerfile
@@ -1,7 +1,7 @@
FROM ubuntu:18.04
ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/"
-ARG version=21.5.1.*
+ARG version=21.6.1.*
RUN apt-get update && \
apt-get install -y apt-transport-https dirmngr && \
diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh
index c21a115289d..a7cc398e5c9 100755
--- a/docker/test/fasttest/run.sh
+++ b/docker/test/fasttest/run.sh
@@ -300,6 +300,7 @@ function run_tests
01663_aes_msan # Depends on OpenSSL
01667_aes_args_check # Depends on OpenSSL
01776_decrypt_aead_size_check # Depends on OpenSSL
+ 01811_filter_by_null # Depends on OpenSSL
01281_unsucceeded_insert_select_queries_counter
01292_create_user
01294_lazy_database_concurrent
@@ -307,10 +308,8 @@ function run_tests
01354_order_by_tuple_collate_const
01355_ilike
01411_bayesian_ab_testing
- 01532_collate_in_low_cardinality
- 01533_collate_in_nullable
- 01542_collate_in_array
- 01543_collate_in_tuple
+ collate
+ collation
_orc_
arrow
avro
@@ -365,6 +364,12 @@ function run_tests
# JSON functions
01666_blns
+
+ # Requires postgresql-client
+ 01802_test_postgresql_protocol_with_row_policy
+
+ # Depends on AWS
+ 01801_s3_cluster
)
(time clickhouse-test --hung-check -j 8 --order=random --use-skip-list --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" -- "$FASTTEST_FOCUS" 2>&1 ||:) | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/test_log.txt"
diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh
index 4bd3fa717a2..626bedb453c 100755
--- a/docker/test/fuzzer/run-fuzzer.sh
+++ b/docker/test/fuzzer/run-fuzzer.sh
@@ -198,7 +198,7 @@ case "$stage" in
# Lost connection to the server. This probably means that the server died
# with abort.
echo "failure" > status.txt
- if ! grep -ao "Received signal.*\|Logical error.*\|Assertion.*failed\|Failed assertion.*\|.*runtime error: .*\|.*is located.*\|SUMMARY: MemorySanitizer:.*\|SUMMARY: ThreadSanitizer:.*\|.*_LIBCPP_ASSERT.*" server.log > description.txt
+ if ! grep -ao "Received signal.*\|Logical error.*\|Assertion.*failed\|Failed assertion.*\|.*runtime error: .*\|.*is located.*\|SUMMARY: AddressSanitizer:.*\|SUMMARY: MemorySanitizer:.*\|SUMMARY: ThreadSanitizer:.*\|.*_LIBCPP_ASSERT.*" server.log > description.txt
then
echo "Lost connection to server. See the logs." > description.txt
fi
diff --git a/docker/test/integration/base/Dockerfile b/docker/test/integration/base/Dockerfile
index 938d8d45ffd..1c962f1bf8f 100644
--- a/docker/test/integration/base/Dockerfile
+++ b/docker/test/integration/base/Dockerfile
@@ -19,7 +19,8 @@ RUN apt-get update \
tar \
krb5-user \
iproute2 \
- lsof
+ lsof \
+ g++
RUN rm -rf \
/var/lib/apt/lists/* \
/var/cache/debconf \
diff --git a/docker/test/integration/runner/Dockerfile b/docker/test/integration/runner/Dockerfile
index e0e5e36a3d6..783e689ed01 100644
--- a/docker/test/integration/runner/Dockerfile
+++ b/docker/test/integration/runner/Dockerfile
@@ -31,6 +31,7 @@ RUN apt-get update \
software-properties-common \
libkrb5-dev \
krb5-user \
+ g++ \
&& rm -rf \
/var/lib/apt/lists/* \
/var/cache/debconf \
diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_cluster.yml b/docker/test/integration/runner/compose/docker_compose_mysql_cluster.yml
new file mode 100644
index 00000000000..d0674362709
--- /dev/null
+++ b/docker/test/integration/runner/compose/docker_compose_mysql_cluster.yml
@@ -0,0 +1,23 @@
+version: '2.3'
+services:
+ mysql2:
+ image: mysql:5.7
+ restart: always
+ environment:
+ MYSQL_ROOT_PASSWORD: clickhouse
+ ports:
+ - 3348:3306
+ mysql3:
+ image: mysql:5.7
+ restart: always
+ environment:
+ MYSQL_ROOT_PASSWORD: clickhouse
+ ports:
+ - 3388:3306
+ mysql4:
+ image: mysql:5.7
+ restart: always
+ environment:
+ MYSQL_ROOT_PASSWORD: clickhouse
+ ports:
+ - 3368:3306
diff --git a/docker/test/integration/runner/compose/docker_compose_postgres.yml b/docker/test/integration/runner/compose/docker_compose_postgres.yml
index 58ed97251fb..5657352e1b3 100644
--- a/docker/test/integration/runner/compose/docker_compose_postgres.yml
+++ b/docker/test/integration/runner/compose/docker_compose_postgres.yml
@@ -11,10 +11,3 @@ services:
default:
aliases:
- postgre-sql.local
- postgres2:
- image: postgres
- restart: always
- environment:
- POSTGRES_PASSWORD: mysecretpassword
- ports:
- - 5441:5432
diff --git a/docker/test/integration/runner/compose/docker_compose_postgres_cluster.yml b/docker/test/integration/runner/compose/docker_compose_postgres_cluster.yml
new file mode 100644
index 00000000000..d04c8a2f3a6
--- /dev/null
+++ b/docker/test/integration/runner/compose/docker_compose_postgres_cluster.yml
@@ -0,0 +1,23 @@
+version: '2.3'
+services:
+ postgres2:
+ image: postgres
+ restart: always
+ environment:
+ POSTGRES_PASSWORD: mysecretpassword
+ ports:
+ - 5421:5432
+ postgres3:
+ image: postgres
+ restart: always
+ environment:
+ POSTGRES_PASSWORD: mysecretpassword
+ ports:
+ - 5441:5432
+ postgres4:
+ image: postgres
+ restart: always
+ environment:
+ POSTGRES_PASSWORD: mysecretpassword
+ ports:
+ - 5461:5432
diff --git a/docker/test/integration/runner/dockerd-entrypoint.sh b/docker/test/integration/runner/dockerd-entrypoint.sh
index c0255d3d706..bda6f5a719d 100755
--- a/docker/test/integration/runner/dockerd-entrypoint.sh
+++ b/docker/test/integration/runner/dockerd-entrypoint.sh
@@ -21,6 +21,7 @@ export CLICKHOUSE_TESTS_SERVER_BIN_PATH=/clickhouse
export CLICKHOUSE_TESTS_CLIENT_BIN_PATH=/clickhouse
export CLICKHOUSE_TESTS_BASE_CONFIG_DIR=/clickhouse-config
export CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH=/clickhouse-odbc-bridge
+export CLICKHOUSE_LIBRARY_BRIDGE_BINARY_PATH=/clickhouse-library-bridge
export DOCKER_MYSQL_GOLANG_CLIENT_TAG=${DOCKER_MYSQL_GOLANG_CLIENT_TAG:=latest}
export DOCKER_MYSQL_JAVA_CLIENT_TAG=${DOCKER_MYSQL_JAVA_CLIENT_TAG:=latest}
diff --git a/docker/test/keeper-jepsen/Dockerfile b/docker/test/keeper-jepsen/Dockerfile
new file mode 100644
index 00000000000..1a62d5e793f
--- /dev/null
+++ b/docker/test/keeper-jepsen/Dockerfile
@@ -0,0 +1,39 @@
+# docker build -t yandex/clickhouse-keeper-jepsen-test .
+FROM yandex/clickhouse-test-base
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV CLOJURE_VERSION=1.10.3.814
+
+# arguments
+ENV PR_TO_TEST=""
+ENV SHA_TO_TEST=""
+
+ENV NODES_USERNAME="root"
+ENV NODES_PASSWORD=""
+ENV TESTS_TO_RUN="30"
+ENV TIME_LIMIT="30"
+
+
+# volumes
+ENV NODES_FILE_PATH="/nodes.txt"
+ENV TEST_OUTPUT="/test_output"
+
+RUN mkdir "/root/.ssh"
+RUN touch "/root/.ssh/known_hosts"
+
+# install java
+RUN apt-get update && apt-get install default-jre default-jdk libjna-java libjna-jni ssh gnuplot graphviz --yes --no-install-recommends
+
+# install clojure
+RUN curl -O "https://download.clojure.org/install/linux-install-${CLOJURE_VERSION}.sh" && \
+ chmod +x "linux-install-${CLOJURE_VERSION}.sh" && \
+ bash "./linux-install-${CLOJURE_VERSION}.sh"
+
+# install leiningen
+RUN curl -O "https://raw.githubusercontent.com/technomancy/leiningen/stable/bin/lein" && \
+ chmod +x ./lein && \
+ mv ./lein /usr/bin
+
+COPY run.sh /
+
+CMD ["/bin/bash", "/run.sh"]
diff --git a/docker/test/keeper-jepsen/run.sh b/docker/test/keeper-jepsen/run.sh
new file mode 100644
index 00000000000..352585e16e3
--- /dev/null
+++ b/docker/test/keeper-jepsen/run.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+
+CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-11_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse"}
+CLICKHOUSE_REPO_PATH=${CLICKHOUSE_REPO_PATH:=""}
+
+
+if [ -z "$CLICKHOUSE_REPO_PATH" ]; then
+ CLICKHOUSE_REPO_PATH=ch
+ rm -rf ch ||:
+ mkdir ch ||:
+ wget -nv -nd -c "https://clickhouse-test-reports.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/repo/clickhouse_no_subs.tar.gz"
+ tar -C ch --strip-components=1 -xf clickhouse_no_subs.tar.gz
+ ls -lath ||:
+fi
+
+cd "$CLICKHOUSE_REPO_PATH/tests/jepsen.clickhouse-keeper"
+
+(lein run test-all --nodes-file "$NODES_FILE_PATH" --username "$NODES_USERNAME" --logging-json --password "$NODES_PASSWORD" --time-limit "$TIME_LIMIT" --concurrency 50 -r 50 --snapshot-distance 100 --stale-log-gap 100 --reserved-log-items 10 --lightweight-run --clickhouse-source "$CLICKHOUSE_PACKAGE" -q --test-count "$TESTS_TO_RUN" || true) | tee "$TEST_OUTPUT/jepsen_run_all_tests.log"
+
+mv store "$TEST_OUTPUT/"
diff --git a/docker/test/performance-comparison/config/config.d/zzz-perf-comparison-tweaks-config.xml b/docker/test/performance-comparison/config/config.d/zzz-perf-comparison-tweaks-config.xml
index 31f5b739c6d..7b941f844de 100644
--- a/docker/test/performance-comparison/config/config.d/zzz-perf-comparison-tweaks-config.xml
+++ b/docker/test/performance-comparison/config/config.d/zzz-perf-comparison-tweaks-config.xml
@@ -1,6 +1,7 @@
+
diff --git a/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml b/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml
index 41bc7f777bf..63e23d8453c 100644
--- a/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml
+++ b/docker/test/performance-comparison/config/users.d/perf-comparison-tweaks-users.xml
@@ -17,6 +17,9 @@
12
+
+
+ 64Mi
diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py
index 4727f485943..2588b9f4213 100755
--- a/docker/test/performance-comparison/perf.py
+++ b/docker/test/performance-comparison/perf.py
@@ -66,7 +66,12 @@ reportStageEnd('parse')
subst_elems = root.findall('substitutions/substitution')
available_parameters = {} # { 'table': ['hits_10m', 'hits_100m'], ... }
for e in subst_elems:
- available_parameters[e.find('name').text] = [v.text for v in e.findall('values/value')]
+ name = e.find('name').text
+ values = [v.text for v in e.findall('values/value')]
+ if not values:
+ raise Exception(f'No values given for substitution {{{name}}}')
+
+ available_parameters[name] = values
# Takes parallel lists of templates, substitutes them with all combos of
# parameters. The set of parameters is determined based on the first list.
diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh
index 9e210dc92a2..8d865431570 100755
--- a/docker/test/stateful/run.sh
+++ b/docker/test/stateful/run.sh
@@ -21,14 +21,14 @@ function start()
-- --path /var/lib/clickhouse1/ --logger.stderr /var/log/clickhouse-server/stderr1.log \
--logger.log /var/log/clickhouse-server/clickhouse-server1.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server1.err.log \
--tcp_port 19000 --tcp_port_secure 19440 --http_port 18123 --https_port 18443 --interserver_http_port 19009 --tcp_with_proxy_port 19010 \
- --mysql_port 19004 \
+ --mysql_port 19004 --postgresql_port 19005 \
--keeper_server.tcp_port 19181 --keeper_server.server_id 2
sudo -E -u clickhouse /usr/bin/clickhouse server --config /etc/clickhouse-server2/config.xml --daemon \
-- --path /var/lib/clickhouse2/ --logger.stderr /var/log/clickhouse-server/stderr2.log \
--logger.log /var/log/clickhouse-server/clickhouse-server2.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server2.err.log \
--tcp_port 29000 --tcp_port_secure 29440 --http_port 28123 --https_port 28443 --interserver_http_port 29009 --tcp_with_proxy_port 29010 \
- --mysql_port 29004 \
+ --mysql_port 29004 --postgresql_port 29005 \
--keeper_server.tcp_port 29181 --keeper_server.server_id 3
fi
diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile
index 61d1b2f4849..658ae1f27ba 100644
--- a/docker/test/stateless/Dockerfile
+++ b/docker/test/stateless/Dockerfile
@@ -28,7 +28,8 @@ RUN apt-get update -y \
tree \
unixodbc \
wget \
- mysql-client=5.7*
+ mysql-client=5.7* \
+ postgresql-client
RUN pip3 install numpy scipy pandas
diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index 20132eafb75..e6f2d678aa9 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -44,7 +44,7 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
-- --path /var/lib/clickhouse1/ --logger.stderr /var/log/clickhouse-server/stderr1.log \
--logger.log /var/log/clickhouse-server/clickhouse-server1.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server1.err.log \
--tcp_port 19000 --tcp_port_secure 19440 --http_port 18123 --https_port 18443 --interserver_http_port 19009 --tcp_with_proxy_port 19010 \
- --mysql_port 19004 \
+ --mysql_port 19004 --postgresql_port 19005 \
--keeper_server.tcp_port 19181 --keeper_server.server_id 2 \
--macros.replica r2 # It doesn't work :(
@@ -52,7 +52,7 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
-- --path /var/lib/clickhouse2/ --logger.stderr /var/log/clickhouse-server/stderr2.log \
--logger.log /var/log/clickhouse-server/clickhouse-server2.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server2.err.log \
--tcp_port 29000 --tcp_port_secure 29440 --http_port 28123 --https_port 28443 --interserver_http_port 29009 --tcp_with_proxy_port 29010 \
- --mysql_port 29004 \
+ --mysql_port 29004 --postgresql_port 29005 \
--keeper_server.tcp_port 29181 --keeper_server.server_id 3 \
--macros.shard s2 # It doesn't work :(
@@ -104,6 +104,12 @@ clickhouse-client -q "system flush logs" ||:
pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.gz &
clickhouse-client -q "select * from system.query_log format TSVWithNamesAndTypes" | pigz > /test_output/query-log.tsv.gz &
clickhouse-client -q "select * from system.query_thread_log format TSVWithNamesAndTypes" | pigz > /test_output/query-thread-log.tsv.gz &
+clickhouse-client --allow_introspection_functions=1 -q "
+ WITH
+ arrayMap(x -> concat(demangle(addressToSymbol(x)), ':', addressToLine(x)), trace) AS trace_array,
+ arrayStringConcat(trace_array, '\n') AS trace_string
+ SELECT * EXCEPT(trace), trace_string FROM system.trace_log FORMAT TSVWithNamesAndTypes
+" | pigz > /test_output/trace-log.tsv.gz &
wait ||:
mv /var/log/clickhouse-server/stderr.log /test_output/ ||:
@@ -112,10 +118,13 @@ if [[ -n "$WITH_COVERAGE" ]] && [[ "$WITH_COVERAGE" -eq 1 ]]; then
fi
tar -chf /test_output/text_log_dump.tar /var/lib/clickhouse/data/system/text_log ||:
tar -chf /test_output/query_log_dump.tar /var/lib/clickhouse/data/system/query_log ||:
+tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||:
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
pigz < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.gz ||:
pigz < /var/log/clickhouse-server/clickhouse-server2.log > /test_output/clickhouse-server2.log.gz ||:
mv /var/log/clickhouse-server/stderr1.log /test_output/ ||:
mv /var/log/clickhouse-server/stderr2.log /test_output/ ||:
+ tar -chf /test_output/coordination1.tar /var/lib/clickhouse1/coordination ||:
+ tar -chf /test_output/coordination2.tar /var/lib/clickhouse2/coordination ||:
fi
diff --git a/docker/test/stateless_unbundled/Dockerfile b/docker/test/stateless_unbundled/Dockerfile
index 9efe08dbf23..c5463ac447d 100644
--- a/docker/test/stateless_unbundled/Dockerfile
+++ b/docker/test/stateless_unbundled/Dockerfile
@@ -14,9 +14,7 @@ RUN apt-get --allow-unauthenticated update -y \
expect \
gdb \
gperf \
- gperf \
heimdal-multidev \
- intel-opencl-icd \
libboost-filesystem-dev \
libboost-iostreams-dev \
libboost-program-options-dev \
@@ -50,9 +48,7 @@ RUN apt-get --allow-unauthenticated update -y \
moreutils \
ncdu \
netcat-openbsd \
- ocl-icd-libopencl1 \
odbcinst \
- opencl-headers \
openssl \
perl \
pigz \
diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh
index 3594eead992..74a88df21e0 100755
--- a/docker/test/stress/run.sh
+++ b/docker/test/stress/run.sh
@@ -108,6 +108,11 @@ zgrep -Fav "ASan doesn't fully support makecontext/swapcontext functions" > /dev
|| echo -e 'No sanitizer asserts\tOK' >> /test_output/test_results.tsv
rm -f /test_output/tmp
+# OOM
+zgrep -Fa " Application: Child process was terminated by signal 9" /var/log/clickhouse-server/clickhouse-server.log > /dev/null \
+ && echo -e 'OOM killer (or signal 9) in clickhouse-server.log\tFAIL' >> /test_output/test_results.tsv \
+ || echo -e 'No OOM messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
+
# Logical errors
zgrep -Fa "Code: 49, e.displayText() = DB::Exception:" /var/log/clickhouse-server/clickhouse-server.log > /dev/null \
&& echo -e 'Logical error thrown (see clickhouse-server.log)\tFAIL' >> /test_output/test_results.tsv \
@@ -118,7 +123,7 @@ zgrep -Fa "########################################" /var/log/clickhouse-server/
&& echo -e 'Killed by signal (in clickhouse-server.log)\tFAIL' >> /test_output/test_results.tsv \
|| echo -e 'Not crashed\tOK' >> /test_output/test_results.tsv
-# It also checks for OOM or crash without stacktrace (printed by watchdog)
+# It also checks for crash without stacktrace (printed by watchdog)
zgrep -Fa " " /var/log/clickhouse-server/clickhouse-server.log > /dev/null \
&& echo -e 'Fatal message in clickhouse-server.log\tFAIL' >> /test_output/test_results.tsv \
|| echo -e 'No fatal messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
@@ -131,6 +136,7 @@ pigz < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhous
tar -chf /test_output/coordination.tar /var/lib/clickhouse/coordination ||:
mv /var/log/clickhouse-server/stderr.log /test_output/
tar -chf /test_output/query_log_dump.tar /var/lib/clickhouse/data/system/query_log ||:
+tar -chf /test_output/trace_log_dump.tar /var/lib/clickhouse/data/system/trace_log ||:
# Write check result into check_status.tsv
clickhouse-local --structure "test String, res String" -q "SELECT 'failure', test FROM table WHERE res != 'OK' order by (lower(test) like '%hung%') LIMIT 1" < /test_output/test_results.tsv > /test_output/check_status.tsv
diff --git a/docker/test/stress/stress b/docker/test/stress/stress
index 25a705ecbd1..4fbedceb0b8 100755
--- a/docker/test/stress/stress
+++ b/docker/test/stress/stress
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from multiprocessing import cpu_count
-from subprocess import Popen, call, STDOUT
+from subprocess import Popen, call, check_output, STDOUT
import os
import sys
import shutil
@@ -85,10 +85,27 @@ def prepare_for_hung_check():
# Issue #21004, live views are experimental, so let's just suppress it
call("""clickhouse client -q "KILL QUERY WHERE upper(query) LIKE 'WATCH %'" """, shell=True, stderr=STDOUT)
- # Wait for last queries to finish if any, not longer than 120 seconds
+ # Kill other queries which known to be slow
+ # It's query from 01232_preparing_sets_race_condition_long, it may take up to 1000 seconds in slow builds
+ call("""clickhouse client -q "KILL QUERY WHERE query LIKE 'insert into tableB select %'" """, shell=True, stderr=STDOUT)
+ # Long query from 00084_external_agregation
+ call("""clickhouse client -q "KILL QUERY WHERE query LIKE 'SELECT URL, uniq(SearchPhrase) AS u FROM test.hits GROUP BY URL ORDER BY u %'" """, shell=True, stderr=STDOUT)
+
+ # Wait for last queries to finish if any, not longer than 300 seconds
call("""clickhouse client -q "select sleepEachRow((
- select maxOrDefault(120 - elapsed) + 1 from system.processes where query not like '%from system.processes%' and elapsed < 120
- ) / 120) from numbers(120) format Null" """, shell=True, stderr=STDOUT)
+ select maxOrDefault(300 - elapsed) + 1 from system.processes where query not like '%from system.processes%' and elapsed < 300
+ ) / 300) from numbers(300) format Null" """, shell=True, stderr=STDOUT)
+
+ # Even if all clickhouse-test processes are finished, there are probably some sh scripts,
+ # which still run some new queries. Let's ignore them.
+ try:
+ query = """clickhouse client -q "SELECT count() FROM system.processes where where elapsed > 300" """
+ output = check_output(query, shell=True, stderr=STDOUT).decode('utf-8').strip()
+ if int(output) == 0:
+ return False
+ except:
+ pass
+ return True
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
@@ -119,12 +136,12 @@ if __name__ == "__main__":
logging.info("All processes finished")
if args.hung_check:
- prepare_for_hung_check()
+ have_long_running_queries = prepare_for_hung_check()
logging.info("Checking if some queries hung")
cmd = "{} {} {}".format(args.test_cmd, "--hung-check", "00001_select_1")
res = call(cmd, shell=True, stderr=STDOUT)
hung_check_status = "No queries hung\tOK\n"
- if res != 0:
+ if res != 0 and have_long_running_queries:
logging.info("Hung check failed with exit code {}".format(res))
hung_check_status = "Hung check failed\tFAIL\n"
open(os.path.join(args.output_folder, "test_results.tsv"), 'w+').write(hung_check_status)
diff --git a/docs/en/commercial/cloud.md b/docs/en/commercial/cloud.md
index 91d2061c0af..953a0ab5748 100644
--- a/docs/en/commercial/cloud.md
+++ b/docs/en/commercial/cloud.md
@@ -31,9 +31,10 @@ toc_title: Cloud
## Alibaba Cloud {#alibaba-cloud}
-Alibaba Cloud Managed Service for ClickHouse [China Site](https://www.aliyun.com/product/clickhouse) (Will be available at international site at May, 2021) provides the following key features:
-- Highly reliable cloud disk storage engine based on Alibaba Cloud Apsara distributed system
-- Expand capacity on demand without manual data migration
+Alibaba Cloud Managed Service for ClickHouse. [China Site](https://www.aliyun.com/product/clickhouse) (will be available at the international site in May 2021). Provides the following key features:
+
+- Highly reliable cloud disk storage engine based on [Alibaba Cloud Apsara](https://www.alibabacloud.com/product/apsara-stack) distributed system
+- Expand capacity on-demand without manual data migration
- Support single-node, single-replica, multi-node, and multi-replica architectures, and support hot and cold data tiering
- Support access allow-list, one-key recovery, multi-layer network security protection, cloud disk encryption
- Seamless integration with cloud log systems, databases, and data application tools
diff --git a/docs/en/development/build-osx.md b/docs/en/development/build-osx.md
index 886e85bbf86..24ecbdc1c2c 100644
--- a/docs/en/development/build-osx.md
+++ b/docs/en/development/build-osx.md
@@ -5,12 +5,13 @@ toc_title: Build on Mac OS X
# How to Build ClickHouse on Mac OS X {#how-to-build-clickhouse-on-mac-os-x}
-Build should work on x86_64 (Intel) based macOS 10.15 (Catalina) and higher with recent Xcode's native AppleClang, or Homebrew's vanilla Clang or GCC compilers.
+Build should work on x86_64 (Intel) and arm64 (Apple Silicon) based macOS 10.15 (Catalina) and higher with recent Xcode's native AppleClang, or Homebrew's vanilla Clang or GCC compilers.
## Install Homebrew {#install-homebrew}
``` bash
-$ /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+# ...and follow the printed instructions on any additional steps required to complete the installation.
```
## Install Xcode and Command Line Tools {#install-xcode-and-command-line-tools}
@@ -22,8 +23,8 @@ Open it at least once to accept the end-user license agreement and automatically
Then, make sure that the latest Comman Line Tools are installed and selected in the system:
``` bash
-$ sudo rm -rf /Library/Developer/CommandLineTools
-$ sudo xcode-select --install
+sudo rm -rf /Library/Developer/CommandLineTools
+sudo xcode-select --install
```
Reboot.
@@ -31,14 +32,15 @@ Reboot.
## Install Required Compilers, Tools, and Libraries {#install-required-compilers-tools-and-libraries}
``` bash
-$ brew update
-$ brew install cmake ninja libtool gettext llvm gcc
+brew update
+brew install cmake ninja libtool gettext llvm gcc
```
## Checkout ClickHouse Sources {#checkout-clickhouse-sources}
``` bash
-$ git clone --recursive git@github.com:ClickHouse/ClickHouse.git # or https://github.com/ClickHouse/ClickHouse.git
+git clone --recursive git@github.com:ClickHouse/ClickHouse.git
+# ...alternatively, you can use https://github.com/ClickHouse/ClickHouse.git as the repo URL.
```
## Build ClickHouse {#build-clickhouse}
@@ -46,37 +48,37 @@ $ git clone --recursive git@github.com:ClickHouse/ClickHouse.git # or https://gi
To build using Xcode's native AppleClang compiler:
``` bash
-$ cd ClickHouse
-$ rm -rf build
-$ mkdir build
-$ cd build
-$ cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_JEMALLOC=OFF ..
-$ cmake --build . --config RelWithDebInfo
-$ cd ..
+cd ClickHouse
+rm -rf build
+mkdir build
+cd build
+cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
+cmake --build . --config RelWithDebInfo
+cd ..
```
To build using Homebrew's vanilla Clang compiler:
``` bash
-$ cd ClickHouse
-$ rm -rf build
-$ mkdir build
-$ cd build
-$ cmake -DCMAKE_C_COMPILER=$(brew --prefix llvm)/bin/clang -DCMAKE_CXX_COMPILER==$(brew --prefix llvm)/bin/clang++ -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_JEMALLOC=OFF ..
-$ cmake --build . --config RelWithDebInfo
-$ cd ..
+cd ClickHouse
+rm -rf build
+mkdir build
+cd build
+cmake -DCMAKE_C_COMPILER=$(brew --prefix llvm)/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix llvm)/bin/clang++ -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
+cmake --build . --config RelWithDebInfo
+cd ..
```
To build using Homebrew's vanilla GCC compiler:
``` bash
-$ cd ClickHouse
-$ rm -rf build
-$ mkdir build
-$ cd build
-$ cmake -DCMAKE_C_COMPILER=$(brew --prefix gcc)/bin/gcc-10 -DCMAKE_CXX_COMPILER=$(brew --prefix gcc)/bin/g++-10 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_JEMALLOC=OFF ..
-$ cmake --build . --config RelWithDebInfo
-$ cd ..
+cd ClickHouse
+rm -rf build
+mkdir build
+cd build
+cmake -DCMAKE_C_COMPILER=$(brew --prefix gcc)/bin/gcc-10 -DCMAKE_CXX_COMPILER=$(brew --prefix gcc)/bin/g++-10 -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
+cmake --build . --config RelWithDebInfo
+cd ..
```
## Caveats {#caveats}
@@ -115,7 +117,7 @@ To do so, create the `/Library/LaunchDaemons/limit.maxfiles.plist` file with the
Execute the following command:
``` bash
-$ sudo chown root:wheel /Library/LaunchDaemons/limit.maxfiles.plist
+sudo chown root:wheel /Library/LaunchDaemons/limit.maxfiles.plist
```
Reboot.
diff --git a/docs/en/development/build.md b/docs/en/development/build.md
index 3181f26800d..852b9de4fb3 100644
--- a/docs/en/development/build.md
+++ b/docs/en/development/build.md
@@ -27,53 +27,20 @@ Or cmake3 instead of cmake on older systems.
On Ubuntu/Debian you can use the automatic installation script (check [official webpage](https://apt.llvm.org/))
-```bash
+```bash
sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
```
For other Linux distribution - check the availability of the [prebuild packages](https://releases.llvm.org/download.html) or build clang [from sources](https://clang.llvm.org/get_started.html).
-#### Use clang-11 for Builds {#use-gcc-10-for-builds}
+#### Use clang-11 for Builds
``` bash
$ export CC=clang-11
$ export CXX=clang++-11
```
-### Install GCC 10 {#install-gcc-10}
-
-We recommend building ClickHouse with clang-11, GCC-10 also supported, but it is not used for production builds.
-
-If you want to use GCC-10 there are several ways to install it.
-
-#### Install from Repository {#install-from-repository}
-
-On Ubuntu 19.10 or newer:
-
- $ sudo apt-get update
- $ sudo apt-get install gcc-10 g++-10
-
-#### Install from a PPA Package {#install-from-a-ppa-package}
-
-On older Ubuntu:
-
-``` bash
-$ sudo apt-get install software-properties-common
-$ sudo apt-add-repository ppa:ubuntu-toolchain-r/test
-$ sudo apt-get update
-$ sudo apt-get install gcc-10 g++-10
-```
-
-#### Install from Sources {#install-from-sources}
-
-See [utils/ci/build-gcc-from-sources.sh](https://github.com/ClickHouse/ClickHouse/blob/master/utils/ci/build-gcc-from-sources.sh)
-
-#### Use GCC 10 for Builds {#use-gcc-10-for-builds}
-
-``` bash
-$ export CC=gcc-10
-$ export CXX=g++-10
-```
+Gcc can also be used though it is discouraged.
### Checkout ClickHouse Sources {#checkout-clickhouse-sources}
diff --git a/docs/en/development/contrib.md b/docs/en/development/contrib.md
index 76a2f647231..64ca2387029 100644
--- a/docs/en/development/contrib.md
+++ b/docs/en/development/contrib.md
@@ -5,36 +5,87 @@ toc_title: Third-Party Libraries Used
# Third-Party Libraries Used {#third-party-libraries-used}
-| Library | License |
-|---------------------|----------------------------------------------------------------------------------------------------------------------------------------------|
-| base64 | [BSD 2-Clause License](https://github.com/aklomp/base64/blob/a27c565d1b6c676beaf297fe503c4518185666f7/LICENSE) |
-| boost | [Boost Software License 1.0](https://github.com/ClickHouse-Extras/boost-extra/blob/6883b40449f378019aec792f9983ce3afc7ff16e/LICENSE_1_0.txt) |
-| brotli | [MIT](https://github.com/google/brotli/blob/master/LICENSE) |
-| capnproto | [MIT](https://github.com/capnproto/capnproto/blob/master/LICENSE) |
-| cctz | [Apache License 2.0](https://github.com/google/cctz/blob/4f9776a310f4952454636363def82c2bf6641d5f/LICENSE.txt) |
-| double-conversion | [BSD 3-Clause License](https://github.com/google/double-conversion/blob/cf2f0f3d547dc73b4612028a155b80536902ba02/LICENSE) |
-| FastMemcpy | [MIT](https://github.com/ClickHouse/ClickHouse/blob/master/libs/libmemcpy/impl/LICENSE) |
-| googletest | [BSD 3-Clause License](https://github.com/google/googletest/blob/master/LICENSE) |
-| h3 | [Apache License 2.0](https://github.com/uber/h3/blob/master/LICENSE) |
-| hyperscan | [BSD 3-Clause License](https://github.com/intel/hyperscan/blob/master/LICENSE) |
-| libcxxabi | [BSD + MIT](https://github.com/ClickHouse/ClickHouse/blob/master/libs/libglibc-compatibility/libcxxabi/LICENSE.TXT) |
-| libdivide | [Zlib License](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libdivide/LICENSE.txt) |
-| libgsasl | [LGPL v2.1](https://github.com/ClickHouse-Extras/libgsasl/blob/3b8948a4042e34fb00b4fb987535dc9e02e39040/LICENSE) |
-| libhdfs3 | [Apache License 2.0](https://github.com/ClickHouse-Extras/libhdfs3/blob/bd6505cbb0c130b0db695305b9a38546fa880e5a/LICENSE.txt) |
-| libmetrohash | [Apache License 2.0](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libmetrohash/LICENSE) |
-| libpcg-random | [Apache License 2.0](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/libpcg-random/LICENSE-APACHE.txt) |
-| libressl | [OpenSSL License](https://github.com/ClickHouse-Extras/ssl/blob/master/COPYING) |
-| librdkafka | [BSD 2-Clause License](https://github.com/edenhill/librdkafka/blob/363dcad5a23dc29381cc626620e68ae418b3af19/LICENSE) |
-| libwidechar_width | [CC0 1.0 Universal](https://github.com/ClickHouse/ClickHouse/blob/master/libs/libwidechar_width/LICENSE) |
-| llvm | [BSD 3-Clause License](https://github.com/ClickHouse-Extras/llvm/blob/163def217817c90fb982a6daf384744d8472b92b/llvm/LICENSE.TXT) |
-| lz4 | [BSD 2-Clause License](https://github.com/lz4/lz4/blob/c10863b98e1503af90616ae99725ecd120265dfb/LICENSE) |
-| mariadb-connector-c | [LGPL v2.1](https://github.com/ClickHouse-Extras/mariadb-connector-c/blob/3.1/COPYING.LIB) |
-| murmurhash | [Public Domain](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/murmurhash/LICENSE) |
-| pdqsort | [Zlib License](https://github.com/ClickHouse/ClickHouse/blob/master/contrib/pdqsort/license.txt) |
-| poco | [Boost Software License - Version 1.0](https://github.com/ClickHouse-Extras/poco/blob/fe5505e56c27b6ecb0dcbc40c49dc2caf4e9637f/LICENSE) |
-| protobuf | [BSD 3-Clause License](https://github.com/ClickHouse-Extras/protobuf/blob/12735370922a35f03999afff478e1c6d7aa917a4/LICENSE) |
-| re2 | [BSD 3-Clause License](https://github.com/google/re2/blob/7cf8b88e8f70f97fd4926b56aa87e7f53b2717e0/LICENSE) |
-| sentry-native | [MIT License](https://github.com/getsentry/sentry-native/blob/master/LICENSE) |
-| UnixODBC | [LGPL v2.1](https://github.com/ClickHouse-Extras/UnixODBC/tree/b0ad30f7f6289c12b76f04bfb9d466374bb32168) |
-| zlib-ng | [Zlib License](https://github.com/ClickHouse-Extras/zlib-ng/blob/develop/LICENSE.md) |
-| zstd | [BSD 3-Clause License](https://github.com/facebook/zstd/blob/dev/LICENSE) |
+The list of third-party libraries can be obtained by the following query:
+
+```
+SELECT library_name, license_type, license_path FROM system.licenses ORDER BY library_name COLLATE 'en'
+```
+
+[Example](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIGxpYnJhcnlfbmFtZSwgbGljZW5zZV90eXBlLCBsaWNlbnNlX3BhdGggRlJPTSBzeXN0ZW0ubGljZW5zZXMgT1JERVIgQlkgbGlicmFyeV9uYW1lIENPTExBVEUgJ2VuJw==)
+
+| library_name | license_type | license_path |
+|:-|:-|:-|
+| abseil-cpp | Apache | /contrib/abseil-cpp/LICENSE |
+| AMQP-CPP | Apache | /contrib/AMQP-CPP/LICENSE |
+| arrow | Apache | /contrib/arrow/LICENSE.txt |
+| avro | Apache | /contrib/avro/LICENSE.txt |
+| aws | Apache | /contrib/aws/LICENSE.txt |
+| aws-c-common | Apache | /contrib/aws-c-common/LICENSE |
+| aws-c-event-stream | Apache | /contrib/aws-c-event-stream/LICENSE |
+| aws-checksums | Apache | /contrib/aws-checksums/LICENSE |
+| base64 | BSD 2-clause | /contrib/base64/LICENSE |
+| boost | Boost | /contrib/boost/LICENSE_1_0.txt |
+| boringssl | BSD | /contrib/boringssl/LICENSE |
+| brotli | MIT | /contrib/brotli/LICENSE |
+| capnproto | MIT | /contrib/capnproto/LICENSE |
+| cassandra | Apache | /contrib/cassandra/LICENSE.txt |
+| cctz | Apache | /contrib/cctz/LICENSE.txt |
+| cityhash102 | MIT | /contrib/cityhash102/COPYING |
+| cppkafka | BSD 2-clause | /contrib/cppkafka/LICENSE |
+| croaring | Apache | /contrib/croaring/LICENSE |
+| curl | Apache | /contrib/curl/docs/LICENSE-MIXING.md |
+| cyrus-sasl | BSD 2-clause | /contrib/cyrus-sasl/COPYING |
+| double-conversion | BSD 3-clause | /contrib/double-conversion/LICENSE |
+| dragonbox | Apache | /contrib/dragonbox/LICENSE-Apache2-LLVM |
+| fast_float | Apache | /contrib/fast_float/LICENSE |
+| fastops | MIT | /contrib/fastops/LICENSE |
+| flatbuffers | Apache | /contrib/flatbuffers/LICENSE.txt |
+| fmtlib | Unknown | /contrib/fmtlib/LICENSE.rst |
+| gcem | Apache | /contrib/gcem/LICENSE |
+| googletest | BSD 3-clause | /contrib/googletest/LICENSE |
+| grpc | Apache | /contrib/grpc/LICENSE |
+| h3 | Apache | /contrib/h3/LICENSE |
+| hyperscan | Boost | /contrib/hyperscan/LICENSE |
+| icu | Public Domain | /contrib/icu/icu4c/LICENSE |
+| icudata | Public Domain | /contrib/icudata/LICENSE |
+| jemalloc | BSD 2-clause | /contrib/jemalloc/COPYING |
+| krb5 | MIT | /contrib/krb5/src/lib/gssapi/LICENSE |
+| libc-headers | LGPL | /contrib/libc-headers/LICENSE |
+| libcpuid | BSD 2-clause | /contrib/libcpuid/COPYING |
+| libcxx | Apache | /contrib/libcxx/LICENSE.TXT |
+| libcxxabi | Apache | /contrib/libcxxabi/LICENSE.TXT |
+| libdivide | zLib | /contrib/libdivide/LICENSE.txt |
+| libfarmhash | MIT | /contrib/libfarmhash/COPYING |
+| libgsasl | LGPL | /contrib/libgsasl/LICENSE |
+| libhdfs3 | Apache | /contrib/libhdfs3/LICENSE.txt |
+| libmetrohash | Apache | /contrib/libmetrohash/LICENSE |
+| libpq | Unknown | /contrib/libpq/COPYRIGHT |
+| libpqxx | BSD 3-clause | /contrib/libpqxx/COPYING |
+| librdkafka | MIT | /contrib/librdkafka/LICENSE.murmur2 |
+| libunwind | Apache | /contrib/libunwind/LICENSE.TXT |
+| libuv | BSD | /contrib/libuv/LICENSE |
+| llvm | Apache | /contrib/llvm/llvm/LICENSE.TXT |
+| lz4 | BSD | /contrib/lz4/LICENSE |
+| mariadb-connector-c | LGPL | /contrib/mariadb-connector-c/COPYING.LIB |
+| miniselect | Boost | /contrib/miniselect/LICENSE_1_0.txt |
+| msgpack-c | Boost | /contrib/msgpack-c/LICENSE_1_0.txt |
+| murmurhash | Public Domain | /contrib/murmurhash/LICENSE |
+| NuRaft | Apache | /contrib/NuRaft/LICENSE |
+| openldap | Unknown | /contrib/openldap/LICENSE |
+| orc | Apache | /contrib/orc/LICENSE |
+| poco | Boost | /contrib/poco/LICENSE |
+| protobuf | BSD 3-clause | /contrib/protobuf/LICENSE |
+| rapidjson | MIT | /contrib/rapidjson/bin/jsonschema/LICENSE |
+| re2 | BSD 3-clause | /contrib/re2/LICENSE |
+| replxx | BSD 3-clause | /contrib/replxx/LICENSE.md |
+| rocksdb | BSD 3-clause | /contrib/rocksdb/LICENSE.leveldb |
+| sentry-native | MIT | /contrib/sentry-native/LICENSE |
+| simdjson | Apache | /contrib/simdjson/LICENSE |
+| snappy | Public Domain | /contrib/snappy/COPYING |
+| sparsehash-c11 | BSD 3-clause | /contrib/sparsehash-c11/LICENSE |
+| stats | Apache | /contrib/stats/LICENSE |
+| thrift | Apache | /contrib/thrift/LICENSE |
+| unixodbc | LGPL | /contrib/unixodbc/COPYING |
+| xz | Public Domain | /contrib/xz/COPYING |
+| zlib-ng | zLib | /contrib/zlib-ng/LICENSE.md |
+| zstd | BSD | /contrib/zstd/LICENSE |
diff --git a/docs/en/development/developer-instruction.md b/docs/en/development/developer-instruction.md
index 5511e8e19c7..35ca4725af8 100644
--- a/docs/en/development/developer-instruction.md
+++ b/docs/en/development/developer-instruction.md
@@ -131,17 +131,18 @@ ClickHouse uses several external libraries for building. All of them do not need
## C++ Compiler {#c-compiler}
-Compilers GCC starting from version 10 and Clang version 8 or above are supported for building ClickHouse.
+Compilers Clang starting from version 11 is supported for building ClickHouse.
-Official Yandex builds currently use GCC because it generates machine code of slightly better performance (yielding a difference of up to several percent according to our benchmarks). And Clang is more convenient for development usually. Though, our continuous integration (CI) platform runs checks for about a dozen of build combinations.
+Clang should be used instead of gcc. Though, our continuous integration (CI) platform runs checks for about a dozen of build combinations.
-To install GCC on Ubuntu run: `sudo apt install gcc g++`
+On Ubuntu/Debian you can use the automatic installation script (check [official webpage](https://apt.llvm.org/))
-Check the version of gcc: `gcc --version`. If it is below 10, then follow the instruction here: https://clickhouse.tech/docs/en/development/build/#install-gcc-10.
+```bash
+sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
+```
-Mac OS X build is supported only for Clang. Just run `brew install llvm`
+Mac OS X build is also supported. Just run `brew install llvm`
-If you decide to use Clang, you can also install `libc++` and `lld`, if you know what it is. Using `ccache` is also recommended.
## The Building Process {#the-building-process}
@@ -152,14 +153,7 @@ Now that you are ready to build ClickHouse we recommend you to create a separate
You can have several different directories (build_release, build_debug, etc.) for different types of build.
-While inside the `build` directory, configure your build by running CMake. Before the first run, you need to define environment variables that specify compiler (version 10 gcc compiler in this example).
-
-Linux:
-
- export CC=gcc-10 CXX=g++-10
- cmake ..
-
-Mac OS X:
+While inside the `build` directory, configure your build by running CMake. Before the first run, you need to define environment variables that specify compiler.
export CC=clang CXX=clang++
cmake ..
diff --git a/docs/en/development/style.md b/docs/en/development/style.md
index 4c620c44aef..b27534d9890 100644
--- a/docs/en/development/style.md
+++ b/docs/en/development/style.md
@@ -701,7 +701,7 @@ But other things being equal, cross-platform or portable code is preferred.
**2.** Language: C++20 (see the list of available [C++20 features](https://en.cppreference.com/w/cpp/compiler_support#C.2B.2B20_features)).
-**3.** Compiler: `gcc`. At this time (August 2020), the code is compiled using version 9.3. (It can also be compiled using `clang 8`.)
+**3.** Compiler: `clang`. At this time (April 2021), the code is compiled using clang version 11. (It can also be compiled using `gcc` version 10, but it's untested and not suitable for production usage).
The standard library is used (`libc++`).
@@ -711,7 +711,7 @@ The standard library is used (`libc++`).
The CPU instruction set is the minimum supported set among our servers. Currently, it is SSE 4.2.
-**6.** Use `-Wall -Wextra -Werror` compilation flags.
+**6.** Use `-Wall -Wextra -Werror` compilation flags. Also `-Weverything` is used with few exceptions.
**7.** Use static linking with all libraries except those that are difficult to connect to statically (see the output of the `ldd` command).
diff --git a/docs/en/engines/database-engines/atomic.md b/docs/en/engines/database-engines/atomic.md
index d8ad18daec2..d897631dd6e 100644
--- a/docs/en/engines/database-engines/atomic.md
+++ b/docs/en/engines/database-engines/atomic.md
@@ -3,15 +3,52 @@ toc_priority: 32
toc_title: Atomic
---
-
# Atomic {#atomic}
-It supports non-blocking `DROP` and `RENAME TABLE` queries and atomic `EXCHANGE TABLES t1 AND t2` queries. `Atomic` database engine is used by default.
+It supports non-blocking [DROP TABLE](#drop-detach-table) and [RENAME TABLE](#rename-table) queries and atomic [EXCHANGE TABLES t1 AND t2](#exchange-tables) queries. `Atomic` database engine is used by default.
## Creating a Database {#creating-a-database}
-```sql
-CREATE DATABASE test ENGINE = Atomic;
+``` sql
+ CREATE DATABASE test[ ENGINE = Atomic];
```
-[Original article](https://clickhouse.tech/docs/en/engines/database-engines/atomic/)
+## Specifics and recommendations {#specifics-and-recommendations}
+
+### Table UUID {#table-uuid}
+
+All tables in database `Atomic` have persistent [UUID](../../sql-reference/data-types/uuid.md) and store data in directory `/clickhouse_path/store/xxx/xxxyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy/`, where `xxxyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy` is UUID of the table.
+Usually, the UUID is generated automatically, but the user can also explicitly specify the UUID in the same way when creating the table (this is not recommended). To display the `SHOW CREATE` query with the UUID you can use setting [show_table_uuid_in_table_create_query_if_not_nil](../../operations/settings/settings.md#show_table_uuid_in_table_create_query_if_not_nil). For example:
+
+```sql
+CREATE TABLE name UUID '28f1c61c-2970-457a-bffe-454156ddcfef' (n UInt64) ENGINE = ...;
+```
+### RENAME TABLE {#rename-table}
+
+`RENAME` queries are performed without changing UUID and moving table data. These queries do not wait for the completion of queries using the table and will be executed instantly.
+
+### DROP/DETACH TABLE {#drop-detach-table}
+
+On `DROP TABLE` no data is removed, database `Atomic` just marks table as dropped by moving metadata to `/clickhouse_path/metadata_dropped/` and notifies background thread. Delay before final table data deletion is specify by [database_atomic_delay_before_drop_table_sec](../../operations/server-configuration-parameters/settings.md#database_atomic_delay_before_drop_table_sec) setting.
+You can specify synchronous mode using `SYNC` modifier. Use the [database_atomic_wait_for_drop_and_detach_synchronously](../../operations/settings/settings.md#database_atomic_wait_for_drop_and_detach_synchronously) setting to do this. In this case `DROP` waits for running `SELECT`, `INSERT` and other queries which are using the table to finish. Table will be actually removed when it's not in use.
+
+### EXCHANGE TABLES {#exchange-tables}
+
+`EXCHANGE` query swaps tables atomically. So instead of this non-atomic operation:
+
+```sql
+RENAME TABLE new_table TO tmp, old_table TO new_table, tmp TO old_table;
+```
+you can use one atomic query:
+
+``` sql
+EXCHANGE TABLES new_table AND old_table;
+```
+
+### ReplicatedMergeTree in Atomic Database {#replicatedmergetree-in-atomic-database}
+
+For [ReplicatedMergeTree](../table-engines/mergetree-family/replication.md#table_engines-replication) tables is recomended do not specify parameters of engine - path in ZooKeeper and replica name. In this case will be used parameters of the configuration [default_replica_path](../../operations/server-configuration-parameters/settings.md#default_replica_path) and [default_replica_name](../../operations/server-configuration-parameters/settings.md#default_replica_name). If you want specify parameters of engine explicitly than recomended to use {uuid} macros. This is useful so that unique paths are automatically generated for each table in the ZooKeeper.
+
+## See Also
+
+- [system.databases](../../operations/system-tables/databases.md) system table
diff --git a/docs/en/engines/table-engines/integrations/postgresql.md b/docs/en/engines/table-engines/integrations/postgresql.md
index ad5bebb3dea..4474b764d2e 100644
--- a/docs/en/engines/table-engines/integrations/postgresql.md
+++ b/docs/en/engines/table-engines/integrations/postgresql.md
@@ -94,10 +94,10 @@ postgres=# INSERT INTO test (int_id, str, "float") VALUES (1,'test',2);
INSERT 0 1
postgresql> SELECT * FROM test;
- int_id | int_nullable | float | str | float_nullable
---------+--------------+-------+------+----------------
- 1 | | 2 | test |
-(1 row)
+ int_id | int_nullable | float | str | float_nullable
+ --------+--------------+-------+------+----------------
+ 1 | | 2 | test |
+ (1 row)
```
Table in ClickHouse, retrieving data from the PostgreSQL table created above:
diff --git a/docs/en/engines/table-engines/integrations/s3.md b/docs/en/engines/table-engines/integrations/s3.md
index 3d02aa13812..6592f8b9752 100644
--- a/docs/en/engines/table-engines/integrations/s3.md
+++ b/docs/en/engines/table-engines/integrations/s3.md
@@ -19,26 +19,26 @@ ENGINE = S3(path, [aws_access_key_id, aws_secret_access_key,] format, structure,
- `path` — Bucket url with path to file. Supports following wildcards in readonly mode: `*`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc'`, `'def'` — strings. For more information see [below](#wildcards-in-path).
- `format` — The [format](../../../interfaces/formats.md#formats) of the file.
- `structure` — Structure of the table. Format `'column1_name column1_type, column2_name column2_type, ...'`.
-- `compression` — Compression type. Supported values: none, gzip/gz, brotli/br, xz/LZMA, zstd/zst. Parameter is optional. By default, it will autodetect compression by file extension.
+- `compression` — Compression type. Supported values: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. Parameter is optional. By default, it will autodetect compression by file extension.
-**Example:**
+**Example**
-**1.** Set up the `s3_engine_table` table:
+1. Set up the `s3_engine_table` table:
-```sql
-CREATE TABLE s3_engine_table (name String, value UInt32) ENGINE=S3('https://storage.yandexcloud.net/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip')
+``` sql
+CREATE TABLE s3_engine_table (name String, value UInt32) ENGINE=S3('https://storage.yandexcloud.net/my-test-bucket-768/test-data.csv.gz', 'CSV', 'name String, value UInt32', 'gzip');
```
-**2.** Fill file:
+2. Fill file:
-```sql
-INSERT INTO s3_engine_table VALUES ('one', 1), ('two', 2), ('three', 3)
+``` sql
+INSERT INTO s3_engine_table VALUES ('one', 1), ('two', 2), ('three', 3);
```
-**3.** Query the data:
+3. Query the data:
-```sql
-SELECT * FROM s3_engine_table LIMIT 2
+``` sql
+SELECT * FROM s3_engine_table LIMIT 2;
```
```text
@@ -73,13 +73,63 @@ For more information about virtual columns see [here](../../../engines/table-eng
Constructions with `{}` are similar to the [remote](../../../sql-reference/table-functions/remote.md) table function.
-## S3-related Settings {#s3-settings}
+**Example**
+
+1. Suppose we have several files in CSV format with the following URIs on S3:
+
+- ‘https://storage.yandexcloud.net/my-test-bucket-768/some_prefix/some_file_1.csv’
+- ‘https://storage.yandexcloud.net/my-test-bucket-768/some_prefix/some_file_2.csv’
+- ‘https://storage.yandexcloud.net/my-test-bucket-768/some_prefix/some_file_3.csv’
+- ‘https://storage.yandexcloud.net/my-test-bucket-768/another_prefix/some_file_1.csv’
+- ‘https://storage.yandexcloud.net/my-test-bucket-768/another_prefix/some_file_2.csv’
+- ‘https://storage.yandexcloud.net/my-test-bucket-768/another_prefix/some_file_3.csv’
+
+There are several ways to make a table consisting of all six files:
+
+The first way:
+
+``` sql
+CREATE TABLE table_with_range (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/some_file_{1..3}', 'CSV');
+```
+
+Another way:
+
+``` sql
+CREATE TABLE table_with_question_mark (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/some_file_?', 'CSV');
+```
+
+Table consists of all the files in both directories (all files should satisfy format and schema described in query):
+
+``` sql
+CREATE TABLE table_with_asterisk (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/*', 'CSV');
+```
+
+If the listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`.
+
+**Example**
+
+Create table with files named `file-000.csv`, `file-001.csv`, … , `file-999.csv`:
+
+``` sql
+CREATE TABLE big_table (name String, value UInt32) ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/big_prefix/file-{000..999}.csv', 'CSV');
+```
+
+## Virtual Columns {#virtual-columns}
+
+- `_path` — Path to the file.
+- `_file` — Name of the file.
+
+**See Also**
+
+- [Virtual columns](../../../engines/table-engines/index.md#table_engines-virtual_columns)
+
+## S3-related settings {#settings}
The following settings can be set before query execution or placed into configuration file.
-- `s3_max_single_part_upload_size` — The maximum size of object to upload using singlepart upload to S3. Default value is `64Mb`.
+- `s3_max_single_part_upload_size` — The maximum size of object to upload using singlepart upload to S3. Default value is `64Mb`.
- `s3_min_upload_part_size` — The minimum size of part to upload during multipart upload to [S3 Multipart upload](https://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html). Default value is `512Mb`.
-- `s3_max_redirects` — Max number of S3 redirects hops allowed. Default value is `10`.
+- `s3_max_redirects` — Max number of S3 redirects hops allowed. Default value is `10`.
Security consideration: if malicious user can specify arbitrary S3 URLs, `s3_max_redirects` must be set to zero to avoid [SSRF](https://en.wikipedia.org/wiki/Server-side_request_forgery) attacks; or alternatively, `remote_host_filter` must be specified in server configuration.
@@ -90,6 +140,7 @@ The following settings can be specified in configuration file for given endpoint
- `endpoint` — Specifies prefix of an endpoint. Mandatory.
- `access_key_id` and `secret_access_key` — Specifies credentials to use with given endpoint. Optional.
- `use_environment_credentials` — If set to `true`, S3 client will try to obtain credentials from environment variables and Amazon EC2 metadata for given endpoint. Optional, default value is `false`.
+- `use_insecure_imds_request` — If set to `true`, S3 client will use insecure IMDS request while obtaining credentials from Amazon EC2 metadata. Optional, default value is `false`.
- `header` — Adds specified HTTP header to a request to given endpoint. Optional, can be speficied multiple times.
- `server_side_encryption_customer_key_base64` — If specified, required headers for accessing S3 objects with SSE-C encryption will be set. Optional.
@@ -102,11 +153,13 @@ The following settings can be specified in configuration file for given endpoint
+
```
+
## Usage {#usage-examples}
Suppose we have several files in TSV format with the following URIs on HDFS:
@@ -149,8 +202,7 @@ ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_p
CREATE TABLE big_table (name String, value UInt32)
ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/big_prefix/file-{000..999}.csv', 'CSV');
```
+
## See also
- [S3 table function](../../../sql-reference/table-functions/s3.md)
-
-[Original article](https://clickhouse.tech/docs/en/engines/table-engines/integrations/s3/)
diff --git a/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md b/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md
index 1a997b6b237..818830646cb 100644
--- a/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md
+++ b/docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md
@@ -3,7 +3,7 @@ toc_priority: 35
toc_title: AggregatingMergeTree
---
-# Aggregatingmergetree {#aggregatingmergetree}
+# AggregatingMergeTree {#aggregatingmergetree}
The engine inherits from [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md#table_engines-mergetree), altering the logic for data parts merging. ClickHouse replaces all rows with the same primary key (or more accurately, with the same [sorting key](../../../engines/table-engines/mergetree-family/mergetree.md)) with a single row (within a one data part) that stores a combination of states of aggregate functions.
diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md
index a24b7229d17..9874e87be78 100644
--- a/docs/en/engines/table-engines/mergetree-family/mergetree.md
+++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md
@@ -767,6 +767,7 @@ Required parameters:
Optional parameters:
- `use_environment_credentials` — Reads AWS credentials from the Environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY and AWS_SESSION_TOKEN if they exist. Default value is `false`.
+- `use_insecure_imds_request` — If set to `true`, S3 client will use insecure IMDS request while obtaining credentials from Amazon EC2 metadata. Default value is `false`.
- `proxy` — Proxy configuration for S3 endpoint. Each `uri` element inside `proxy` block should contain a proxy URL.
- `connect_timeout_ms` — Socket connect timeout in milliseconds. Default value is `10 seconds`.
- `request_timeout_ms` — Request timeout in milliseconds. Default value is `5 seconds`.
diff --git a/docs/en/engines/table-engines/special/buffer.md b/docs/en/engines/table-engines/special/buffer.md
index bf6c08f8f6c..8245cd19e8c 100644
--- a/docs/en/engines/table-engines/special/buffer.md
+++ b/docs/en/engines/table-engines/special/buffer.md
@@ -18,11 +18,17 @@ Engine parameters:
- `num_layers` – Parallelism layer. Physically, the table will be represented as `num_layers` of independent buffers. Recommended value: 16.
- `min_time`, `max_time`, `min_rows`, `max_rows`, `min_bytes`, and `max_bytes` – Conditions for flushing data from the buffer.
+Optional engine parameters:
+
+- `flush_time`, `flush_rows`, `flush_bytes` – Conditions for flushing data from the buffer, that will happen only in background (ommited or zero means no `flush*` parameters).
+
Data is flushed from the buffer and written to the destination table if all the `min*` conditions or at least one `max*` condition are met.
-- `min_time`, `max_time` – Condition for the time in seconds from the moment of the first write to the buffer.
-- `min_rows`, `max_rows` – Condition for the number of rows in the buffer.
-- `min_bytes`, `max_bytes` – Condition for the number of bytes in the buffer.
+Also if at least one `flush*` condition are met flush initiated in background, this is different from `max*`, since `flush*` allows you to configure background flushes separately to avoid adding latency for `INSERT` (into `Buffer`) queries.
+
+- `min_time`, `max_time`, `flush_time` – Condition for the time in seconds from the moment of the first write to the buffer.
+- `min_rows`, `max_rows`, `flush_rows` – Condition for the number of rows in the buffer.
+- `min_bytes`, `max_bytes`, `flush_bytes` – Condition for the number of bytes in the buffer.
During the write operation, data is inserted to a `num_layers` number of random buffers. Or, if the data part to insert is large enough (greater than `max_rows` or `max_bytes`), it is written directly to the destination table, omitting the buffer.
diff --git a/docs/en/getting-started/example-datasets/cell-towers.md b/docs/en/getting-started/example-datasets/cell-towers.md
index 76effdd4c62..7028b650ad1 100644
--- a/docs/en/getting-started/example-datasets/cell-towers.md
+++ b/docs/en/getting-started/example-datasets/cell-towers.md
@@ -3,31 +3,31 @@ toc_priority: 21
toc_title: Cell Towers
---
-# Cell Towers
+# Cell Towers {#cell-towers}
This dataset is from [OpenCellid](https://www.opencellid.org/) - The world's largest Open Database of Cell Towers.
-As of 2021 it contains more than 40 million records about cell towers (GSM, LTE, UMTS, etc.) around the world with their geographical coordinates and metadata (country code, network, etc).
+As of 2021, it contains more than 40 million records about cell towers (GSM, LTE, UMTS, etc.) around the world with their geographical coordinates and metadata (country code, network, etc).
-OpenCelliD Project is licensed under a Creative Commons Attribution-ShareAlike 4.0 International License, and we redistribute a snapshot of this dataset under the terms of the same license. The up to date version of the dataset is available to download after sign in.
+OpenCelliD Project is licensed under a Creative Commons Attribution-ShareAlike 4.0 International License, and we redistribute a snapshot of this dataset under the terms of the same license. The up-to-date version of the dataset is available to download after sign in.
-## Get the Dataset
+## Get the Dataset {#get-the-dataset}
-Download the snapshot of the dataset from Feb 2021: [https://datasets.clickhouse.tech/cell_towers.csv.xz] (729 MB).
+1. Download the snapshot of the dataset from February 2021: [https://datasets.clickhouse.tech/cell_towers.csv.xz] (729 MB).
-Optionally validate the integrity:
+2. Validate the integrity (optional step):
```
md5sum cell_towers.csv.xz
8cf986f4a0d9f12c6f384a0e9192c908 cell_towers.csv.xz
```
-Decompress it with the following command:
+3. Decompress it with the following command:
```
xz -d cell_towers.csv.xz
```
-Create a table:
+4. Create a table:
```
CREATE TABLE cell_towers
@@ -50,15 +50,15 @@ CREATE TABLE cell_towers
ENGINE = MergeTree ORDER BY (radio, mcc, net, created);
```
-Insert the dataset:
+5. Insert the dataset:
```
clickhouse-client --query "INSERT INTO cell_towers FORMAT CSVWithNames" < cell_towers.csv
```
+## Examples {#examples}
-## Run some queries
+1. A number of cell towers by type:
-Number of cell towers by type:
```
SELECT radio, count() AS c FROM cell_towers GROUP BY radio ORDER BY c DESC
@@ -73,7 +73,8 @@ SELECT radio, count() AS c FROM cell_towers GROUP BY radio ORDER BY c DESC
5 rows in set. Elapsed: 0.011 sec. Processed 43.28 million rows, 43.28 MB (3.83 billion rows/s., 3.83 GB/s.)
```
-Cell towers by mobile country code (MCC):
+2. Cell towers by [mobile country code (MCC)](https://en.wikipedia.org/wiki/Mobile_country_code):
+
```
SELECT mcc, count() FROM cell_towers GROUP BY mcc ORDER BY count() DESC LIMIT 10
@@ -93,28 +94,28 @@ SELECT mcc, count() FROM cell_towers GROUP BY mcc ORDER BY count() DESC LIMIT 10
10 rows in set. Elapsed: 0.019 sec. Processed 43.28 million rows, 86.55 MB (2.33 billion rows/s., 4.65 GB/s.)
```
-See the dictionary here: [https://en.wikipedia.org/wiki/Mobile_country_code](https://en.wikipedia.org/wiki/Mobile_country_code).
+So, the top countries are: the USA, Germany, and Russia.
-So, the top countries are USA, Germany and Russia.
-
-You may want to create an [External Dictionary](../../sql-reference/dictionaries/external-dictionaries/external-dicts/) in ClickHouse to decode these values.
+You may want to create an [External Dictionary](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md) in ClickHouse to decode these values.
-### Example of using `pointInPolygon` function
+## Use case {#use-case}
-Create a table where we will store polygons:
+Using `pointInPolygon` function.
+
+1. Create a table where we will store polygons:
```
CREATE TEMPORARY TABLE moscow (polygon Array(Tuple(Float64, Float64)));
```
-This is a rough shape of Moscow (without "new Moscow"):
+2. This is a rough shape of Moscow (without "new Moscow"):
```
INSERT INTO moscow VALUES ([(37.84172564285271, 55.78000432402266), (37.8381207618713, 55.775874525970494), (37.83979446823122, 55.775626746008065), (37.84243326983639, 55.77446586811748), (37.84262672750849, 55.771974101091104), (37.84153238623039, 55.77114545193181), (37.841124690460184, 55.76722010265554), (37.84239076983644, 55.76654891107098), (37.842283558197025, 55.76258709833121), (37.8421759312134, 55.758073999993734), (37.84198330422974, 55.75381499999371), (37.8416827275085, 55.749277102484484), (37.84157576190186, 55.74794544108413), (37.83897929098507, 55.74525257875241), (37.83739676451868, 55.74404373042019), (37.838732481460525, 55.74298009816793), (37.841183997352545, 55.743060321833575), (37.84097476190185, 55.73938799999373), (37.84048155819702, 55.73570799999372), (37.840095812164286, 55.73228210777237), (37.83983814285274, 55.73080491981639), (37.83846476321406, 55.729799917464675), (37.83835745269769, 55.72919751082619), (37.838636380279524, 55.72859509486539), (37.8395161005249, 55.727705075632784), (37.83897964285276, 55.722727886185154), (37.83862557539366, 55.72034817326636), (37.83559735744853, 55.71944437307499), (37.835370708803126, 55.71831419154461), (37.83738169402022, 55.71765218986692), (37.83823396494291, 55.71691750159089), (37.838056931213345, 55.71547311301385), (37.836812846557606, 55.71221445615604), (37.83522525396725, 55.709331054395555), (37.83269301586908, 55.70953687463627), (37.829667367706236, 55.70903403789297), (37.83311126588435, 55.70552351822608), (37.83058993121339, 55.70041317726053), (37.82983872750851, 55.69883771404813), (37.82934501586913, 55.69718947487017), (37.828926414016685, 55.69504441658371), (37.82876530422971, 55.69287499999378), (37.82894754100031, 55.690759754047335), (37.827697554878185, 55.68951421135665), (37.82447346292115, 55.68965045405069), (37.83136543914793, 55.68322046195302), (37.833554015869154, 55.67814012759211), (37.83544184655761, 55.67295011628339), (37.837480388885474, 55.6672498719639), (37.838960677246064, 55.66316274139358), (37.83926093121332, 55.66046999999383), (37.839025050262435, 55.65869897264431), (37.83670784390257, 55.65794084879904), (37.835656529083245, 55.65694309303843), (37.83704060449217, 55.65689306460552), (37.83696819873806, 55.65550363526252), (37.83760389616388, 55.65487847246661), (37.83687972750851, 55.65356745541324), (37.83515216004943, 55.65155951234079), (37.83312418518067, 55.64979413590619), (37.82801726983639, 55.64640836412121), (37.820614174591, 55.64164525405531), (37.818908190475426, 55.6421883258084), (37.81717543386075, 55.64112490388471), (37.81690987037274, 55.63916106913107), (37.815099354492155, 55.637925371757085), (37.808769150787356, 55.633798276884455), (37.80100123544311, 55.62873670012244), (37.79598013491824, 55.62554336109055), (37.78634567724606, 55.62033499605651), (37.78334147619623, 55.618768681480326), (37.77746201055901, 55.619855533402706), (37.77527329626457, 55.61909966711279), (37.77801986242668, 55.618770300976294), (37.778212973541216, 55.617257701952106), (37.77784818518065, 55.61574504433011), (37.77016867724609, 55.61148576294007), (37.760191219573976, 55.60599579539028), (37.75338926983641, 55.60227892751446), (37.746329965606634, 55.59920577639331), (37.73939925396728, 55.59631430313617), (37.73273665739439, 55.5935318803559), (37.7299954450912, 55.59350760316188), (37.7268679946899, 55.59469840523759), (37.72626726983634, 55.59229549697373), (37.7262673598022, 55.59081598950582), (37.71897193121335, 55.5877595845419), (37.70871550793456, 55.58393177431724), (37.700497489410374, 55.580917323756644), (37.69204305026244, 55.57778089778455), (37.68544477378839, 55.57815154690915), (37.68391050793454, 55.57472945079756), (37.678803592590306, 55.57328235936491), (37.6743402539673, 55.57255251445782), (37.66813862698363, 55.57216388774464), (37.617927457672096, 55.57505691895805), (37.60443099999999, 55.5757737568051), (37.599683515869145, 55.57749105910326), (37.59754177842709, 55.57796291823627), (37.59625834786988, 55.57906686095235), (37.59501783265684, 55.57746616444403), (37.593090671936025, 55.57671634534502), (37.587018007904, 55.577944600233785), (37.578692203704804, 55.57982895000019), (37.57327546607398, 55.58116294118248), (37.57385012109279, 55.581550362779), (37.57399562266922, 55.5820107079112), (37.5735356072979, 55.58226289171689), (37.57290393054962, 55.582393529795155), (37.57037722355653, 55.581919415056234), (37.5592298306885, 55.584471614867844), (37.54189249206543, 55.58867650795186), (37.5297256269836, 55.59158133551745), (37.517837865081766, 55.59443656218868), (37.51200186508174, 55.59635625174229), (37.506808949737554, 55.59907823904434), (37.49820432275389, 55.6062944994944), (37.494406071441674, 55.60967103463367), (37.494760001358024, 55.61066689753365), (37.49397137107085, 55.61220931698269), (37.49016528606031, 55.613417718449064), (37.48773249206542, 55.61530616333343), (37.47921386508177, 55.622640129112334), (37.470652153442394, 55.62993723476164), (37.46273446298218, 55.6368075123157), (37.46350692265317, 55.64068225239439), (37.46050283203121, 55.640794546982576), (37.457627470916734, 55.64118904154646), (37.450718034393326, 55.64690488145138), (37.44239252645875, 55.65397824729769), (37.434587576721185, 55.66053543155961), (37.43582144975277, 55.661693766520735), (37.43576786245721, 55.662755031737014), (37.430982915344174, 55.664610641628116), (37.428547447097685, 55.66778515273695), (37.42945134592044, 55.668633314343566), (37.42859571562949, 55.66948145750025), (37.4262836402282, 55.670813882451405), (37.418709037048295, 55.6811141674414), (37.41922139651101, 55.68235377885389), (37.419218771842885, 55.68359335082235), (37.417196501327446, 55.684375235224735), (37.41607020370478, 55.68540557585352), (37.415640857147146, 55.68686637150793), (37.414632153442334, 55.68903015131686), (37.413344899475064, 55.690896881757396), (37.41171432275391, 55.69264232162232), (37.40948282275393, 55.69455101638112), (37.40703674603271, 55.69638690385348), (37.39607169577025, 55.70451821283731), (37.38952706878662, 55.70942491932811), (37.387778313491815, 55.71149057784176), (37.39049275399779, 55.71419814298992), (37.385557272491454, 55.7155489617061), (37.38388335714726, 55.71849856042102), (37.378368238098155, 55.7292763261685), (37.37763597123337, 55.730845879211614), (37.37890062088197, 55.73167906388319), (37.37750451918789, 55.734703664681774), (37.375610832015965, 55.734851959522246), (37.3723813571472, 55.74105626086403), (37.37014935714723, 55.746115620904355), (37.36944173016362, 55.750883999993725), (37.36975304365541, 55.76335905525834), (37.37244070571134, 55.76432079697595), (37.3724259757175, 55.76636979670426), (37.369922155757884, 55.76735417953104), (37.369892695770275, 55.76823419316575), (37.370214730163575, 55.782312184391266), (37.370493611114505, 55.78436801120489), (37.37120164550783, 55.78596427165359), (37.37284851456452, 55.7874378183096), (37.37608325135799, 55.7886695054807), (37.3764587460632, 55.78947647305964), (37.37530000265506, 55.79146512926804), (37.38235915344241, 55.79899647809345), (37.384344043655396, 55.80113596939471), (37.38594269577028, 55.80322699999366), (37.38711208598329, 55.804919036911976), (37.3880239841309, 55.806610999993666), (37.38928977249147, 55.81001864976979), (37.39038389947512, 55.81348641242801), (37.39235781481933, 55.81983538336746), (37.393709457672124, 55.82417822811877), (37.394685720901464, 55.82792275755836), (37.39557615344238, 55.830447148154136), (37.39844478226658, 55.83167107969975), (37.40019761214057, 55.83151823557964), (37.400398790382326, 55.83264967594742), (37.39659544313046, 55.83322180909622), (37.39667059524539, 55.83402792148566), (37.39682089947515, 55.83638877400216), (37.39643489154053, 55.83861656112751), (37.3955338994751, 55.84072348043264), (37.392680272491454, 55.84502158126453), (37.39241188227847, 55.84659117913199), (37.392529730163616, 55.84816071336481), (37.39486835714723, 55.85288092980303), (37.39873052645878, 55.859893456073635), (37.40272161111449, 55.86441833633205), (37.40697072750854, 55.867579567544375), (37.410007082016016, 55.868369880337), (37.4120992989502, 55.86920843741314), (37.412668021163924, 55.87055369615854), (37.41482461111453, 55.87170587948249), (37.41862266137694, 55.873183961039565), (37.42413732540892, 55.874879126654704), (37.4312182698669, 55.875614937236705), (37.43111093783558, 55.8762723478417), (37.43332105622856, 55.87706546369396), (37.43385747619623, 55.87790681284802), (37.441303050262405, 55.88027084462084), (37.44747234260555, 55.87942070143253), (37.44716141796871, 55.88072960917233), (37.44769797085568, 55.88121221323979), (37.45204320500181, 55.882080694420715), (37.45673176190186, 55.882346110794586), (37.463383999999984, 55.88252729504517), (37.46682797486874, 55.88294937719063), (37.470014457672086, 55.88361266759345), (37.47751410450743, 55.88546991372396), (37.47860317658232, 55.88534929207307), (37.48165826025772, 55.882563306475106), (37.48316434442331, 55.8815803226785), (37.483831555817645, 55.882427612793315), (37.483182967125686, 55.88372791409729), (37.483092277908824, 55.88495581062434), (37.4855716508179, 55.8875561994203), (37.486440636245746, 55.887827444039566), (37.49014203439328, 55.88897899871799), (37.493210285705544, 55.890208937135604), (37.497512451065035, 55.891342397444696), (37.49780744510645, 55.89174030252967), (37.49940333499519, 55.89239745507079), (37.50018383334346, 55.89339220941865), (37.52421672750851, 55.903869074155224), (37.52977457672118, 55.90564076517974), (37.53503220370484, 55.90661661218259), (37.54042858064267, 55.90714113744566), (37.54320461007303, 55.905645048442985), (37.545686966066306, 55.906608607018505), (37.54743976120755, 55.90788552162358), (37.55796999999999, 55.90901557907218), (37.572711542327866, 55.91059395704873), (37.57942799999998, 55.91073854155573), (37.58502865872187, 55.91009969268444), (37.58739968913264, 55.90794809960554), (37.59131567193598, 55.908713267595054), (37.612687423278814, 55.902866854295375), (37.62348079629517, 55.90041967242986), (37.635797880950896, 55.898141151686396), (37.649487626983664, 55.89639275532968), (37.65619302513125, 55.89572360207488), (37.66294133862307, 55.895295577183965), (37.66874564418033, 55.89505457604897), (37.67375601586915, 55.89254677027454), (37.67744661901856, 55.8947775867987), (37.688347, 55.89450045676125), (37.69480554232789, 55.89422926332761), (37.70107096560668, 55.89322256101114), (37.705962965606716, 55.891763491662616), (37.711885134918205, 55.889110234998974), (37.71682005026245, 55.886577568759876), (37.7199315476074, 55.88458159806678), (37.72234560316464, 55.882281005794134), (37.72364385977171, 55.8809452036196), (37.725371142837474, 55.8809722706006), (37.727870902099546, 55.88037213862385), (37.73394330422971, 55.877941504088696), (37.745339592590376, 55.87208120378722), (37.75525267724611, 55.86703807949492), (37.76919976190188, 55.859821640197474), (37.827835219574, 55.82962968399116), (37.83341438888553, 55.82575289922351), (37.83652584655761, 55.82188784027888), (37.83809213491821, 55.81612575504693), (37.83605359521481, 55.81460347077685), (37.83632178569025, 55.81276696067908), (37.838623105812026, 55.811486181656385), (37.83912198147584, 55.807329380532785), (37.839079078033414, 55.80510270463816), (37.83965844708251, 55.79940712529036), (37.840581150787344, 55.79131399999368), (37.84172564285271, 55.78000432402266)]);
```
-Check how many cell towers are in Moscow:
+3. Check how many cell towers are in Moscow:
```
SELECT count() FROM cell_towers WHERE pointInPolygon((lon, lat), (SELECT * FROM moscow))
@@ -128,6 +129,4 @@ SELECT count() FROM cell_towers WHERE pointInPolygon((lon, lat), (SELECT * FROM
The data is also available for interactive queries in the [Playground](https://gh-api.clickhouse.tech/play?user=play), [example](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIG1jYywgY291bnQoKSBGUk9NIGNlbGxfdG93ZXJzIEdST1VQIEJZIG1jYyBPUkRFUiBCWSBjb3VudCgpIERFU0M=).
-Although you cannot create temporary tables there.
-
-[Original article](https://clickhouse.tech/docs/en/getting_started/example_datasets/cell-towers/)
+Although you cannot create temporary tables there.
\ No newline at end of file
diff --git a/docs/en/guides/apply-catboost-model.md b/docs/en/guides/apply-catboost-model.md
index f614b121714..7c2c8a575ec 100644
--- a/docs/en/guides/apply-catboost-model.md
+++ b/docs/en/guides/apply-catboost-model.md
@@ -159,6 +159,9 @@ The fastest way to evaluate a CatBoost model is compile `libcatboostmodel./home/catboost/models/*_model.xml
```
+!!! note "Note"
+ You can change path to the CatBoost model configuration later without restarting server.
+
## 4. Run the Model Inference from SQL {#run-model-inference}
For test model run the ClickHouse client `$ clickhouse client`.
diff --git a/docs/en/interfaces/third-party/client-libraries.md b/docs/en/interfaces/third-party/client-libraries.md
index c08eec61b1c..f5c85289171 100644
--- a/docs/en/interfaces/third-party/client-libraries.md
+++ b/docs/en/interfaces/third-party/client-libraries.md
@@ -23,6 +23,7 @@ toc_title: Client Libraries
- [SeasClick C++ client](https://github.com/SeasX/SeasClick)
- [one-ck](https://github.com/lizhichao/one-ck)
- [glushkovds/phpclickhouse-laravel](https://packagist.org/packages/glushkovds/phpclickhouse-laravel)
+ - [kolya7k ClickHouse PHP extension](https://github.com//kolya7k/clickhouse-php)
- Go
- [clickhouse](https://github.com/kshvakov/clickhouse/)
- [go-clickhouse](https://github.com/roistat/go-clickhouse)
diff --git a/docs/en/interfaces/third-party/gui.md b/docs/en/interfaces/third-party/gui.md
index 5d14b3aa3cc..e54e40441ca 100644
--- a/docs/en/interfaces/third-party/gui.md
+++ b/docs/en/interfaces/third-party/gui.md
@@ -169,19 +169,21 @@ Features:
### SeekTable {#seektable}
-[SeekTable](https://www.seektable.com) is a self-service BI tool for data exploration and operational reporting. SeekTable is available both as a cloud service and a self-hosted version. SeekTable reports may be embedded into any web-app.
+[SeekTable](https://www.seektable.com) is a self-service BI tool for data exploration and operational reporting. It is available both as a cloud service and a self-hosted version. Reports from SeekTable may be embedded into any web-app.
Features:
- Business users-friendly reports builder.
- Powerful report parameters for SQL filtering and report-specific query customizations.
- Can connect to ClickHouse both with a native TCP/IP endpoint and a HTTP(S) interface (2 different drivers).
-- It is possible to use all power of CH SQL dialect in dimensions/measures definitions
+- It is possible to use all power of ClickHouse SQL dialect in dimensions/measures definitions.
- [Web API](https://www.seektable.com/help/web-api-integration) for automated reports generation.
-- Supports reports development flow with account data [backup/restore](https://www.seektable.com/help/self-hosted-backup-restore), data models (cubes) / reports configuration is a human-readable XML and can be stored under version control.
+- Supports reports development flow with account data [backup/restore](https://www.seektable.com/help/self-hosted-backup-restore); data models (cubes) / reports configuration is a human-readable XML and can be stored under version control system.
SeekTable is [free](https://www.seektable.com/help/cloud-pricing) for personal/individual usage.
[How to configure ClickHouse connection in SeekTable.](https://www.seektable.com/help/clickhouse-pivot-table)
-[Original article](https://clickhouse.tech/docs/en/interfaces/third-party/gui/)
+### Chadmin {#chadmin}
+
+[Chadmin](https://github.com/bun4uk/chadmin) is a simple UI where you can visualize your currently running queries on your ClickHouse cluster and info about them and kill them if you want.
diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md
index 012d86b1ef7..fa257a84173 100644
--- a/docs/en/introduction/adopters.md
+++ b/docs/en/introduction/adopters.md
@@ -13,6 +13,7 @@ toc_title: Adopters
| 2gis | Maps | Monitoring | — | — | [Talk in Russian, July 2019](https://youtu.be/58sPkXfq6nw) |
| Admiral | Martech | Engagement Management | — | — | [Webinar Slides, June 2020](https://altinity.com/presentations/2020/06/16/big-data-in-real-time-how-clickhouse-powers-admirals-visitor-relationships-for-publishers) |
| AdScribe | Ads | TV Analytics | — | — | [A quote from CTO](https://altinity.com/24x7-support/) |
+| Ahrefs | SEO | Analytics | — | — | [Job listing](https://ahrefs.com/jobs/data-scientist-search) |
| Alibaba Cloud | Cloud | Managed Service | — | — | [Official Website](https://help.aliyun.com/product/144466.html) |
| Aloha Browser | Mobile App | Browser backend | — | — | [Slides in Russian, May 2019](https://presentations.clickhouse.tech/meetup22/aloha.pdf) |
| Altinity | Cloud, SaaS | Main product | — | — | [Official Website](https://altinity.com/) |
@@ -47,7 +48,8 @@ toc_title: Adopters
| Diva-e | Digital consulting | Main Product | — | — | [Slides in English, September 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup29/ClickHouse-MeetUp-Unusual-Applications-sd-2019-09-17.pdf) |
| Ecwid | E-commerce SaaS | Metrics, Logging | — | — | [Slides in Russian, April 2019](https://nastachku.ru/var/files/1/presentation/backend/2_Backend_6.pdf) |
| eBay | E-commerce | Logs, Metrics and Events | — | — | [Official website, Sep 2020](https://tech.ebayinc.com/engineering/ou-online-analytical-processing/) |
-| Exness | Trading | Metrics, Logging | — | — | [Talk in Russian, May 2019](https://youtu.be/_rpU-TvSfZ8?t=3215) |
+| Exness | Trading | Metrics, Logging | — | — | [Talk in Russian, May 2019](https://youtu.be/_rpU-TvSfZ8?t=3215) |
+| EventBunker.io | Serverless Data Processing | — | — | — | [Tweet, April 2021](https://twitter.com/Halil_D_/status/1379839133472985091) |
| FastNetMon | DDoS Protection | Main Product | | — | [Official website](https://fastnetmon.com/docs-fnm-advanced/fastnetmon-advanced-traffic-persistency/) |
| Flipkart | e-Commerce | — | — | — | [Talk in English, July 2020](https://youtu.be/GMiXCMFDMow?t=239) |
| FunCorp | Games | | — | 14 bn records/day as of Jan 2021 | [Article](https://www.altinity.com/blog/migrating-from-redshift-to-clickhouse) |
@@ -75,7 +77,8 @@ toc_title: Adopters
| Marilyn | Advertising | Statistics | — | — | [Talk in Russian, June 2017](https://www.youtube.com/watch?v=iXlIgx2khwc) |
| Mello | Marketing | Analytics | 1 server | — | [Article, Oct 2020](https://vc.ru/marketing/166180-razrabotka-tipovogo-otcheta-skvoznoy-analitiki) |
| MessageBird | Telecommunications | Statistics | — | — | [Slides in English, November 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup20/messagebird.pdf) |
-| MindsDB | Machine Learning | Main Product | — | — | [Official Website](https://www.mindsdb.com/blog/machine-learning-models-as-tables-in-ch) |x
+| Microsoft | Web Analytics | Clarity (Main Product) | — | — | [A question on GitHub](https://github.com/ClickHouse/ClickHouse/issues/21556) |
+| MindsDB | Machine Learning | Main Product | — | — | [Official Website](https://www.mindsdb.com/blog/machine-learning-models-as-tables-in-ch) |
| MUX | Online Video | Video Analytics | — | — | [Talk in English, August 2019](https://altinity.com/presentations/2019/8/13/how-clickhouse-became-the-default-analytics-database-for-mux/) |
| MGID | Ad network | Web-analytics | — | — | [Blog post in Russian, April 2020](http://gs-studio.com/news-about-it/32777----clickhouse---c) |
| Netskope | Network Security | — | — | — | [Job advertisement, March 2021](https://www.mendeley.com/careers/job/senior-software-developer-backend-developer-1346348) |
diff --git a/docs/en/operations/performance-test.md b/docs/en/operations/performance-test.md
index ca805923ba9..a808ffd0a85 100644
--- a/docs/en/operations/performance-test.md
+++ b/docs/en/operations/performance-test.md
@@ -12,6 +12,7 @@ With this instruction you can run basic ClickHouse performance test on any serve
3. Copy the link to `clickhouse` binary for amd64 or aarch64.
4. ssh to the server and download it with wget:
```bash
+# These links are outdated, please obtain the fresh link from the "commits" page.
# For amd64:
wget https://clickhouse-builds.s3.yandex.net/0/e29c4c3cc47ab2a6c4516486c1b77d57e7d42643/clickhouse_build_check/gcc-10_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse
# For aarch64:
diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md
index 0b45488ebf7..f86e9668f00 100644
--- a/docs/en/operations/server-configuration-parameters/settings.md
+++ b/docs/en/operations/server-configuration-parameters/settings.md
@@ -100,6 +100,11 @@ Default value: `1073741824` (1 GB).
1073741824
```
+## database_atomic_delay_before_drop_table_sec {#database_atomic_delay_before_drop_table_sec}
+
+Sets the delay before remove table data in seconds. If the query has `SYNC` modifier, this setting is ignored.
+
+Default value: `480` (8 minute).
## default_database {#default-database}
@@ -125,6 +130,25 @@ Settings profiles are located in the file specified in the parameter `user_confi
default
```
+## default_replica_path {#default_replica_path}
+
+The path to the table in ZooKeeper.
+
+**Example**
+
+``` xml
+/clickhouse/tables/{uuid}/{shard}
+```
+## default_replica_name {#default_replica_name}
+
+ The replica name in ZooKeeper.
+
+**Example**
+
+``` xml
+{replica}
+```
+
## dictionaries_config {#server_configuration_parameters-dictionaries_config}
The path to the config file for external dictionaries.
@@ -321,7 +345,8 @@ Similar to `interserver_http_host`, except that this hostname can be used by oth
The username and password used to authenticate during [replication](../../engines/table-engines/mergetree-family/replication.md) with the Replicated\* engines. These credentials are used only for communication between replicas and are unrelated to credentials for ClickHouse clients. The server is checking these credentials for connecting replicas and use the same credentials when connecting to other replicas. So, these credentials should be set the same for all replicas in a cluster.
By default, the authentication is not used.
-**Note:** These credentials are common for replication through `HTTP` and `HTTPS`.
+!!! note "Note"
+ These credentials are common for replication through `HTTP` and `HTTPS`.
This section contains the following parameters:
diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md
index 77b68715ba9..b2470207dcc 100644
--- a/docs/en/operations/settings/merge-tree-settings.md
+++ b/docs/en/operations/settings/merge-tree-settings.md
@@ -56,6 +56,26 @@ Default value: 150.
ClickHouse artificially executes `INSERT` longer (adds ‘sleep’) so that the background merge process can merge parts faster than they are added.
+## inactive_parts_to_throw_insert {#inactive-parts-to-throw-insert}
+
+If the number of inactive parts in a single partition more than the `inactive_parts_to_throw_insert` value, `INSERT` is interrupted with the "Too many inactive parts (N). Parts cleaning are processing significantly slower than inserts" exception.
+
+Possible values:
+
+- Any positive integer.
+
+Default value: 0 (unlimited).
+
+## inactive_parts_to_delay_insert {#inactive-parts-to-delay-insert}
+
+If the number of inactive parts in a single partition in the table at least that many the `inactive_parts_to_delay_insert` value, an `INSERT` artificially slows down. It is useful when a server fails to clean up parts quickly enough.
+
+Possible values:
+
+- Any positive integer.
+
+Default value: 0 (unlimited).
+
## max_delay_to_insert {#max-delay-to-insert}
The value in seconds, which is used to calculate the `INSERT` delay, if the number of active parts in a single partition exceeds the [parts_to_delay_insert](#parts-to-delay-insert) value.
diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index 3696c89b93e..1b422785b4e 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -854,8 +854,6 @@ For example, when reading from a table, if it is possible to evaluate expression
Default value: the number of physical CPU cores.
-If less than one SELECT query is normally run on a server at a time, set this parameter to a value slightly less than the actual number of processor cores.
-
For queries that are completed quickly because of a LIMIT, you can set a lower ‘max_threads’. For example, if the necessary number of entries are located in every block and max_threads = 8, then 8 blocks are retrieved, although it would have been enough to read just one.
The smaller the `max_threads` value, the less memory is consumed.
@@ -1565,6 +1563,17 @@ Possible values:
Default value: 0
+## optimize_skip_unused_shards_rewrite_in {#optimize-skip-unused-shardslrewrite-in}
+
+Rewrite IN in query for remote shards to exclude values that does not belong to the shard (requires optimize_skip_unused_shards).
+
+Possible values:
+
+- 0 — Disabled.
+- 1 — Enabled.
+
+Default value: 1 (since it requires `optimize_skip_unused_shards` anyway, which `0` by default)
+
## allow_nondeterministic_optimize_skip_unused_shards {#allow-nondeterministic-optimize-skip-unused-shards}
Allow nondeterministic (like `rand` or `dictGet`, since later has some caveats with updates) functions in sharding key.
@@ -2787,6 +2796,28 @@ Possible values:
Default value: `0`.
+## database_atomic_wait_for_drop_and_detach_synchronously {#database_atomic_wait_for_drop_and_detach_synchronously}
+
+Adds a modifier `SYNC` to all `DROP` and `DETACH` queries.
+
+Possible values:
+
+- 0 — Queries will be executed with delay.
+- 1 — Queries will be executed without delay.
+
+Default value: `0`.
+
+## show_table_uuid_in_table_create_query_if_not_nil {#show_table_uuid_in_table_create_query_if_not_nil}
+
+Sets the `SHOW TABLE` query display.
+
+Possible values:
+
+- 0 — The query will be displayed without table UUID.
+- 1 — The query will be displayed with table UUID.
+
+Default value: `0`.
+
## allow_experimental_live_view {#allow-experimental-live-view}
Allows creation of experimental [live views](../../sql-reference/statements/create/view.md#live-view).
@@ -2822,6 +2853,17 @@ Sets the interval in seconds after which periodically refreshed [live view](../.
Default value: `60`.
+## check_query_single_value_result {#check_query_single_value_result}
+
+Defines the level of detail for the [CHECK TABLE](../../sql-reference/statements/check-table.md#checking-mergetree-tables) query result for `MergeTree` family engines .
+
+Possible values:
+
+- 0 — the query shows a check status for every individual data part of a table.
+- 1 — the query shows the general table check status.
+
+Default value: `0`.
+
## limit {#limit}
Sets the number of rows to get from the query result. It adjust the limit previously set by the [LIMIT](../../sql-reference/statements/select/limit.md#limit-clause) clause.
diff --git a/docs/en/operations/system-tables/columns.md b/docs/en/operations/system-tables/columns.md
index 92a6315d06b..9160dca9a1a 100644
--- a/docs/en/operations/system-tables/columns.md
+++ b/docs/en/operations/system-tables/columns.md
@@ -4,7 +4,9 @@ Contains information about columns in all the tables.
You can use this table to get information similar to the [DESCRIBE TABLE](../../sql-reference/statements/misc.md#misc-describe-table) query, but for multiple tables at once.
-The `system.columns` table contains the following columns (the column type is shown in brackets):
+Columns from [temporary tables](../../sql-reference/statements/create/table.md#temporary-tables) are visible in the `system.columns` only in those session where they have been created. They are shown with the empty `database` field.
+
+Columns:
- `database` ([String](../../sql-reference/data-types/string.md)) — Database name.
- `table` ([String](../../sql-reference/data-types/string.md)) — Table name.
@@ -26,7 +28,7 @@ The `system.columns` table contains the following columns (the column type is sh
**Example**
```sql
-:) select * from system.columns LIMIT 2 FORMAT Vertical;
+SELECT * FROM system.columns LIMIT 2 FORMAT Vertical;
```
```text
@@ -65,8 +67,6 @@ is_in_sorting_key: 0
is_in_primary_key: 0
is_in_sampling_key: 0
compression_codec:
-
-2 rows in set. Elapsed: 0.002 sec.
```
[Original article](https://clickhouse.tech/docs/en/operations/system_tables/columns)
diff --git a/docs/en/operations/system-tables/replication_queue.md b/docs/en/operations/system-tables/replication_queue.md
index f3e3a35f13b..539a29432ac 100644
--- a/docs/en/operations/system-tables/replication_queue.md
+++ b/docs/en/operations/system-tables/replication_queue.md
@@ -15,16 +15,16 @@ Columns:
- `node_name` ([String](../../sql-reference/data-types/string.md)) — Node name in ZooKeeper.
- `type` ([String](../../sql-reference/data-types/string.md)) — Type of the task in the queue, one of:
- - `GET_PART` - Get the part from another replica.
- - `ATTACH_PART` - Attach the part, possibly from our own replica (if found in `detached` folder).
- You may think of it as a `GET_PART` with some optimisations as they're nearly identical.
- - `MERGE_PARTS` - Merge the parts.
- - `DROP_RANGE` - Delete the parts in the specified partition in the specified number range.
- - `CLEAR_COLUMN` - NOTE: Deprecated. Drop specific column from specified partition.
- - `CLEAR_INDEX` - NOTE: Deprecated. Drop specific index from specified partition.
- - `REPLACE_RANGE` - Drop certain range of partitions and replace them by new ones
- - `MUTATE_PART` - Apply one or several mutations to the part.
- - `ALTER_METADATA` - Apply alter modification according to global /metadata and /columns paths
+
+ - `GET_PART` — Get the part from another replica.
+ - `ATTACH_PART` — Attach the part, possibly from our own replica (if found in the `detached` folder). You may think of it as a `GET_PART` with some optimizations as they're nearly identical.
+ - `MERGE_PARTS` — Merge the parts.
+ - `DROP_RANGE` — Delete the parts in the specified partition in the specified number range.
+ - `CLEAR_COLUMN` — NOTE: Deprecated. Drop specific column from specified partition.
+ - `CLEAR_INDEX` — NOTE: Deprecated. Drop specific index from specified partition.
+ - `REPLACE_RANGE` — Drop a certain range of parts and replace them with new ones.
+ - `MUTATE_PART` — Apply one or several mutations to the part.
+ - `ALTER_METADATA` — Apply alter modification according to global /metadata and /columns paths.
- `create_time` ([Datetime](../../sql-reference/data-types/datetime.md)) — Date and time when the task was submitted for execution.
diff --git a/docs/en/operations/system-tables/tables.md b/docs/en/operations/system-tables/tables.md
index 6ad1425e032..ccc9ab94f8b 100644
--- a/docs/en/operations/system-tables/tables.md
+++ b/docs/en/operations/system-tables/tables.md
@@ -1,59 +1,65 @@
# system.tables {#system-tables}
-Contains metadata of each table that the server knows about. Detached tables are not shown in `system.tables`.
+Contains metadata of each table that the server knows about.
-This table contains the following columns (the column type is shown in brackets):
+[Detached](../../sql-reference/statements/detach.md) tables are not shown in `system.tables`.
-- `database` (String) — The name of the database the table is in.
+[Temporary tables](../../sql-reference/statements/create/table.md#temporary-tables) are visible in the `system.tables` only in those session where they have been created. They are shown with the empty `database` field and with the `is_temporary` flag switched on.
-- `name` (String) — Table name.
+Columns:
-- `engine` (String) — Table engine name (without parameters).
+- `database` ([String](../../sql-reference/data-types/string.md)) — The name of the database the table is in.
-- `is_temporary` (UInt8) - Flag that indicates whether the table is temporary.
+- `name` ([String](../../sql-reference/data-types/string.md)) — Table name.
-- `data_path` (String) - Path to the table data in the file system.
+- `engine` ([String](../../sql-reference/data-types/string.md)) — Table engine name (without parameters).
-- `metadata_path` (String) - Path to the table metadata in the file system.
+- `is_temporary` ([UInt8](../../sql-reference/data-types/int-uint.md)) - Flag that indicates whether the table is temporary.
-- `metadata_modification_time` (DateTime) - Time of latest modification of the table metadata.
+- `data_path` ([String](../../sql-reference/data-types/string.md)) - Path to the table data in the file system.
-- `dependencies_database` (Array(String)) - Database dependencies.
+- `metadata_path` ([String](../../sql-reference/data-types/string.md)) - Path to the table metadata in the file system.
-- `dependencies_table` (Array(String)) - Table dependencies ([MaterializedView](../../engines/table-engines/special/materializedview.md) tables based on the current table).
+- `metadata_modification_time` ([DateTime](../../sql-reference/data-types/datetime.md)) - Time of latest modification of the table metadata.
-- `create_table_query` (String) - The query that was used to create the table.
+- `dependencies_database` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) - Database dependencies.
-- `engine_full` (String) - Parameters of the table engine.
+- `dependencies_table` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) - Table dependencies ([MaterializedView](../../engines/table-engines/special/materializedview.md) tables based on the current table).
-- `partition_key` (String) - The partition key expression specified in the table.
+- `create_table_query` ([String](../../sql-reference/data-types/string.md)) - The query that was used to create the table.
-- `sorting_key` (String) - The sorting key expression specified in the table.
+- `engine_full` ([String](../../sql-reference/data-types/string.md)) - Parameters of the table engine.
-- `primary_key` (String) - The primary key expression specified in the table.
+- `partition_key` ([String](../../sql-reference/data-types/string.md)) - The partition key expression specified in the table.
-- `sampling_key` (String) - The sampling key expression specified in the table.
+- `sorting_key` ([String](../../sql-reference/data-types/string.md)) - The sorting key expression specified in the table.
-- `storage_policy` (String) - The storage policy:
+- `primary_key` ([String](../../sql-reference/data-types/string.md)) - The primary key expression specified in the table.
+
+- `sampling_key` ([String](../../sql-reference/data-types/string.md)) - The sampling key expression specified in the table.
+
+- `storage_policy` ([String](../../sql-reference/data-types/string.md)) - The storage policy:
- [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes)
- [Distributed](../../engines/table-engines/special/distributed.md#distributed)
-- `total_rows` (Nullable(UInt64)) - Total number of rows, if it is possible to quickly determine exact number of rows in the table, otherwise `Null` (including underying `Buffer` table).
+- `total_rows` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) - Total number of rows, if it is possible to quickly determine exact number of rows in the table, otherwise `NULL` (including underying `Buffer` table).
-- `total_bytes` (Nullable(UInt64)) - Total number of bytes, if it is possible to quickly determine exact number of bytes for the table on storage, otherwise `Null` (**does not** includes any underlying storage).
+- `total_bytes` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) - Total number of bytes, if it is possible to quickly determine exact number of bytes for the table on storage, otherwise `NULL` (does not includes any underlying storage).
- If the table stores data on disk, returns used space on disk (i.e. compressed).
- If the table stores data in memory, returns approximated number of used bytes in memory.
-- `lifetime_rows` (Nullable(UInt64)) - Total number of rows INSERTed since server start (only for `Buffer` tables).
+- `lifetime_rows` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) - Total number of rows INSERTed since server start (only for `Buffer` tables).
-- `lifetime_bytes` (Nullable(UInt64)) - Total number of bytes INSERTed since server start (only for `Buffer` tables).
+- `lifetime_bytes` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) - Total number of bytes INSERTed since server start (only for `Buffer` tables).
The `system.tables` table is used in `SHOW TABLES` query implementation.
+**Example**
+
```sql
-:) SELECT * FROM system.tables LIMIT 2 FORMAT Vertical;
+SELECT * FROM system.tables LIMIT 2 FORMAT Vertical;
```
```text
@@ -100,8 +106,6 @@ sampling_key:
storage_policy:
total_rows: ᴺᵁᴸᴸ
total_bytes: ᴺᵁᴸᴸ
-
-2 rows in set. Elapsed: 0.004 sec.
```
[Original article](https://clickhouse.tech/docs/en/operations/system_tables/tables)
diff --git a/docs/en/operations/system-tables/trace_log.md b/docs/en/operations/system-tables/trace_log.md
index b3b04795a60..e4c01a65d9d 100644
--- a/docs/en/operations/system-tables/trace_log.md
+++ b/docs/en/operations/system-tables/trace_log.md
@@ -20,10 +20,12 @@ Columns:
When connecting to the server by `clickhouse-client`, you see the string similar to `Connected to ClickHouse server version 19.18.1 revision 54429.`. This field contains the `revision`, but not the `version` of a server.
-- `timer_type` ([Enum8](../../sql-reference/data-types/enum.md)) — Timer type:
+- `trace_type` ([Enum8](../../sql-reference/data-types/enum.md)) — Trace type:
- - `Real` represents wall-clock time.
- - `CPU` represents CPU time.
+ - `Real` represents collecting stack traces by wall-clock time.
+ - `CPU` represents collecting stack traces by CPU time.
+ - `Memory` represents collecting allocations and deallocations when memory allocation exceeds the subsequent watermark.
+ - `MemorySample` represents collecting random allocations and deallocations.
- `thread_number` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Thread identifier.
diff --git a/docs/en/operations/tips.md b/docs/en/operations/tips.md
index e62dea0b04e..865fe58d7cd 100644
--- a/docs/en/operations/tips.md
+++ b/docs/en/operations/tips.md
@@ -191,8 +191,9 @@ dynamicConfigFile=/etc/zookeeper-{{ '{{' }} cluster['name'] {{ '}}' }}/conf/zoo.
Java version:
``` text
-Java(TM) SE Runtime Environment (build 1.8.0_25-b17)
-Java HotSpot(TM) 64-Bit Server VM (build 25.25-b02, mixed mode)
+openjdk 11.0.5-shenandoah 2019-10-15
+OpenJDK Runtime Environment (build 11.0.5-shenandoah+10-adhoc.heretic.src)
+OpenJDK 64-Bit Server VM (build 11.0.5-shenandoah+10-adhoc.heretic.src, mixed mode)
```
JVM parameters:
@@ -204,7 +205,7 @@ ZOOCFGDIR=/etc/$NAME/conf
# TODO this is really ugly
# How to find out, which jars are needed?
# seems, that log4j requires the log4j.properties file to be in the classpath
-CLASSPATH="$ZOOCFGDIR:/usr/build/classes:/usr/build/lib/*.jar:/usr/share/zookeeper/zookeeper-3.5.1-metrika.jar:/usr/share/zookeeper/slf4j-log4j12-1.7.5.jar:/usr/share/zookeeper/slf4j-api-1.7.5.jar:/usr/share/zookeeper/servlet-api-2.5-20081211.jar:/usr/share/zookeeper/netty-3.7.0.Final.jar:/usr/share/zookeeper/log4j-1.2.16.jar:/usr/share/zookeeper/jline-2.11.jar:/usr/share/zookeeper/jetty-util-6.1.26.jar:/usr/share/zookeeper/jetty-6.1.26.jar:/usr/share/zookeeper/javacc.jar:/usr/share/zookeeper/jackson-mapper-asl-1.9.11.jar:/usr/share/zookeeper/jackson-core-asl-1.9.11.jar:/usr/share/zookeeper/commons-cli-1.2.jar:/usr/src/java/lib/*.jar:/usr/etc/zookeeper"
+CLASSPATH="$ZOOCFGDIR:/usr/build/classes:/usr/build/lib/*.jar:/usr/share/zookeeper-3.6.2/lib/audience-annotations-0.5.0.jar:/usr/share/zookeeper-3.6.2/lib/commons-cli-1.2.jar:/usr/share/zookeeper-3.6.2/lib/commons-lang-2.6.jar:/usr/share/zookeeper-3.6.2/lib/jackson-annotations-2.10.3.jar:/usr/share/zookeeper-3.6.2/lib/jackson-core-2.10.3.jar:/usr/share/zookeeper-3.6.2/lib/jackson-databind-2.10.3.jar:/usr/share/zookeeper-3.6.2/lib/javax.servlet-api-3.1.0.jar:/usr/share/zookeeper-3.6.2/lib/jetty-http-9.4.24.v20191120.jar:/usr/share/zookeeper-3.6.2/lib/jetty-io-9.4.24.v20191120.jar:/usr/share/zookeeper-3.6.2/lib/jetty-security-9.4.24.v20191120.jar:/usr/share/zookeeper-3.6.2/lib/jetty-server-9.4.24.v20191120.jar:/usr/share/zookeeper-3.6.2/lib/jetty-servlet-9.4.24.v20191120.jar:/usr/share/zookeeper-3.6.2/lib/jetty-util-9.4.24.v20191120.jar:/usr/share/zookeeper-3.6.2/lib/jline-2.14.6.jar:/usr/share/zookeeper-3.6.2/lib/json-simple-1.1.1.jar:/usr/share/zookeeper-3.6.2/lib/log4j-1.2.17.jar:/usr/share/zookeeper-3.6.2/lib/metrics-core-3.2.5.jar:/usr/share/zookeeper-3.6.2/lib/netty-buffer-4.1.50.Final.jar:/usr/share/zookeeper-3.6.2/lib/netty-codec-4.1.50.Final.jar:/usr/share/zookeeper-3.6.2/lib/netty-common-4.1.50.Final.jar:/usr/share/zookeeper-3.6.2/lib/netty-handler-4.1.50.Final.jar:/usr/share/zookeeper-3.6.2/lib/netty-resolver-4.1.50.Final.jar:/usr/share/zookeeper-3.6.2/lib/netty-transport-4.1.50.Final.jar:/usr/share/zookeeper-3.6.2/lib/netty-transport-native-epoll-4.1.50.Final.jar:/usr/share/zookeeper-3.6.2/lib/netty-transport-native-unix-common-4.1.50.Final.jar:/usr/share/zookeeper-3.6.2/lib/simpleclient-0.6.0.jar:/usr/share/zookeeper-3.6.2/lib/simpleclient_common-0.6.0.jar:/usr/share/zookeeper-3.6.2/lib/simpleclient_hotspot-0.6.0.jar:/usr/share/zookeeper-3.6.2/lib/simpleclient_servlet-0.6.0.jar:/usr/share/zookeeper-3.6.2/lib/slf4j-api-1.7.25.jar:/usr/share/zookeeper-3.6.2/lib/slf4j-log4j12-1.7.25.jar:/usr/share/zookeeper-3.6.2/lib/snappy-java-1.1.7.jar:/usr/share/zookeeper-3.6.2/lib/zookeeper-3.6.2.jar:/usr/share/zookeeper-3.6.2/lib/zookeeper-jute-3.6.2.jar:/usr/share/zookeeper-3.6.2/lib/zookeeper-prometheus-metrics-3.6.2.jar:/usr/share/zookeeper-3.6.2/etc"
ZOOCFG="$ZOOCFGDIR/zoo.cfg"
ZOO_LOG_DIR=/var/log/$NAME
@@ -213,27 +214,17 @@ GROUP=zookeeper
PIDDIR=/var/run/$NAME
PIDFILE=$PIDDIR/$NAME.pid
SCRIPTNAME=/etc/init.d/$NAME
-JAVA=/usr/bin/java
+JAVA=/usr/local/jdk-11/bin/java
ZOOMAIN="org.apache.zookeeper.server.quorum.QuorumPeerMain"
ZOO_LOG4J_PROP="INFO,ROLLINGFILE"
JMXLOCALONLY=false
JAVA_OPTS="-Xms{{ '{{' }} cluster.get('xms','128M') {{ '}}' }} \
-Xmx{{ '{{' }} cluster.get('xmx','1G') {{ '}}' }} \
- -Xloggc:/var/log/$NAME/zookeeper-gc.log \
- -XX:+UseGCLogFileRotation \
- -XX:NumberOfGCLogFiles=16 \
- -XX:GCLogFileSize=16M \
+ -Xlog:safepoint,gc*=info,age*=debug:file=/var/log/$NAME/zookeeper-gc.log:time,level,tags:filecount=16,filesize=16M
-verbose:gc \
- -XX:+PrintGCTimeStamps \
- -XX:+PrintGCDateStamps \
- -XX:+PrintGCDetails
- -XX:+PrintTenuringDistribution \
- -XX:+PrintGCApplicationStoppedTime \
- -XX:+PrintGCApplicationConcurrentTime \
- -XX:+PrintSafepointStatistics \
- -XX:+UseParNewGC \
- -XX:+UseConcMarkSweepGC \
--XX:+CMSParallelRemarkEnabled"
+ -XX:+UseG1GC \
+ -Djute.maxbuffer=8388608 \
+ -XX:MaxGCPauseMillis=50"
```
Salt init:
diff --git a/docs/en/operations/update.md b/docs/en/operations/update.md
index 9fa9c44e130..dbcf9ae2b3e 100644
--- a/docs/en/operations/update.md
+++ b/docs/en/operations/update.md
@@ -15,7 +15,8 @@ $ sudo service clickhouse-server restart
If you installed ClickHouse using something other than the recommended `deb` packages, use the appropriate update method.
-ClickHouse does not support a distributed update. The operation should be performed consecutively on each separate server. Do not update all the servers on a cluster simultaneously, or the cluster will be unavailable for some time.
+!!! note "Note"
+ You can update multiple servers at once as soon as there is no moment when all replicas of one shard are offline.
The upgrade of older version of ClickHouse to specific version:
@@ -28,7 +29,3 @@ $ sudo apt-get update
$ sudo apt-get install clickhouse-server=xx.yy.a.b clickhouse-client=xx.yy.a.b clickhouse-common-static=xx.yy.a.b
$ sudo service clickhouse-server restart
```
-
-
-
-
diff --git a/docs/en/sql-reference/aggregate-functions/combinators.md b/docs/en/sql-reference/aggregate-functions/combinators.md
index cddef68d49c..259202805d3 100644
--- a/docs/en/sql-reference/aggregate-functions/combinators.md
+++ b/docs/en/sql-reference/aggregate-functions/combinators.md
@@ -27,7 +27,37 @@ Example 2: `uniqArray(arr)` – Counts the number of unique elements in all ‘a
## -SimpleState {#agg-functions-combinator-simplestate}
-If you apply this combinator, the aggregate function returns the same value but with a different type. This is an `SimpleAggregateFunction(...)` that can be stored in a table to work with [AggregatingMergeTree](../../engines/table-engines/mergetree-family/aggregatingmergetree.md) table engines.
+If you apply this combinator, the aggregate function returns the same value but with a different type. This is a [SimpleAggregateFunction(...)](../../sql-reference/data-types/simpleaggregatefunction.md) that can be stored in a table to work with [AggregatingMergeTree](../../engines/table-engines/mergetree-family/aggregatingmergetree.md) tables.
+
+**Syntax**
+
+``` sql
+SimpleState(x)
+```
+
+**Arguments**
+
+- `x` — Aggregate function parameters.
+
+**Returned values**
+
+The value of an aggregate function with the `SimpleAggregateFunction(...)` type.
+
+**Example**
+
+Query:
+
+``` sql
+WITH anySimpleState(number) AS c SELECT toTypeName(c), c FROM numbers(1);
+```
+
+Result:
+
+``` text
+┌─toTypeName(c)────────────────────────┬─c─┐
+│ SimpleAggregateFunction(any, UInt64) │ 0 │
+└──────────────────────────────────────┴───┘
+```
## -State {#agg-functions-combinator-state}
@@ -249,4 +279,3 @@ FROM people
└────────┴───────────────────────────┘
```
-
diff --git a/docs/en/sql-reference/aggregate-functions/reference/argmax.md b/docs/en/sql-reference/aggregate-functions/reference/argmax.md
index 72aa607a751..0630e2f585e 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/argmax.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/argmax.md
@@ -6,20 +6,12 @@ toc_priority: 106
Calculates the `arg` value for a maximum `val` value. If there are several different values of `arg` for maximum values of `val`, returns the first of these values encountered.
-Tuple version of this function will return the tuple with the maximum `val` value. It is convenient for use with [SimpleAggregateFunction](../../../sql-reference/data-types/simpleaggregatefunction.md).
-
**Syntax**
``` sql
argMax(arg, val)
```
-or
-
-``` sql
-argMax(tuple(arg, val))
-```
-
**Arguments**
- `arg` — Argument.
@@ -29,13 +21,7 @@ argMax(tuple(arg, val))
- `arg` value that corresponds to maximum `val` value.
-Type: matches `arg` type.
-
-For tuple in the input:
-
-- Tuple `(arg, val)`, where `val` is the maximum value and `arg` is a corresponding value.
-
-Type: [Tuple](../../../sql-reference/data-types/tuple.md).
+Type: matches `arg` type.
**Example**
@@ -52,15 +38,13 @@ Input table:
Query:
``` sql
-SELECT argMax(user, salary), argMax(tuple(user, salary), salary), argMax(tuple(user, salary)) FROM salary;
+SELECT argMax(user, salary) FROM salary;
```
Result:
``` text
-┌─argMax(user, salary)─┬─argMax(tuple(user, salary), salary)─┬─argMax(tuple(user, salary))─┐
-│ director │ ('director',5000) │ ('director',5000) │
-└──────────────────────┴─────────────────────────────────────┴─────────────────────────────┘
+┌─argMax(user, salary)─┐
+│ director │
+└──────────────────────┘
```
-
-[Original article](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/argmax/)
diff --git a/docs/en/sql-reference/aggregate-functions/reference/argmin.md b/docs/en/sql-reference/aggregate-functions/reference/argmin.md
index 7ddc38cd28a..a259a76b7d7 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/argmin.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/argmin.md
@@ -6,20 +6,12 @@ toc_priority: 105
Calculates the `arg` value for a minimum `val` value. If there are several different values of `arg` for minimum values of `val`, returns the first of these values encountered.
-Tuple version of this function will return the tuple with the minimum `val` value. It is convenient for use with [SimpleAggregateFunction](../../../sql-reference/data-types/simpleaggregatefunction.md).
-
**Syntax**
``` sql
argMin(arg, val)
```
-or
-
-``` sql
-argMin(tuple(arg, val))
-```
-
**Arguments**
- `arg` — Argument.
@@ -29,13 +21,7 @@ argMin(tuple(arg, val))
- `arg` value that corresponds to minimum `val` value.
-Type: matches `arg` type.
-
-For tuple in the input:
-
-- Tuple `(arg, val)`, where `val` is the minimum value and `arg` is a corresponding value.
-
-Type: [Tuple](../../../sql-reference/data-types/tuple.md).
+Type: matches `arg` type.
**Example**
@@ -52,15 +38,13 @@ Input table:
Query:
``` sql
-SELECT argMin(user, salary), argMin(tuple(user, salary)) FROM salary;
+SELECT argMin(user, salary) FROM salary
```
Result:
``` text
-┌─argMin(user, salary)─┬─argMin(tuple(user, salary))─┐
-│ worker │ ('worker',1000) │
-└──────────────────────┴─────────────────────────────┘
+┌─argMin(user, salary)─┐
+│ worker │
+└──────────────────────┘
```
-
-[Original article](https://clickhouse.tech/docs/en/sql-reference/aggregate-functions/reference/argmin/)
diff --git a/docs/en/sql-reference/aggregate-functions/reference/quantiletdigest.md b/docs/en/sql-reference/aggregate-functions/reference/quantiletdigest.md
index dcc665a68af..dd0d59978d1 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/quantiletdigest.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/quantiletdigest.md
@@ -6,7 +6,7 @@ toc_priority: 207
Computes an approximate [quantile](https://en.wikipedia.org/wiki/Quantile) of a numeric data sequence using the [t-digest](https://github.com/tdunning/t-digest/blob/master/docs/t-digest-paper/histo.pdf) algorithm.
-The maximum error is 1%. Memory consumption is `log(n)`, where `n` is a number of values. The result depends on the order of running the query, and is nondeterministic.
+Memory consumption is `log(n)`, where `n` is a number of values. The result depends on the order of running the query, and is nondeterministic.
The performance of the function is lower than performance of [quantile](../../../sql-reference/aggregate-functions/reference/quantile.md#quantile) or [quantileTiming](../../../sql-reference/aggregate-functions/reference/quantiletiming.md#quantiletiming). In terms of the ratio of State size to precision, this function is much better than `quantile`.
diff --git a/docs/en/sql-reference/aggregate-functions/reference/uniqhll12.md b/docs/en/sql-reference/aggregate-functions/reference/uniqhll12.md
index 5b23ea81eae..4983220ed7f 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/uniqhll12.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/uniqhll12.md
@@ -26,7 +26,7 @@ Function:
- Uses the HyperLogLog algorithm to approximate the number of different argument values.
- 212 5-bit cells are used. The size of the state is slightly more than 2.5 KB. The result is not very accurate (up to ~10% error) for small data sets (<10K elements). However, the result is fairly accurate for high-cardinality data sets (10K-100M), with a maximum error of ~1.6%. Starting from 100M, the estimation error increases, and the function will return very inaccurate results for data sets with extremely high cardinality (1B+ elements).
+ 2^12 5-bit cells are used. The size of the state is slightly more than 2.5 KB. The result is not very accurate (up to ~10% error) for small data sets (<10K elements). However, the result is fairly accurate for high-cardinality data sets (10K-100M), with a maximum error of ~1.6%. Starting from 100M, the estimation error increases, and the function will return very inaccurate results for data sets with extremely high cardinality (1B+ elements).
- Provides the determinate result (it doesn’t depend on the query processing order).
diff --git a/docs/en/sql-reference/data-types/date.md b/docs/en/sql-reference/data-types/date.md
index 886e93f433c..0cfac4d59fe 100644
--- a/docs/en/sql-reference/data-types/date.md
+++ b/docs/en/sql-reference/data-types/date.md
@@ -5,7 +5,7 @@ toc_title: Date
# Date {#data_type-date}
-A date. Stored in two bytes as the number of days since 1970-01-01 (unsigned). Allows storing values from just after the beginning of the Unix Epoch to the upper threshold defined by a constant at the compilation stage (currently, this is until the year 2106, but the final fully-supported year is 2105).
+A date. Stored in two bytes as the number of days since 1970-01-01 (unsigned). Allows storing values from just after the beginning of the Unix Epoch to the upper threshold defined by a constant at the compilation stage (currently, this is until the year 2149, but the final fully-supported year is 2148).
The date value is stored without the time zone.
diff --git a/docs/en/sql-reference/data-types/datetime64.md b/docs/en/sql-reference/data-types/datetime64.md
index 5cba8315090..1d3725b9fb3 100644
--- a/docs/en/sql-reference/data-types/datetime64.md
+++ b/docs/en/sql-reference/data-types/datetime64.md
@@ -9,7 +9,7 @@ Allows to store an instant in time, that can be expressed as a calendar date and
Tick size (precision): 10-precision seconds
-Syntax:
+**Syntax:**
``` sql
DateTime64(precision, [timezone])
@@ -17,9 +17,11 @@ DateTime64(precision, [timezone])
Internally, stores data as a number of ‘ticks’ since epoch start (1970-01-01 00:00:00 UTC) as Int64. The tick resolution is determined by the precision parameter. Additionally, the `DateTime64` type can store time zone that is the same for the entire column, that affects how the values of the `DateTime64` type values are displayed in text format and how the values specified as strings are parsed (‘2020-01-01 05:00:01.000’). The time zone is not stored in the rows of the table (or in resultset), but is stored in the column metadata. See details in [DateTime](../../sql-reference/data-types/datetime.md).
+Supported range from January 1, 1925 till December 31, 2283.
+
## Examples {#examples}
-**1.** Creating a table with `DateTime64`-type column and inserting data into it:
+1. Creating a table with `DateTime64`-type column and inserting data into it:
``` sql
CREATE TABLE dt
@@ -27,15 +29,15 @@ CREATE TABLE dt
`timestamp` DateTime64(3, 'Europe/Moscow'),
`event_id` UInt8
)
-ENGINE = TinyLog
+ENGINE = TinyLog;
```
``` sql
-INSERT INTO dt Values (1546300800000, 1), ('2019-01-01 00:00:00', 2)
+INSERT INTO dt Values (1546300800000, 1), ('2019-01-01 00:00:00', 2);
```
``` sql
-SELECT * FROM dt
+SELECT * FROM dt;
```
``` text
@@ -45,13 +47,13 @@ SELECT * FROM dt
└─────────────────────────┴──────────┘
```
-- When inserting datetime as an integer, it is treated as an appropriately scaled Unix Timestamp (UTC). `1546300800000` (with precision 3) represents `'2019-01-01 00:00:00'` UTC. However, as `timestamp` column has `Europe/Moscow` (UTC+3) timezone specified, when outputting as a string the value will be shown as `'2019-01-01 03:00:00'`
+- When inserting datetime as an integer, it is treated as an appropriately scaled Unix Timestamp (UTC). `1546300800000` (with precision 3) represents `'2019-01-01 00:00:00'` UTC. However, as `timestamp` column has `Europe/Moscow` (UTC+3) timezone specified, when outputting as a string the value will be shown as `'2019-01-01 03:00:00'`.
- When inserting string value as datetime, it is treated as being in column timezone. `'2019-01-01 00:00:00'` will be treated as being in `Europe/Moscow` timezone and stored as `1546290000000`.
-**2.** Filtering on `DateTime64` values
+2. Filtering on `DateTime64` values
``` sql
-SELECT * FROM dt WHERE timestamp = toDateTime64('2019-01-01 00:00:00', 3, 'Europe/Moscow')
+SELECT * FROM dt WHERE timestamp = toDateTime64('2019-01-01 00:00:00', 3, 'Europe/Moscow');
```
``` text
@@ -60,12 +62,12 @@ SELECT * FROM dt WHERE timestamp = toDateTime64('2019-01-01 00:00:00', 3, 'Europ
└─────────────────────────┴──────────┘
```
-Unlike `DateTime`, `DateTime64` values are not converted from `String` automatically
+Unlike `DateTime`, `DateTime64` values are not converted from `String` automatically.
-**3.** Getting a time zone for a `DateTime64`-type value:
+3. Getting a time zone for a `DateTime64`-type value:
``` sql
-SELECT toDateTime64(now(), 3, 'Europe/Moscow') AS column, toTypeName(column) AS x
+SELECT toDateTime64(now(), 3, 'Europe/Moscow') AS column, toTypeName(column) AS x;
```
``` text
@@ -74,13 +76,13 @@ SELECT toDateTime64(now(), 3, 'Europe/Moscow') AS column, toTypeName(column) AS
└─────────────────────────┴────────────────────────────────┘
```
-**4.** Timezone conversion
+4. Timezone conversion
``` sql
SELECT
toDateTime64(timestamp, 3, 'Europe/London') as lon_time,
toDateTime64(timestamp, 3, 'Europe/Moscow') as mos_time
-FROM dt
+FROM dt;
```
``` text
@@ -90,7 +92,7 @@ FROM dt
└─────────────────────────┴─────────────────────────┘
```
-## See Also {#see-also}
+**See Also**
- [Type conversion functions](../../sql-reference/functions/type-conversion-functions.md)
- [Functions for working with dates and times](../../sql-reference/functions/date-time-functions.md)
diff --git a/docs/en/sql-reference/data-types/simpleaggregatefunction.md b/docs/en/sql-reference/data-types/simpleaggregatefunction.md
index 244779c5ca8..f3a245e9627 100644
--- a/docs/en/sql-reference/data-types/simpleaggregatefunction.md
+++ b/docs/en/sql-reference/data-types/simpleaggregatefunction.md
@@ -2,6 +2,8 @@
`SimpleAggregateFunction(name, types_of_arguments…)` data type stores current value of the aggregate function, and does not store its full state as [`AggregateFunction`](../../sql-reference/data-types/aggregatefunction.md) does. This optimization can be applied to functions for which the following property holds: the result of applying a function `f` to a row set `S1 UNION ALL S2` can be obtained by applying `f` to parts of the row set separately, and then again applying `f` to the results: `f(S1 UNION ALL S2) = f(f(S1) UNION ALL f(S2))`. This property guarantees that partial aggregation results are enough to compute the combined one, so we don’t have to store and process any extra data.
+The common way to produce an aggregate function value is by calling the aggregate function with the [-SimpleState](../../sql-reference/aggregate-functions/combinators.md#agg-functions-combinator-simplestate) suffix.
+
The following aggregate functions are supported:
- [`any`](../../sql-reference/aggregate-functions/reference/any.md#agg_function-any)
@@ -18,8 +20,6 @@ The following aggregate functions are supported:
- [`sumMap`](../../sql-reference/aggregate-functions/reference/summap.md#agg_functions-summap)
- [`minMap`](../../sql-reference/aggregate-functions/reference/minmap.md#agg_functions-minmap)
- [`maxMap`](../../sql-reference/aggregate-functions/reference/maxmap.md#agg_functions-maxmap)
-- [`argMin`](../../sql-reference/aggregate-functions/reference/argmin.md)
-- [`argMax`](../../sql-reference/aggregate-functions/reference/argmax.md)
!!! note "Note"
diff --git a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md
index dbf2fa67ac5..f22d2a0b59e 100644
--- a/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md
+++ b/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md
@@ -159,14 +159,14 @@ Configuration fields:
| Tag | Description | Required |
|------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|
| `name` | Column name. | Yes |
-| `type` | ClickHouse data type.
ClickHouse tries to cast value from dictionary to the specified data type. For example, for MySQL, the field might be `TEXT`, `VARCHAR`, or `BLOB` in the MySQL source table, but it can be uploaded as `String` in ClickHouse.
[Nullable](../../../sql-reference/data-types/nullable.md) is not supported. | Yes |
-| `null_value` | Default value for a non-existing element.
In the example, it is an empty string. You cannot use `NULL` in this field. | Yes |
+| `type` | ClickHouse data type.
ClickHouse tries to cast value from dictionary to the specified data type. For example, for MySQL, the field might be `TEXT`, `VARCHAR`, or `BLOB` in the MySQL source table, but it can be uploaded as `String` in ClickHouse.
[Nullable](../../../sql-reference/data-types/nullable.md) is currently supported for [Flat](external-dicts-dict-layout.md#flat), [Hashed](external-dicts-dict-layout.md#dicts-external_dicts_dict_layout-hashed), [ComplexKeyHashed](external-dicts-dict-layout.md#complex-key-hashed), [Direct](external-dicts-dict-layout.md#direct), [ComplexKeyDirect](external-dicts-dict-layout.md#complex-key-direct), [RangeHashed](external-dicts-dict-layout.md#range-hashed), [Polygon](external-dicts-dict-polygon.md) dictionaries. In [Cache](external-dicts-dict-layout.md#cache), [ComplexKeyCache](external-dicts-dict-layout.md#complex-key-cache), [SSDCache](external-dicts-dict-layout.md#ssd-cache), [SSDComplexKeyCache](external-dicts-dict-layout.md#complex-key-ssd-cache), [IPTrie](external-dicts-dict-layout.md#ip-trie) dictionaries `Nullable` types are not supported. | Yes |
+| `null_value` | Default value for a non-existing element.
In the example, it is an empty string. [NULL](../../syntax.md#null-literal) value can be used only for the `Nullable` types (see the previous line with types description). | Yes |
| `expression` | [Expression](../../../sql-reference/syntax.md#syntax-expressions) that ClickHouse executes on the value.
The expression can be a column name in the remote SQL database. Thus, you can use it to create an alias for the remote column.
Default value: no expression. | No |
| `hierarchical` | If `true`, the attribute contains the value of a parent key for the current key. See [Hierarchical Dictionaries](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-hierarchical.md).
Default value: `false`. | No |
| `injective` | Flag that shows whether the `id -> attribute` image is [injective](https://en.wikipedia.org/wiki/Injective_function).
If `true`, ClickHouse can automatically place after the `GROUP BY` clause the requests to dictionaries with injection. Usually it significantly reduces the amount of such requests.
Default value: `false`. | No |
| `is_object_id` | Flag that shows whether the query is executed for a MongoDB document by `ObjectID`.
Default value: `false`. | No |
-## See Also {#see-also}
+**See Also**
- [Functions for working with external dictionaries](../../../sql-reference/functions/ext-dict-functions.md).
diff --git a/docs/en/sql-reference/dictionaries/index.md b/docs/en/sql-reference/dictionaries/index.md
index fa127dab103..22f4182a1c0 100644
--- a/docs/en/sql-reference/dictionaries/index.md
+++ b/docs/en/sql-reference/dictionaries/index.md
@@ -10,8 +10,6 @@ A dictionary is a mapping (`key -> attributes`) that is convenient for various t
ClickHouse supports special functions for working with dictionaries that can be used in queries. It is easier and more efficient to use dictionaries with functions than a `JOIN` with reference tables.
-[NULL](../../sql-reference/syntax.md#null-literal) values can’t be stored in a dictionary.
-
ClickHouse supports:
- [Built-in dictionaries](../../sql-reference/dictionaries/internal-dicts.md#internal_dicts) with a specific [set of functions](../../sql-reference/functions/ym-dict-functions.md).
diff --git a/docs/en/sql-reference/functions/bitmap-functions.md b/docs/en/sql-reference/functions/bitmap-functions.md
index 7ec400949e9..4875532605e 100644
--- a/docs/en/sql-reference/functions/bitmap-functions.md
+++ b/docs/en/sql-reference/functions/bitmap-functions.md
@@ -33,7 +33,7 @@ SELECT bitmapBuild([1, 2, 3, 4, 5]) AS res, toTypeName(res);
``` text
┌─res─┬─toTypeName(bitmapBuild([1, 2, 3, 4, 5]))─────┐
-│ │ AggregateFunction(groupBitmap, UInt8) │
+│ │ AggregateFunction(groupBitmap, UInt8) │
└─────┴──────────────────────────────────────────────┘
```
diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md
index 6b26dae4546..b0636b0305e 100644
--- a/docs/en/sql-reference/functions/date-time-functions.md
+++ b/docs/en/sql-reference/functions/date-time-functions.md
@@ -147,6 +147,9 @@ Result:
└────────────────┘
```
+!!! attention "Attention"
+ The return type `toStartOf*` functions described below is `Date` or `DateTime`. Though these functions can take `DateTime64` as an argument, passing them a `DateTime64` that is out of normal range (years 1970 - 2105) will give incorrect result.
+
## toStartOfYear {#tostartofyear}
Rounds down a date or date with time to the first day of the year.
@@ -388,13 +391,13 @@ SELECT toDate('2016-12-27') AS date, toYearWeek(date) AS yearWeek0, toYearWeek(d
Truncates date and time data to the specified part of date.
-**Syntax**
+**Syntax**
``` sql
date_trunc(unit, value[, timezone])
```
-Alias: `dateTrunc`.
+Alias: `dateTrunc`.
**Arguments**
@@ -457,13 +460,13 @@ Result:
Adds the time interval or date interval to the provided date or date with time.
-**Syntax**
+**Syntax**
``` sql
date_add(unit, value, date)
```
-Aliases: `dateAdd`, `DATE_ADD`.
+Aliases: `dateAdd`, `DATE_ADD`.
**Arguments**
@@ -478,7 +481,7 @@ Aliases: `dateAdd`, `DATE_ADD`.
- `month`
- `quarter`
- `year`
-
+
- `value` — Value of interval to add. [Int](../../sql-reference/data-types/int-uint.md).
- `date` — The date or date with time to which `value` is added. [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md).
@@ -583,7 +586,7 @@ Aliases: `dateSub`, `DATE_SUB`.
- `month`
- `quarter`
- `year`
-
+
- `value` — Value of interval to subtract. [Int](../../sql-reference/data-types/int-uint.md).
- `date` — The date or date with time from which `value` is subtracted. [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md).
@@ -613,16 +616,16 @@ Result:
Adds the specified time value with the provided date or date time value.
-**Syntax**
+**Syntax**
``` sql
timestamp_add(date, INTERVAL value unit)
```
-Aliases: `timeStampAdd`, `TIMESTAMP_ADD`.
+Aliases: `timeStampAdd`, `TIMESTAMP_ADD`.
**Arguments**
-
+
- `date` — Date or date with time. [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md).
- `value` — Value of interval to add. [Int](../../sql-reference/data-types/int-uint.md).
- `unit` — The type of interval to add. [String](../../sql-reference/data-types/string.md).
@@ -642,7 +645,7 @@ Aliases: `timeStampAdd`, `TIMESTAMP_ADD`.
Date or date with time with the specified `value` expressed in `unit` added to `date`.
Type: [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md).
-
+
**Example**
Query:
@@ -663,13 +666,13 @@ Result:
Subtracts the time interval from the provided date or date with time.
-**Syntax**
+**Syntax**
``` sql
timestamp_sub(unit, value, date)
```
-Aliases: `timeStampSub`, `TIMESTAMP_SUB`.
+Aliases: `timeStampSub`, `TIMESTAMP_SUB`.
**Arguments**
@@ -684,7 +687,7 @@ Aliases: `timeStampSub`, `TIMESTAMP_SUB`.
- `month`
- `quarter`
- `year`
-
+
- `value` — Value of interval to subtract. [Int](../../sql-reference/data-types/int-uint.md).
- `date` — Date or date with time. [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md).
@@ -709,12 +712,12 @@ Result:
│ 2018-07-18 01:02:03 │
└──────────────────────────────────────────────────────────────┘
```
-
+
## now {#now}
-Returns the current date and time.
+Returns the current date and time.
-**Syntax**
+**Syntax**
``` sql
now([timezone])
@@ -1069,4 +1072,3 @@ Result:
│ 2020-01-01 │
└────────────────────────────────────┘
```
-
diff --git a/docs/en/sql-reference/functions/hash-functions.md b/docs/en/sql-reference/functions/hash-functions.md
index c60067b06af..0ea4cfd6fbe 100644
--- a/docs/en/sql-reference/functions/hash-functions.md
+++ b/docs/en/sql-reference/functions/hash-functions.md
@@ -437,13 +437,13 @@ A [FixedString(16)](../../sql-reference/data-types/fixedstring.md) data type has
**Example**
``` sql
-SELECT murmurHash3_128('example_string') AS MurmurHash3, toTypeName(MurmurHash3) AS type;
+SELECT hex(murmurHash3_128('example_string')) AS MurmurHash3, toTypeName(MurmurHash3) AS type;
```
``` text
-┌─MurmurHash3──────┬─type────────────┐
-│ 6�1�4"S5KT�~~q │ FixedString(16) │
-└──────────────────┴─────────────────┘
+┌─MurmurHash3──────────────────────┬─type───┐
+│ 368A1A311CB7342253354B548E7E7E71 │ String │
+└──────────────────────────────────┴────────┘
```
## xxHash32, xxHash64 {#hash-functions-xxhash32}
diff --git a/docs/en/sql-reference/functions/json-functions.md b/docs/en/sql-reference/functions/json-functions.md
index ca6ef684faf..d545a0ae4e6 100644
--- a/docs/en/sql-reference/functions/json-functions.md
+++ b/docs/en/sql-reference/functions/json-functions.md
@@ -16,46 +16,60 @@ The following assumptions are made:
## visitParamHas(params, name) {#visitparamhasparams-name}
-Checks whether there is a field with the ‘name’ name.
+Checks whether there is a field with the `name` name.
+
+Alias: `simpleJSONHas`.
## visitParamExtractUInt(params, name) {#visitparamextractuintparams-name}
-Parses UInt64 from the value of the field named ‘name’. If this is a string field, it tries to parse a number from the beginning of the string. If the field doesn’t exist, or it exists but doesn’t contain a number, it returns 0.
+Parses UInt64 from the value of the field named `name`. If this is a string field, it tries to parse a number from the beginning of the string. If the field doesn’t exist, or it exists but doesn’t contain a number, it returns 0.
+
+Alias: `simpleJSONExtractUInt`.
## visitParamExtractInt(params, name) {#visitparamextractintparams-name}
The same as for Int64.
+Alias: `simpleJSONExtractInt`.
+
## visitParamExtractFloat(params, name) {#visitparamextractfloatparams-name}
The same as for Float64.
+Alias: `simpleJSONExtractFloat`.
+
## visitParamExtractBool(params, name) {#visitparamextractboolparams-name}
Parses a true/false value. The result is UInt8.
+Alias: `simpleJSONExtractBool`.
+
## visitParamExtractRaw(params, name) {#visitparamextractrawparams-name}
Returns the value of a field, including separators.
+Alias: `simpleJSONExtractRaw`.
+
Examples:
``` sql
-visitParamExtractRaw('{"abc":"\\n\\u0000"}', 'abc') = '"\\n\\u0000"'
-visitParamExtractRaw('{"abc":{"def":[1,2,3]}}', 'abc') = '{"def":[1,2,3]}'
+visitParamExtractRaw('{"abc":"\\n\\u0000"}', 'abc') = '"\\n\\u0000"';
+visitParamExtractRaw('{"abc":{"def":[1,2,3]}}', 'abc') = '{"def":[1,2,3]}';
```
## visitParamExtractString(params, name) {#visitparamextractstringparams-name}
Parses the string in double quotes. The value is unescaped. If unescaping failed, it returns an empty string.
+Alias: `simpleJSONExtractString`.
+
Examples:
``` sql
-visitParamExtractString('{"abc":"\\n\\u0000"}', 'abc') = '\n\0'
-visitParamExtractString('{"abc":"\\u263a"}', 'abc') = '☺'
-visitParamExtractString('{"abc":"\\u263"}', 'abc') = ''
-visitParamExtractString('{"abc":"hello}', 'abc') = ''
+visitParamExtractString('{"abc":"\\n\\u0000"}', 'abc') = '\n\0';
+visitParamExtractString('{"abc":"\\u263a"}', 'abc') = '☺';
+visitParamExtractString('{"abc":"\\u263"}', 'abc') = '';
+visitParamExtractString('{"abc":"hello}', 'abc') = '';
```
There is currently no support for code points in the format `\uXXXX\uYYYY` that are not from the basic multilingual plane (they are converted to CESU-8 instead of UTF-8).
diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md
index c40aa3d1eae..9d7743e186f 100644
--- a/docs/en/sql-reference/functions/other-functions.md
+++ b/docs/en/sql-reference/functions/other-functions.md
@@ -1192,6 +1192,109 @@ SELECT defaultValueOfTypeName('Nullable(Int8)')
└──────────────────────────────────────────┘
```
+## indexHint {#indexhint}
+The function is intended for debugging and introspection purposes. The function ignores it's argument and always returns 1. Arguments are not even evaluated.
+
+But for the purpose of index analysis, the argument of this function is analyzed as if it was present directly without being wrapped inside `indexHint` function. This allows to select data in index ranges by the corresponding condition but without further filtering by this condition. The index in ClickHouse is sparse and using `indexHint` will yield more data than specifying the same condition directly.
+
+**Syntax**
+
+```sql
+SELECT * FROM table WHERE indexHint()
+```
+
+**Returned value**
+
+1. Type: [Uint8](https://clickhouse.yandex/docs/en/data_types/int_uint/#diapazony-uint).
+
+**Example**
+
+Here is the example of test data from the table [ontime](../../getting-started/example-datasets/ontime.md).
+
+Input table:
+
+```sql
+SELECT count() FROM ontime
+```
+
+```text
+┌─count()─┐
+│ 4276457 │
+└─────────┘
+```
+
+The table has indexes on the fields `(FlightDate, (Year, FlightDate))`.
+
+Create a query, where the index is not used.
+
+Query:
+
+```sql
+SELECT FlightDate AS k, count() FROM ontime GROUP BY k ORDER BY k
+```
+
+ClickHouse processed the entire table (`Processed 4.28 million rows`).
+
+Result:
+
+```text
+┌──────────k─┬─count()─┐
+│ 2017-01-01 │ 13970 │
+│ 2017-01-02 │ 15882 │
+........................
+│ 2017-09-28 │ 16411 │
+│ 2017-09-29 │ 16384 │
+│ 2017-09-30 │ 12520 │
+└────────────┴─────────┘
+```
+
+To apply the index, select a specific date.
+
+Query:
+
+```sql
+SELECT FlightDate AS k, count() FROM ontime WHERE k = '2017-09-15' GROUP BY k ORDER BY k
+```
+
+By using the index, ClickHouse processed a significantly smaller number of rows (`Processed 32.74 thousand rows`).
+
+Result:
+
+```text
+┌──────────k─┬─count()─┐
+│ 2017-09-15 │ 16428 │
+└────────────┴─────────┘
+```
+
+Now wrap the expression `k = '2017-09-15'` into `indexHint` function.
+
+Query:
+
+```sql
+SELECT
+ FlightDate AS k,
+ count()
+FROM ontime
+WHERE indexHint(k = '2017-09-15')
+GROUP BY k
+ORDER BY k ASC
+```
+
+ClickHouse used the index in the same way as the previous time (`Processed 32.74 thousand rows`).
+The expression `k = '2017-09-15'` was not used when generating the result.
+In examle the `indexHint` function allows to see adjacent dates.
+
+Result:
+
+```text
+┌──────────k─┬─count()─┐
+│ 2017-09-14 │ 7071 │
+│ 2017-09-15 │ 16428 │
+│ 2017-09-16 │ 1077 │
+│ 2017-09-30 │ 8167 │
+└────────────┴─────────┘
+```
+
## replicate {#other-functions-replicate}
Creates an array with a single value.
diff --git a/docs/en/sql-reference/functions/string-functions.md b/docs/en/sql-reference/functions/string-functions.md
index 3d3caaf6e23..85570cb408d 100644
--- a/docs/en/sql-reference/functions/string-functions.md
+++ b/docs/en/sql-reference/functions/string-functions.md
@@ -649,3 +649,65 @@ Result:
- [List of XML and HTML character entity references](https://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_entity_references)
+
+## extractTextFromHTML {#extracttextfromhtml}
+
+A function to extract text from HTML or XHTML.
+It does not necessarily 100% conform to any of the HTML, XML or XHTML standards, but the implementation is reasonably accurate and it is fast. The rules are the following:
+
+1. Comments are skipped. Example: ``. Comment must end with `-->`. Nested comments are not possible.
+Note: constructions like `` and `` are not valid comments in HTML but they are skipped by other rules.
+2. CDATA is pasted verbatim. Note: CDATA is XML/XHTML specific. But it is processed for "best-effort" approach.
+3. `script` and `style` elements are removed with all their content. Note: it is assumed that closing tag cannot appear inside content. For example, in JS string literal has to be escaped like `"<\/script>"`.
+Note: comments and CDATA are possible inside `script` or `style` - then closing tags are not searched inside CDATA. Example: `]]>`. But they are still searched inside comments. Sometimes it becomes complicated: ` var y = "-->"; alert(x + y);`
+Note: `script` and `style` can be the names of XML namespaces - then they are not treated like usual `script` or `style` elements. Example: `Hello`.
+Note: whitespaces are possible after closing tag name: `` but not before: `< / script>`.
+4. Other tags or tag-like elements are skipped without inner content. Example: `.`
+Note: it is expected that this HTML is illegal: ``
+Note: it also skips something like tags: `<>`, ``, etc.
+Note: tag without end is skipped to the end of input: `world`, `Helloworld` - there is no whitespace in HTML, but the function inserts it. Also consider: `Helloworld
`, `Hello
world`. This behavior is reasonable for data analysis, e.g. to convert HTML to a bag of words.
+7. Also note that correct handling of whitespaces requires the support of `` and CSS `display` and `white-space` properties.
+
+**Syntax**
+
+``` sql
+extractTextFromHTML(x)
+```
+
+**Arguments**
+
+- `x` — input text. [String](../../sql-reference/data-types/string.md).
+
+**Returned value**
+
+- Extracted text.
+
+Type: [String](../../sql-reference/data-types/string.md).
+
+**Example**
+
+The first example contains several tags and a comment and also shows whitespace processing.
+The second example shows `CDATA` and `script` tag processing.
+In the third example text is extracted from the full HTML response received by the [url](../../sql-reference/table-functions/url.md) function.
+
+Query:
+
+``` sql
+SELECT extractTextFromHTML(' A text withtags.
');
+SELECT extractTextFromHTML('CDATA]]> ');
+SELECT extractTextFromHTML(html) FROM url('http://www.donothingfor2minutes.com/', RawBLOB, 'html String');
+```
+
+Result:
+
+``` text
+A text with tags .
+The content within CDATA
+Do Nothing for 2 Minutes 2:00
+```
diff --git a/docs/en/sql-reference/statements/alter/column.md b/docs/en/sql-reference/statements/alter/column.md
index 3ece30be5b8..d661bd4cd59 100644
--- a/docs/en/sql-reference/statements/alter/column.md
+++ b/docs/en/sql-reference/statements/alter/column.md
@@ -74,6 +74,9 @@ Deletes the column with the name `name`. If the `IF EXISTS` clause is specified,
Deletes data from the file system. Since this deletes entire files, the query is completed almost instantly.
+!!! warning "Warning"
+ You can’t delete a column if it is referenced by [materialized view](../../../sql-reference/statements/create/view.md#materialized). Otherwise, it returns an error.
+
Example:
``` sql
@@ -180,7 +183,7 @@ ALTER TABLE table_name MODIFY column_name REMOVE property;
ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL;
```
-## See Also
+**See Also**
- [REMOVE TTL](ttl.md).
diff --git a/docs/en/sql-reference/statements/alter/partition.md b/docs/en/sql-reference/statements/alter/partition.md
index f7183ba525c..b22f89928b9 100644
--- a/docs/en/sql-reference/statements/alter/partition.md
+++ b/docs/en/sql-reference/statements/alter/partition.md
@@ -16,7 +16,7 @@ The following operations with [partitions](../../../engines/table-engines/merget
- [CLEAR COLUMN IN PARTITION](#alter_clear-column-partition) — Resets the value of a specified column in a partition.
- [CLEAR INDEX IN PARTITION](#alter_clear-index-partition) — Resets the specified secondary index in a partition.
- [FREEZE PARTITION](#alter_freeze-partition) — Creates a backup of a partition.
-- [FETCH PARTITION](#alter_fetch-partition) — Downloads a partition from another server.
+- [FETCH PARTITION\|PART](#alter_fetch-partition) — Downloads a part or partition from another server.
- [MOVE PARTITION\|PART](#alter_move-partition) — Move partition/data part to another disk or volume.
@@ -88,12 +88,10 @@ Read more about setting the partition expression in a section [How to specify th
This query is replicated. The replica-initiator checks whether there is data in the `detached` directory.
If data exists, the query checks its integrity. If everything is correct, the query adds the data to the table.
-If the non-initiator replica, receiving the attach command, finds the part with the correct checksums in its own
-`detached` folder, it attaches the data without fetching it from other replicas.
+If the non-initiator replica, receiving the attach command, finds the part with the correct checksums in its own `detached` folder, it attaches the data without fetching it from other replicas.
If there is no part with the correct checksums, the data is downloaded from any replica having the part.
-You can put data to the `detached` directory on one replica and use the `ALTER ... ATTACH` query to add it to the
-table on all replicas.
+You can put data to the `detached` directory on one replica and use the `ALTER ... ATTACH` query to add it to the table on all replicas.
## ATTACH PARTITION FROM {#alter_attach-partition-from}
@@ -101,8 +99,8 @@ table on all replicas.
ALTER TABLE table2 ATTACH PARTITION partition_expr FROM table1
```
-This query copies the data partition from the `table1` to `table2`.
-Note that data won't be deleted neither from `table1` nor from `table2`.
+This query copies the data partition from `table1` to `table2`.
+Note that data will be deleted neither from `table1` nor from `table2`.
For the query to run successfully, the following conditions must be met:
@@ -198,29 +196,35 @@ ALTER TABLE table_name CLEAR INDEX index_name IN PARTITION partition_expr
The query works similar to `CLEAR COLUMN`, but it resets an index instead of a column data.
-## FETCH PARTITION {#alter_fetch-partition}
+## FETCH PARTITION|PART {#alter_fetch-partition}
``` sql
-ALTER TABLE table_name FETCH PARTITION partition_expr FROM 'path-in-zookeeper'
+ALTER TABLE table_name FETCH PARTITION|PART partition_expr FROM 'path-in-zookeeper'
```
Downloads a partition from another server. This query only works for the replicated tables.
The query does the following:
-1. Downloads the partition from the specified shard. In ‘path-in-zookeeper’ you must specify a path to the shard in ZooKeeper.
+1. Downloads the partition|part from the specified shard. In ‘path-in-zookeeper’ you must specify a path to the shard in ZooKeeper.
2. Then the query puts the downloaded data to the `detached` directory of the `table_name` table. Use the [ATTACH PARTITION\|PART](#alter_attach-partition) query to add the data to the table.
For example:
+1. FETCH PARTITION
``` sql
ALTER TABLE users FETCH PARTITION 201902 FROM '/clickhouse/tables/01-01/visits';
ALTER TABLE users ATTACH PARTITION 201902;
```
+2. FETCH PART
+``` sql
+ALTER TABLE users FETCH PART 201901_2_2_0 FROM '/clickhouse/tables/01-01/visits';
+ALTER TABLE users ATTACH PART 201901_2_2_0;
+```
Note that:
-- The `ALTER ... FETCH PARTITION` query isn’t replicated. It places the partition to the `detached` directory only on the local server.
+- The `ALTER ... FETCH PARTITION|PART` query isn’t replicated. It places the part or partition to the `detached` directory only on the local server.
- The `ALTER TABLE ... ATTACH` query is replicated. It adds the data to all replicas. The data is added to one of the replicas from the `detached` directory, and to the others - from neighboring replicas.
Before downloading, the system checks if the partition exists and the table structure matches. The most appropriate replica is selected automatically from the healthy replicas.
diff --git a/docs/en/sql-reference/statements/alter/ttl.md b/docs/en/sql-reference/statements/alter/ttl.md
index aa7ee838e10..9cd63d3b8fe 100644
--- a/docs/en/sql-reference/statements/alter/ttl.md
+++ b/docs/en/sql-reference/statements/alter/ttl.md
@@ -79,7 +79,7 @@ The `TTL` is no longer there, so the second row is not deleted:
└───────────────────────┴─────────┴──────────────┘
```
-### See Also
+**See Also**
- More about the [TTL-expression](../../../sql-reference/statements/create/table.md#ttl-expression).
- Modify column [with TTL](../../../sql-reference/statements/alter/column.md#alter_modify-column).
diff --git a/docs/en/sql-reference/statements/attach.md b/docs/en/sql-reference/statements/attach.md
index ffb577a8839..01783e9cb2f 100644
--- a/docs/en/sql-reference/statements/attach.md
+++ b/docs/en/sql-reference/statements/attach.md
@@ -5,13 +5,14 @@ toc_title: ATTACH
# ATTACH Statement {#attach}
-This query is exactly the same as [CREATE](../../sql-reference/statements/create/table.md), but
+Attaches the table, for example, when moving a database to another server.
-- Instead of the word `CREATE` it uses the word `ATTACH`.
-- The query does not create data on the disk, but assumes that data is already in the appropriate places, and just adds information about the table to the server.
- After executing an ATTACH query, the server will know about the existence of the table.
+The query does not create data on the disk, but assumes that data is already in the appropriate places, and just adds information about the table to the server. After executing an `ATTACH` query, the server will know about the existence of the table.
-If the table was previously detached ([DETACH](../../sql-reference/statements/detach.md)), meaning that its structure is known, you can use shorthand without defining the structure.
+If the table was previously detached ([DETACH](../../sql-reference/statements/detach.md)) query, meaning that its structure is known, you can use shorthand without defining the structure.
+
+## Syntax Forms {#syntax-forms}
+### Attach Existing Table {#attach-existing-table}
``` sql
ATTACH TABLE [IF NOT EXISTS] [db.]name [ON CLUSTER cluster]
@@ -21,4 +22,38 @@ This query is used when starting the server. The server stores table metadata as
If the table was detached permanently, it won't be reattached at the server start, so you need to use `ATTACH` query explicitly.
-[Original article](https://clickhouse.tech/docs/en/sql-reference/statements/attach/)
+### Сreate New Table And Attach Data {#create-new-table-and-attach-data}
+
+**With specify path to table data**
+
+```sql
+ATTACH TABLE name FROM 'path/to/data/' (col1 Type1, ...)
+```
+
+It creates new table with provided structure and attaches table data from provided directory in `user_files`.
+
+**Example**
+
+Query:
+
+```sql
+DROP TABLE IF EXISTS test;
+INSERT INTO TABLE FUNCTION file('01188_attach/test/data.TSV', 'TSV', 's String, n UInt8') VALUES ('test', 42);
+ATTACH TABLE test FROM '01188_attach/test' (s String, n UInt8) ENGINE = File(TSV);
+SELECT * FROM test;
+```
+Result:
+
+```sql
+┌─s────┬──n─┐
+│ test │ 42 │
+└──────┴────┘
+```
+
+**With specify table UUID** (Only for `Atomic` database)
+
+```sql
+ATTACH TABLE name UUID '' (col1 Type1, ...)
+```
+
+It creates new table with provided structure and attaches data from table with the specified UUID.
\ No newline at end of file
diff --git a/docs/en/sql-reference/statements/check-table.md b/docs/en/sql-reference/statements/check-table.md
index 450447acaf8..65e6238ebbc 100644
--- a/docs/en/sql-reference/statements/check-table.md
+++ b/docs/en/sql-reference/statements/check-table.md
@@ -30,9 +30,36 @@ Performed over the tables with another table engines causes an exception.
Engines from the `*Log` family don’t provide automatic data recovery on failure. Use the `CHECK TABLE` query to track data loss in a timely manner.
-For `MergeTree` family engines, the `CHECK TABLE` query shows a check status for every individual data part of a table on the local server.
+## Checking the MergeTree Family Tables {#checking-mergetree-tables}
-**If the data is corrupted**
+For `MergeTree` family engines, if [check_query_single_value_result](../../operations/settings/settings.md#check_query_single_value_result) = 0, the `CHECK TABLE` query shows a check status for every individual data part of a table on the local server.
+
+```sql
+SET check_query_single_value_result = 0;
+CHECK TABLE test_table;
+```
+
+```text
+┌─part_path─┬─is_passed─┬─message─┐
+│ all_1_4_1 │ 1 │ │
+│ all_1_4_2 │ 1 │ │
+└───────────┴───────────┴─────────┘
+```
+
+If `check_query_single_value_result` = 0, the `CHECK TABLE` query shows the general table check status.
+
+```sql
+SET check_query_single_value_result = 1;
+CHECK TABLE test_table;
+```
+
+```text
+┌─result─┐
+│ 1 │
+└────────┘
+```
+
+## If the Data Is Corrupted {#if-data-is-corrupted}
If the table is corrupted, you can copy the non-corrupted data to another table. To do this:
diff --git a/docs/en/sql-reference/statements/create/row-policy.md b/docs/en/sql-reference/statements/create/row-policy.md
index cbe639c6fc5..5a1fa218fad 100644
--- a/docs/en/sql-reference/statements/create/row-policy.md
+++ b/docs/en/sql-reference/statements/create/row-policy.md
@@ -5,39 +5,81 @@ toc_title: ROW POLICY
# CREATE ROW POLICY {#create-row-policy-statement}
-Creates [filters for rows](../../../operations/access-rights.md#row-policy-management), which a user can read from a table.
+Creates a [row policy](../../../operations/access-rights.md#row-policy-management), i.e. a filter used to determine which rows a user can read from a table.
Syntax:
``` sql
CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name1 [ON CLUSTER cluster_name1] ON [db1.]table1
[, policy_name2 [ON CLUSTER cluster_name2] ON [db2.]table2 ...]
+ [FOR SELECT] USING condition
[AS {PERMISSIVE | RESTRICTIVE}]
- [FOR SELECT]
- [USING condition]
[TO {role1 [, role2 ...] | ALL | ALL EXCEPT role1 [, role2 ...]}]
```
-`ON CLUSTER` clause allows creating row policies on a cluster, see [Distributed DDL](../../../sql-reference/distributed-ddl.md).
+## USING Clause {#create-row-policy-using}
-## AS Clause {#create-row-policy-as}
-
-Using this section you can create permissive or restrictive policies.
-
-Permissive policy grants access to rows. Permissive policies which apply to the same table are combined together using the boolean `OR` operator. Policies are permissive by default.
-
-Restrictive policy restricts access to rows. Restrictive policies which apply to the same table are combined together using the boolean `AND` operator.
-
-Restrictive policies apply to rows that passed the permissive filters. If you set restrictive policies but no permissive policies, the user can’t get any row from the table.
+Allows to specify a condition to filter rows. An user will see a row if the condition is calculated to non-zero for the row.
## TO Clause {#create-row-policy-to}
-In the section `TO` you can provide a mixed list of roles and users, for example, `CREATE ROW POLICY ... TO accountant, john@localhost`.
+In the section `TO` you can provide a list of users and roles this policy should work for. For example, `CREATE ROW POLICY ... TO accountant, john@localhost`.
-Keyword `ALL` means all the ClickHouse users including current user. Keywords `ALL EXCEPT` allow to exclude some users from the all users list, for example, `CREATE ROW POLICY ... TO ALL EXCEPT accountant, john@localhost`
+Keyword `ALL` means all the ClickHouse users including current user. Keyword `ALL EXCEPT` allow to exclude some users from the all users list, for example, `CREATE ROW POLICY ... TO ALL EXCEPT accountant, john@localhost`
-## Examples {#examples}
+!!! note "Note"
+ If there are no row policies defined for a table then any user can `SELECT` all the row from the table. Defining one or more row policies for the table makes the access to the table depending on the row policies no matter if those row policies are defined for the current user or not. For example, the following policy
+
+ `CREATE ROW POLICY pol1 ON mydb.table1 USING b=1 TO mira, peter`
-`CREATE ROW POLICY filter ON mydb.mytable FOR SELECT USING a<1000 TO accountant, john@localhost`
+ forbids the users `mira` and `peter` to see the rows with `b != 1`, and any non-mentioned user (e.g., the user `paul`) will see no rows from `mydb.table1` at all.
+
+ If that's not desirable it can't be fixed by adding one more row policy, like the following:
-`CREATE ROW POLICY filter ON mydb.mytable FOR SELECT USING a<1000 TO ALL EXCEPT mira`
+ `CREATE ROW POLICY pol2 ON mydb.table1 USING 1 TO ALL EXCEPT mira, peter`
+
+## AS Clause {#create-row-policy-as}
+
+It's allowed to have more than one policy enabled on the same table for the same user at the one time. So we need a way to combine the conditions from multiple policies.
+
+By default policies are combined using the boolean `OR` operator. For example, the following policies
+
+``` sql
+CREATE ROW POLICY pol1 ON mydb.table1 USING b=1 TO mira, peter
+CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 TO peter, antonio
+```
+
+enables the user `peter` to see rows with either `b=1` or `c=2`.
+
+The `AS` clause specifies how policies should be combined with other policies. Policies can be either permissive or restrictive. By default policies are permissive, which means they are combined using the boolean `OR` operator.
+
+A policy can be defined as restrictive as an alternative. Restrictive policies are combined using the boolean `AND` operator.
+
+Here is the general formula:
+
+```
+row_is_visible = (one or more of the permissive policies' conditions are non-zero) AND
+ (all of the restrictive policies's conditions are non-zero)
+```
+
+For example, the following policies
+
+``` sql
+CREATE ROW POLICY pol1 ON mydb.table1 USING b=1 TO mira, peter
+CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 AS RESTRICTIVE TO peter, antonio
+```
+
+enables the user `peter` to see rows only if both `b=1` AND `c=2`.
+
+## ON CLUSTER Clause {#create-row-policy-on-cluster}
+
+Allows creating row policies on a cluster, see [Distributed DDL](../../../sql-reference/distributed-ddl.md).
+
+
+## Examples
+
+`CREATE ROW POLICY filter1 ON mydb.mytable USING a<1000 TO accountant, john@localhost`
+
+`CREATE ROW POLICY filter2 ON mydb.mytable USING a<1000 AND b=5 TO ALL EXCEPT mira`
+
+`CREATE ROW POLICY filter3 ON mydb.mytable USING 1 TO admin`
diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md
index b98888f7bfa..5f1f0151350 100644
--- a/docs/en/sql-reference/statements/create/table.md
+++ b/docs/en/sql-reference/statements/create/table.md
@@ -50,15 +50,32 @@ Creates a table with the same result as that of the [table function](../../../sq
### From SELECT query {#from-select-query}
``` sql
-CREATE TABLE [IF NOT EXISTS] [db.]table_name ENGINE = engine AS SELECT ...
+CREATE TABLE [IF NOT EXISTS] [db.]table_name[(name1 [type1], name2 [type2], ...)] ENGINE = engine AS SELECT ...
```
-Creates a table with a structure like the result of the `SELECT` query, with the `engine` engine, and fills it with data from SELECT.
+Creates a table with a structure like the result of the `SELECT` query, with the `engine` engine, and fills it with data from `SELECT`. Also you can explicitly specify columns description.
-In all cases, if `IF NOT EXISTS` is specified, the query won’t return an error if the table already exists. In this case, the query won’t do anything.
+If the table already exists and `IF NOT EXISTS` is specified, the query won’t do anything.
There can be other clauses after the `ENGINE` clause in the query. See detailed documentation on how to create tables in the descriptions of [table engines](../../../engines/table-engines/index.md#table_engines).
+**Example**
+
+Query:
+
+``` sql
+CREATE TABLE t1 (x String) ENGINE = Memory AS SELECT 1;
+SELECT x, toTypeName(x) FROM t1;
+```
+
+Result:
+
+```text
+┌─x─┬─toTypeName(x)─┐
+│ 1 │ String │
+└───┴───────────────┘
+```
+
## NULL Or NOT NULL Modifiers {#null-modifiers}
`NULL` and `NOT NULL` modifiers after data type in column definition allow or do not allow it to be [Nullable](../../../sql-reference/data-types/nullable.md#data_type-nullable).
@@ -287,7 +304,9 @@ REPLACE TABLE myOldTable SELECT * FROM myOldTable WHERE CounterID <12345;
### Syntax
-{CREATE [OR REPLACE]|REPLACE} TABLE [db.]table_name
+``` sql
+{CREATE [OR REPLACE] | REPLACE} TABLE [db.]table_name
+```
All syntax forms for `CREATE` query also work for this query. `REPLACE` for a non-existent table will cause an error.
diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md
index 0afc9b5b95f..89f35b5f701 100644
--- a/docs/en/sql-reference/statements/grant.md
+++ b/docs/en/sql-reference/statements/grant.md
@@ -91,7 +91,7 @@ Hierarchy of privileges:
- `ALTER ADD CONSTRAINT`
- `ALTER DROP CONSTRAINT`
- `ALTER TTL`
- - `ALTER MATERIALIZE TTL`
+ - `ALTER MATERIALIZE TTL`
- `ALTER SETTINGS`
- `ALTER MOVE PARTITION`
- `ALTER FETCH PARTITION`
@@ -102,9 +102,9 @@ Hierarchy of privileges:
- [CREATE](#grant-create)
- `CREATE DATABASE`
- `CREATE TABLE`
+ - `CREATE TEMPORARY TABLE`
- `CREATE VIEW`
- `CREATE DICTIONARY`
- - `CREATE TEMPORARY TABLE`
- [DROP](#grant-drop)
- `DROP DATABASE`
- `DROP TABLE`
@@ -150,7 +150,7 @@ Hierarchy of privileges:
- `SYSTEM RELOAD`
- `SYSTEM RELOAD CONFIG`
- `SYSTEM RELOAD DICTIONARY`
- - `SYSTEM RELOAD EMBEDDED DICTIONARIES`
+ - `SYSTEM RELOAD EMBEDDED DICTIONARIES`
- `SYSTEM MERGES`
- `SYSTEM TTL MERGES`
- `SYSTEM FETCHES`
@@ -276,10 +276,10 @@ Allows executing [ALTER](../../sql-reference/statements/alter/index.md) queries
- `ALTER ADD CONSTRAINT`. Level: `TABLE`. Aliases: `ADD CONSTRAINT`
- `ALTER DROP CONSTRAINT`. Level: `TABLE`. Aliases: `DROP CONSTRAINT`
- `ALTER TTL`. Level: `TABLE`. Aliases: `ALTER MODIFY TTL`, `MODIFY TTL`
- - `ALTER MATERIALIZE TTL`. Level: `TABLE`. Aliases: `MATERIALIZE TTL`
+ - `ALTER MATERIALIZE TTL`. Level: `TABLE`. Aliases: `MATERIALIZE TTL`
- `ALTER SETTINGS`. Level: `TABLE`. Aliases: `ALTER SETTING`, `ALTER MODIFY SETTING`, `MODIFY SETTING`
- `ALTER MOVE PARTITION`. Level: `TABLE`. Aliases: `ALTER MOVE PART`, `MOVE PARTITION`, `MOVE PART`
- - `ALTER FETCH PARTITION`. Level: `TABLE`. Aliases: `FETCH PARTITION`
+ - `ALTER FETCH PARTITION`. Level: `TABLE`. Aliases: `ALTER FETCH PART`, `FETCH PARTITION`, `FETCH PART`
- `ALTER FREEZE PARTITION`. Level: `TABLE`. Aliases: `FREEZE PARTITION`
- `ALTER VIEW` Level: `GROUP`
- `ALTER VIEW REFRESH`. Level: `VIEW`. Aliases: `ALTER LIVE VIEW REFRESH`, `REFRESH VIEW`
@@ -304,9 +304,9 @@ Allows executing [CREATE](../../sql-reference/statements/create/index.md) and [A
- `CREATE`. Level: `GROUP`
- `CREATE DATABASE`. Level: `DATABASE`
- `CREATE TABLE`. Level: `TABLE`
+ - `CREATE TEMPORARY TABLE`. Level: `GLOBAL`
- `CREATE VIEW`. Level: `VIEW`
- `CREATE DICTIONARY`. Level: `DICTIONARY`
- - `CREATE TEMPORARY TABLE`. Level: `GLOBAL`
**Notes**
@@ -401,7 +401,7 @@ Allows a user to execute [SYSTEM](../../sql-reference/statements/system.md) quer
- `SYSTEM RELOAD`. Level: `GROUP`
- `SYSTEM RELOAD CONFIG`. Level: `GLOBAL`. Aliases: `RELOAD CONFIG`
- `SYSTEM RELOAD DICTIONARY`. Level: `GLOBAL`. Aliases: `SYSTEM RELOAD DICTIONARIES`, `RELOAD DICTIONARY`, `RELOAD DICTIONARIES`
- - `SYSTEM RELOAD EMBEDDED DICTIONARIES`. Level: `GLOBAL`. Aliases: R`ELOAD EMBEDDED DICTIONARIES`
+ - `SYSTEM RELOAD EMBEDDED DICTIONARIES`. Level: `GLOBAL`. Aliases: `RELOAD EMBEDDED DICTIONARIES`
- `SYSTEM MERGES`. Level: `TABLE`. Aliases: `SYSTEM STOP MERGES`, `SYSTEM START MERGES`, `STOP MERGES`, `START MERGES`
- `SYSTEM TTL MERGES`. Level: `TABLE`. Aliases: `SYSTEM STOP TTL MERGES`, `SYSTEM START TTL MERGES`, `STOP TTL MERGES`, `START TTL MERGES`
- `SYSTEM FETCHES`. Level: `TABLE`. Aliases: `SYSTEM STOP FETCHES`, `SYSTEM START FETCHES`, `STOP FETCHES`, `START FETCHES`
diff --git a/docs/en/sql-reference/statements/optimize.md b/docs/en/sql-reference/statements/optimize.md
index 49a7404d76e..247252d3f4e 100644
--- a/docs/en/sql-reference/statements/optimize.md
+++ b/docs/en/sql-reference/statements/optimize.md
@@ -5,13 +5,18 @@ toc_title: OPTIMIZE
# OPTIMIZE Statement {#misc_operations-optimize}
+This query tries to initialize an unscheduled merge of data parts for tables.
+
+!!! warning "Warning"
+ `OPTIMIZE` can’t fix the `Too many parts` error.
+
+**Syntax**
+
``` sql
OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION ID 'partition_id'] [FINAL] [DEDUPLICATE [BY expression]]
```
-This query tries to initialize an unscheduled merge of data parts for tables with a table engine from the [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) family.
-
-The `OPTMIZE` query is also supported for the [MaterializedView](../../engines/table-engines/special/materializedview.md) and the [Buffer](../../engines/table-engines/special/buffer.md) engines. Other table engines aren’t supported.
+The `OPTMIZE` query is supported for [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) family, the [MaterializedView](../../engines/table-engines/special/materializedview.md) and the [Buffer](../../engines/table-engines/special/buffer.md) engines. Other table engines aren’t supported.
When `OPTIMIZE` is used with the [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md) family of table engines, ClickHouse creates a task for merging and waits for execution on all nodes (if the `replication_alter_partitions_sync` setting is enabled).
@@ -21,12 +26,13 @@ When `OPTIMIZE` is used with the [ReplicatedMergeTree](../../engines/table-engin
- If you specify `DEDUPLICATE`, then completely identical rows (unless by-clause is specified) will be deduplicated (all columns are compared), it makes sense only for the MergeTree engine.
-### BY expression {#by-expression}
+## BY expression {#by-expression}
If you want to perform deduplication on custom set of columns rather than on all, you can specify list of columns explicitly or use any combination of [`*`](../../sql-reference/statements/select/index.md#asterisk), [`COLUMNS`](../../sql-reference/statements/select/index.md#columns-expression) or [`EXCEPT`](../../sql-reference/statements/select/index.md#except-modifier) expressions. The explictly written or implicitly expanded list of columns must include all columns specified in row ordering expression (both primary and sorting keys) and partitioning expression (partitioning key).
-Note that `*` behaves just like in `SELECT`: `MATERIALIZED`, and `ALIAS` columns are not used for expansion.
-Also, it is an error to specify empty list of columns, or write an expression that results in an empty list of columns, or deduplicate by an ALIAS column.
+!!! note "Note"
+ Notice that `*` behaves just like in `SELECT`: `MATERIALIZED` and `ALIAS` columns are not used for expansion.
+ Also, it is an error to specify empty list of columns, or write an expression that results in an empty list of columns, or deduplicate by an ALIAS column.
``` sql
OPTIMIZE TABLE table DEDUPLICATE; -- the old one
@@ -39,9 +45,10 @@ OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex') EXCEPT co
OPTIMIZE TABLE table DEDUPLICATE BY COLUMNS('column-matched-by-regex') EXCEPT (colX, colY);
```
-**Example:**
+**Examples**
+
+Create a table:
-A silly synthetic table.
``` sql
CREATE TABLE example (
primary_key Int32,
@@ -56,31 +63,31 @@ PARTITION BY partition_key
ORDER BY (primary_key, secondary_key);
```
+The 'old' deduplicate, all columns are taken into account, i.e. row is removed only if all values in all columns are equal to corresponding values in previous row.
+
``` sql
--- The 'old' deduplicate, all columns are taken into account, i.e. row is removed only if all values in all columns are equal to corresponding values in previous row.
OPTIMIZE TABLE example FINAL DEDUPLICATE;
```
+Deduplicate by all columns that are not `ALIAS` or `MATERIALIZED`: `primary_key`, `secondary_key`, `value`, `partition_key`, and `materialized_value` columns.
+
``` sql
--- Deduplicate by all columns that are not `ALIAS` or `MATERIALIZED`: `primary_key`, `secondary_key`, `value`, `partition_key`, and `materialized_value` columns.
OPTIMIZE TABLE example FINAL DEDUPLICATE BY *;
```
+Deduplicate by all columns that are not `ALIAS` or `MATERIALIZED` and explicitly not `materialized_value`: `primary_key`, `secondary_key`, `value`, and `partition_key` columns.
+
``` sql
--- Deduplicate by all columns that are not `ALIAS` or `MATERIALIZED` and explicitly not `materialized_value`: `primary_key`, `secondary_key`, `value`, and `partition_key` columns.
OPTIMIZE TABLE example FINAL DEDUPLICATE BY * EXCEPT materialized_value;
```
+Deduplicate explicitly by `primary_key`, `secondary_key`, and `partition_key` columns.
``` sql
--- Deduplicate explicitly by `primary_key`, `secondary_key`, and `partition_key` columns.
OPTIMIZE TABLE example FINAL DEDUPLICATE BY primary_key, secondary_key, partition_key;
```
+Deduplicate by any column matching a regex: `primary_key`, `secondary_key`, and `partition_key` columns.
+
``` sql
--- Deduplicate by any column matching a regex: `primary_key`, `secondary_key`, and `partition_key` columns.
OPTIMIZE TABLE example FINAL DEDUPLICATE BY COLUMNS('.*_key');
```
-
-
-!!! warning "Warning"
- `OPTIMIZE` can’t fix the “Too many parts” error.
diff --git a/docs/en/sql-reference/statements/rename.md b/docs/en/sql-reference/statements/rename.md
index 4f14ad016a3..a9dda6ed3b2 100644
--- a/docs/en/sql-reference/statements/rename.md
+++ b/docs/en/sql-reference/statements/rename.md
@@ -5,6 +5,14 @@ toc_title: RENAME
# RENAME Statement {#misc_operations-rename}
+## RENAME DATABASE {#misc_operations-rename_database}
+Renames database, support only for Atomic database engine
+
+```
+RENAME DATABASE atomic_database1 TO atomic_database2 [ON CLUSTER cluster]
+```
+
+## RENAME TABLE {#misc_operations-rename_table}
Renames one or more tables.
``` sql
diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md
index 2348a2a2668..7871894ccac 100644
--- a/docs/en/sql-reference/statements/system.md
+++ b/docs/en/sql-reference/statements/system.md
@@ -169,7 +169,7 @@ SYSTEM START MERGES [ON VOLUME | [db.]merge_tree_family_table_name
### STOP TTL MERGES {#query_language-stop-ttl-merges}
Provides possibility to stop background delete old data according to [TTL expression](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-ttl) for tables in the MergeTree family:
-Return `Ok.` even table doesn’t exists or table have not MergeTree engine. Return error when database doesn’t exists:
+Returns `Ok.` even if table doesn’t exist or table has not MergeTree engine. Returns error when database doesn’t exist:
``` sql
SYSTEM STOP TTL MERGES [[db.]merge_tree_family_table_name]
@@ -178,7 +178,7 @@ SYSTEM STOP TTL MERGES [[db.]merge_tree_family_table_name]
### START TTL MERGES {#query_language-start-ttl-merges}
Provides possibility to start background delete old data according to [TTL expression](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-ttl) for tables in the MergeTree family:
-Return `Ok.` even table doesn’t exists. Return error when database doesn’t exists:
+Returns `Ok.` even if table doesn’t exist. Returns error when database doesn’t exist:
``` sql
SYSTEM START TTL MERGES [[db.]merge_tree_family_table_name]
@@ -187,7 +187,7 @@ SYSTEM START TTL MERGES [[db.]merge_tree_family_table_name]
### STOP MOVES {#query_language-stop-moves}
Provides possibility to stop background move data according to [TTL table expression with TO VOLUME or TO DISK clause](../../engines/table-engines/mergetree-family/mergetree.md#mergetree-table-ttl) for tables in the MergeTree family:
-Return `Ok.` even table doesn’t exists. Return error when database doesn’t exists:
+Returns `Ok.` even if table doesn’t exist. Returns error when database doesn’t exist:
``` sql
SYSTEM STOP MOVES [[db.]merge_tree_family_table_name]
@@ -196,7 +196,7 @@ SYSTEM STOP MOVES [[db.]merge_tree_family_table_name]
### START MOVES {#query_language-start-moves}
Provides possibility to start background move data according to [TTL table expression with TO VOLUME and TO DISK clause](../../engines/table-engines/mergetree-family/mergetree.md#mergetree-table-ttl) for tables in the MergeTree family:
-Return `Ok.` even table doesn’t exists. Return error when database doesn’t exists:
+Returns `Ok.` even if table doesn’t exist. Returns error when database doesn’t exist:
``` sql
SYSTEM STOP MOVES [[db.]merge_tree_family_table_name]
@@ -209,7 +209,7 @@ ClickHouse can manage background replication related processes in [ReplicatedMer
### STOP FETCHES {#query_language-system-stop-fetches}
Provides possibility to stop background fetches for inserted parts for tables in the `ReplicatedMergeTree` family:
-Always returns `Ok.` regardless of the table engine and even table or database doesn’t exists.
+Always returns `Ok.` regardless of the table engine and even if table or database doesn’t exist.
``` sql
SYSTEM STOP FETCHES [[db.]replicated_merge_tree_family_table_name]
@@ -218,7 +218,7 @@ SYSTEM STOP FETCHES [[db.]replicated_merge_tree_family_table_name]
### START FETCHES {#query_language-system-start-fetches}
Provides possibility to start background fetches for inserted parts for tables in the `ReplicatedMergeTree` family:
-Always returns `Ok.` regardless of the table engine and even table or database doesn’t exists.
+Always returns `Ok.` regardless of the table engine and even if table or database doesn’t exist.
``` sql
SYSTEM START FETCHES [[db.]replicated_merge_tree_family_table_name]
@@ -264,9 +264,7 @@ Wait until a `ReplicatedMergeTree` table will be synced with other replicas in a
SYSTEM SYNC REPLICA [db.]replicated_merge_tree_family_table_name
```
-After running this statement the `[db.]replicated_merge_tree_family_table_name` fetches commands from
-the common replicated log into its own replication queue, and then the query waits till the replica processes all
-of the fetched commands.
+After running this statement the `[db.]replicated_merge_tree_family_table_name` fetches commands from the common replicated log into its own replication queue, and then the query waits till the replica processes all of the fetched commands.
### RESTART REPLICA {#query_language-system-restart-replica}
@@ -280,4 +278,3 @@ SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name
### RESTART REPLICAS {#query_language-system-restart-replicas}
Provides possibility to reinitialize Zookeeper sessions state for all `ReplicatedMergeTree` tables, will compare current state with Zookeeper as source of true and add tasks to Zookeeper queue if needed
-
diff --git a/docs/en/sql-reference/table-functions/postgresql.md b/docs/en/sql-reference/table-functions/postgresql.md
index bfb5fdf9be6..3eab572ac12 100644
--- a/docs/en/sql-reference/table-functions/postgresql.md
+++ b/docs/en/sql-reference/table-functions/postgresql.md
@@ -65,9 +65,9 @@ postgres=# INSERT INTO test (int_id, str, "float") VALUES (1,'test',2);
INSERT 0 1
postgresql> SELECT * FROM test;
- int_id | int_nullable | float | str | float_nullable
---------+--------------+-------+------+----------------
- 1 | | 2 | test |
+ int_id | int_nullable | float | str | float_nullable
+ --------+--------------+-------+------+----------------
+ 1 | | 2 | test |
(1 row)
```
diff --git a/docs/en/sql-reference/table-functions/s3.md b/docs/en/sql-reference/table-functions/s3.md
index 34f0607b94c..285ec862aab 100644
--- a/docs/en/sql-reference/table-functions/s3.md
+++ b/docs/en/sql-reference/table-functions/s3.md
@@ -18,7 +18,7 @@ s3(path, [aws_access_key_id, aws_secret_access_key,] format, structure, [compres
- `path` — Bucket url with path to file. Supports following wildcards in readonly mode: `*`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc'`, `'def'` — strings. For more information see [here](../../engines/table-engines/integrations/s3.md#wildcards-in-path).
- `format` — The [format](../../interfaces/formats.md#formats) of the file.
- `structure` — Structure of the table. Format `'column1_name column1_type, column2_name column2_type, ...'`.
-- `compression` — Parameter is optional. Supported values: none, gzip/gz, brotli/br, xz/LZMA, zstd/zst. By default, it will autodetect compression by file extension.
+- `compression` — Parameter is optional. Supported values: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. By default, it will autodetect compression by file extension.
**Returned value**
diff --git a/docs/ja/development/build.md b/docs/ja/development/build.md
index e44ba45485e..191fa665ccd 100644
--- a/docs/ja/development/build.md
+++ b/docs/ja/development/build.md
@@ -19,28 +19,17 @@ $ sudo apt-get install git cmake python ninja-build
古いシステムではcmakeの代わりにcmake3。
-## GCC9のインストール {#install-gcc-10}
+## Clang 11 のインストール
-これを行うにはいくつかの方法があります。
+On Ubuntu/Debian you can use the automatic installation script (check [official webpage](https://apt.llvm.org/))
-### PPAパッケージからインストール {#install-from-a-ppa-package}
-
-``` bash
-$ sudo apt-get install software-properties-common
-$ sudo apt-add-repository ppa:ubuntu-toolchain-r/test
-$ sudo apt-get update
-$ sudo apt-get install gcc-10 g++-10
+```bash
+sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
```
-### ソースからインスト {#install-from-sources}
-
-見て [utils/ci/build-gcc-from-sources.sh](https://github.com/ClickHouse/ClickHouse/blob/master/utils/ci/build-gcc-from-sources.sh)
-
-## ビルドにGCC9を使用する {#use-gcc-10-for-builds}
-
``` bash
-$ export CC=gcc-10
-$ export CXX=g++-10
+$ export CC=clang
+$ export CXX=clang++
```
## ツつィツ姪"ツ債ツつケ {#checkout-clickhouse-sources}
@@ -76,7 +65,7 @@ $ cd ..
- Git(ソースをチェックアウトするためにのみ使用され、ビルドには必要ありません)
- CMake3.10以降
- 忍者(推奨)または作る
-- C++コンパイラ:gcc9またはclang8以降
+- C++コンパイラ:clang11以降
- リンカ:lldまたはgold(古典的なGNU ldは動作しません)
- Python(LLVMビルド内でのみ使用され、オプションです)
diff --git a/docs/ja/development/developer-instruction.md b/docs/ja/development/developer-instruction.md
index ccc3a177d1f..d7e5217b3b6 100644
--- a/docs/ja/development/developer-instruction.md
+++ b/docs/ja/development/developer-instruction.md
@@ -133,19 +133,19 @@ ArchまたはGentooを使用する場合は、おそらくCMakeのインスト
ClickHouseはビルドに複数の外部ライブラリを使用します。 それらのすべては、サブモジュールにあるソースからClickHouseと一緒に構築されているので、別々にインストールする必要はありません。 リストは次の場所で確認できます `contrib`.
-# C++コンパイラ {#c-compiler}
+## C++ Compiler {#c-compiler}
-ClickHouseのビルドには、バージョン9以降のGCCとClangバージョン8以降のコンパイラがサポートされます。
+Compilers Clang starting from version 11 is supported for building ClickHouse.
-公式のYandexビルドは、わずかに優れたパフォーマンスのマシンコードを生成するため、GCCを使用しています(私たちのベンチマークに応じて最大数パーセントの そしてClangは開発のために通常より便利です。 が、当社の継続的インテグレーション(CI)プラットフォームを運チェックのための十数の組み合わせとなります。
+Clang should be used instead of gcc. Though, our continuous integration (CI) platform runs checks for about a dozen of build combinations.
-UBUNTUにGCCをインストールするには: `sudo apt install gcc g++`
+On Ubuntu/Debian you can use the automatic installation script (check [official webpage](https://apt.llvm.org/))
-Gccのバージョンを確認する: `gcc --version`. の場合は下記9その指示に従う。https://clickhouse.tech/docs/ja/development/build/#install-gcc-10.
+```bash
+sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
+```
-Mac OS XのビルドはClangでのみサポートされています。 ちょうど実行 `brew install llvm`
-
-Clangを使用する場合は、次のものもインストールできます `libc++` と `lld` あなたがそれが何であるか知っていれば。 を使用して `ccache` また、推奨されます。
+Mac OS X build is also supported. Just run `brew install llvm`
# 建築プロセス {#the-building-process}
@@ -158,13 +158,6 @@ ClickHouseを構築する準備ができたので、別のディレクトリを
中の間 `build` cmakeを実行してビルドを構成します。 最初の実行の前に、コンパイラ(この例ではバージョン9gccコンパイラ)を指定する環境変数を定義する必要があります。
-Linux:
-
- export CC=gcc-10 CXX=g++-10
- cmake ..
-
-Mac OS X:
-
export CC=clang CXX=clang++
cmake ..
diff --git a/docs/ja/sql-reference/aggregate-functions/reference.md b/docs/ja/sql-reference/aggregate-functions/reference.md
index 465f36179da..c66e9b54746 100644
--- a/docs/ja/sql-reference/aggregate-functions/reference.md
+++ b/docs/ja/sql-reference/aggregate-functions/reference.md
@@ -624,7 +624,7 @@ uniqHLL12(x[, ...])
- HyperLogLogアルゴリズムを使用して、異なる引数値の数を近似します。
- 212 5-bit cells are used. The size of the state is slightly more than 2.5 KB. The result is not very accurate (up to ~10% error) for small data sets (<10K elements). However, the result is fairly accurate for high-cardinality data sets (10K-100M), with a maximum error of ~1.6%. Starting from 100M, the estimation error increases, and the function will return very inaccurate results for data sets with extremely high cardinality (1B+ elements).
+ 2^12 5-bit cells are used. The size of the state is slightly more than 2.5 KB. The result is not very accurate (up to ~10% error) for small data sets (<10K elements). However, the result is fairly accurate for high-cardinality data sets (10K-100M), with a maximum error of ~1.6%. Starting from 100M, the estimation error increases, and the function will return very inaccurate results for data sets with extremely high cardinality (1B+ elements).
- 決定的な結果を提供します(クエリ処理順序に依存しません)。
diff --git a/docs/ja/sql-reference/functions/bitmap-functions.md b/docs/ja/sql-reference/functions/bitmap-functions.md
index cc57e762610..de3ce938444 100644
--- a/docs/ja/sql-reference/functions/bitmap-functions.md
+++ b/docs/ja/sql-reference/functions/bitmap-functions.md
@@ -35,7 +35,7 @@ SELECT bitmapBuild([1, 2, 3, 4, 5]) AS res, toTypeName(res)
``` text
┌─res─┬─toTypeName(bitmapBuild([1, 2, 3, 4, 5]))─────┐
-│ │ AggregateFunction(groupBitmap, UInt8) │
+│ │ AggregateFunction(groupBitmap, UInt8) │
└─────┴──────────────────────────────────────────────┘
```
diff --git a/docs/ja/sql-reference/functions/hash-functions.md b/docs/ja/sql-reference/functions/hash-functions.md
index d48e6846bb4..a98ae60690d 100644
--- a/docs/ja/sql-reference/functions/hash-functions.md
+++ b/docs/ja/sql-reference/functions/hash-functions.md
@@ -434,13 +434,13 @@ A [FixedString(16)](../../sql-reference/data-types/fixedstring.md) データ型
**例**
``` sql
-SELECT murmurHash3_128('example_string') AS MurmurHash3, toTypeName(MurmurHash3) AS type
+SELECT hex(murmurHash3_128('example_string')) AS MurmurHash3, toTypeName(MurmurHash3) AS type;
```
``` text
-┌─MurmurHash3──────┬─type────────────┐
-│ 6�1�4"S5KT�~~q │ FixedString(16) │
-└──────────────────┴─────────────────┘
+┌─MurmurHash3──────────────────────┬─type───┐
+│ 368A1A311CB7342253354B548E7E7E71 │ String │
+└──────────────────────────────────┴────────┘
```
## xxHash32,xxHash64 {#hash-functions-xxhash32}
diff --git a/docs/ru/commercial/cloud.md b/docs/ru/commercial/cloud.md
index 610f0f00a99..e00fc3be673 100644
--- a/docs/ru/commercial/cloud.md
+++ b/docs/ru/commercial/cloud.md
@@ -29,3 +29,30 @@ toc_title: "Поставщики облачных услуг ClickHouse"
- cross-az масштабирование для повышения производительности и обеспечения высокой доступности
- встроенный мониторинг и редактор SQL-запросов
+## Alibaba Cloud {#alibaba-cloud}
+
+Управляемый облачный сервис Alibaba для ClickHouse: [китайская площадка](https://www.aliyun.com/product/clickhouse), будет доступен на международной площадке в мае 2021 года. Сервис предоставляет следующие возможности:
+
+- надежный сервер для облачного хранилища на основе распределенной системы [Alibaba Cloud Apsara](https://www.alibabacloud.com/product/apsara-stack);
+- расширяемая по запросу емкость, без переноса данных вручную;
+- поддержка одноузловой и многоузловой архитектуры, архитектуры с одной или несколькими репликами, а также многоуровневого хранения cold и hot data;
+- поддержка прав доступа, one-key восстановления, многоуровневая защита сети, шифрование облачного диска;
+- полная интеграция с облачными системами логирования, базами данных и инструментами обработки данных;
+- встроенная платформа для мониторинга и управления базами данных;
+- техническая поддержка от экспертов по работе с базами данных.
+
+## SberCloud {#sbercloud}
+
+[Облачная платформа SberCloud.Advanced](https://sbercloud.ru/ru/advanced):
+
+- предоставляет более 50 высокотехнологичных сервисов;
+- позволяет быстро создавать и эффективно управлять ИТ-инфраструктурой, приложениями и интернет-сервисами;
+- радикально минимизирует ресурсы, требуемые для работы корпоративных ИТ-систем;
+- в разы сокращает время вывода новых продуктов на рынок.
+
+SberCloud.Advanced предоставляет [MapReduce Service (MRS)](https://docs.sbercloud.ru/mrs/ug/topics/ug__clickhouse.html) — надежную, безопасную и простую в использовании платформу корпоративного уровня для хранения, обработки и анализа больших данных. MRS позволяет быстро создавать и управлять кластерами ClickHouse.
+
+- Инстанс ClickHouse состоит из трех узлов ZooKeeper и нескольких узлов ClickHouse. Выделенный режим реплики используется для обеспечения высокой надежности двойных копий данных.
+- MRS предлагает возможности гибкого масштабирования при быстром росте сервисов в сценариях, когда емкости кластерного хранилища или вычислительных ресурсов процессора недостаточно. MRS в один клик предоставляет инструмент для балансировки данных при расширении узлов ClickHouse в кластере. Вы можете определить режим и время балансировки данных на основе характеристик сервиса, чтобы обеспечить доступность сервиса.
+- MRS использует архитектуру развертывания высокой доступности на основе Elastic Load Balance (ELB) — сервиса для автоматического распределения трафика на несколько внутренних узлов. Благодаря ELB, данные записываются в локальные таблицы и считываются из распределенных таблиц на разных узлах. Такая архитектура повышает отказоустойчивость кластера и гарантирует высокую доступность приложений.
+
diff --git a/docs/ru/development/architecture.md b/docs/ru/development/architecture.md
index 9f43fabba4f..d2cfc44b711 100644
--- a/docs/ru/development/architecture.md
+++ b/docs/ru/development/architecture.md
@@ -27,7 +27,7 @@ ClickHouse - полноценная колоночная СУБД. Данные
`IColumn` предоставляет методы для общих реляционных преобразований данных, но они не отвечают всем потребностям. Например, `ColumnUInt64` не имеет метода для вычисления суммы двух столбцов, а `ColumnString` не имеет метода для запуска поиска по подстроке. Эти бесчисленные процедуры реализованы вне `IColumn`.
-Различные функции на колонках могут быть реализованы обобщенным, неэффективным путем, используя `IColumn` методы для извлечения значений `Field`, или специальным путем, используя знания о внутреннем распределение данных в памяти в конкретной реализации `IColumn`. Для этого функции приводятся к конкретному типу `IColumn` и работают напрямую с его внутренним представлением. Например, в `ColumnUInt64` есть метод getData, который возвращает ссылку на внутренний массив, чтение и заполнение которого, выполняется отдельной процедурой напрямую. Фактически, мы имеем "дырявую абстракции", обеспечивающие эффективные специализации различных процедур.
+Различные функции на колонках могут быть реализованы обобщенным, неэффективным путем, используя `IColumn` методы для извлечения значений `Field`, или специальным путем, используя знания о внутреннем распределение данных в памяти в конкретной реализации `IColumn`. Для этого функции приводятся к конкретному типу `IColumn` и работают напрямую с его внутренним представлением. Например, в `ColumnUInt64` есть метод `getData`, который возвращает ссылку на внутренний массив, чтение и заполнение которого, выполняется отдельной процедурой напрямую. Фактически, мы имеем "дырявые абстракции", обеспечивающие эффективные специализации различных процедур.
## Типы данных (Data Types) {#data_types}
@@ -42,7 +42,7 @@ ClickHouse - полноценная колоночная СУБД. Данные
## Блоки (Block) {#block}
-`Block` это контейнер, который представляет фрагмент (chunk) таблицы в памяти. Это набор троек - `(IColumn, IDataType, имя колонки)`. В процессе выполнения запроса, данные обрабатываются `Block`ами. Если у нас есть `Block`, значит у нас есть данные (в объекте `IColumn`), информация о типе (в `IDataType`), которая говорит нам, как работать с колонкой, и имя колонки (оригинальное имя колонки таблицы или служебное имя, присвоенное для получения промежуточных результатов вычислений).
+`Block` это контейнер, который представляет фрагмент (chunk) таблицы в памяти. Это набор троек - `(IColumn, IDataType, имя колонки)`. В процессе выполнения запроса, данные обрабатываются `Block`-ами. Если у нас есть `Block`, значит у нас есть данные (в объекте `IColumn`), информация о типе (в `IDataType`), которая говорит нам, как работать с колонкой, и имя колонки (оригинальное имя колонки таблицы или служебное имя, присвоенное для получения промежуточных результатов вычислений).
При вычислении некоторой функции на колонках в блоке мы добавляем еще одну колонку с результатами в блок, не трогая колонки аргументов функции, потому что операции иммутабельные. Позже ненужные колонки могут быть удалены из блока, но не модифицированы. Это удобно для устранения общих подвыражений.
@@ -58,7 +58,7 @@ ClickHouse - полноценная колоночная СУБД. Данные
2. Реализацию форматов данных. Например, при выводе данных в терминал в формате `Pretty`, вы создаете выходной поток блоков, который форматирует поступающие в него блоки.
3. Трансформацию данных. Допустим, у вас есть `IBlockInputStream` и вы хотите создать отфильтрованный поток. Вы создаете `FilterBlockInputStream` и инициализируете его вашим потоком. Затем вы тянете (pull) блоки из `FilterBlockInputStream`, а он тянет блоки исходного потока, фильтрует их и возвращает отфильтрованные блоки вам. Таким образом построены конвейеры выполнения запросов.
-Имеются и более сложные трансформации. Например, когда вы тянете блоки из `AggregatingBlockInputStream`, он считывает все данные из своего источника, агрегирует их, и возвращает поток агрегированных данных вам. Другой пример: конструктор `UnionBlockInputStream` принимает множество источников входных данных и число потоков. Такой `Stream` работает в несколько потоков и читает данные источников параллельно.
+Имеются и более сложные трансформации. Например, когда вы тянете блоки из `AggregatingBlockInputStream`, он считывает все данные из своего источника, агрегирует их, и возвращает поток агрегированных данных вам. Другой пример: конструктор `UnionBlockInputStream` принимает множество источников входных данных и число потоков. Такой `Stream` работает в несколько потоков и читает данные источников параллельно.
> Потоки блоков используют «втягивающий» (pull) подход к управлению потоком выполнения: когда вы вытягиваете блок из первого потока, он, следовательно, вытягивает необходимые блоки из вложенных потоков, так и работает весь конвейер выполнения. Ни «pull» ни «push» не имеют явного преимущества, потому что поток управления неявный, и это ограничивает в реализации различных функций, таких как одновременное выполнение нескольких запросов (слияние нескольких конвейеров вместе). Это ограничение можно преодолеть с помощью сопрограмм (coroutines) или просто запуском дополнительных потоков, которые ждут друг друга. У нас может быть больше возможностей, если мы сделаем поток управления явным: если мы локализуем логику для передачи данных из одной расчетной единицы в другую вне этих расчетных единиц. Читайте эту [статью](http://journal.stuffwithstuff.com/2013/01/13/iteration-inside-and-out/) для углубленного изучения.
@@ -110,9 +110,9 @@ ClickHouse - полноценная колоночная СУБД. Данные
> Генераторы парсеров не используются по историческим причинам.
## Интерпретаторы {#interpreters}
-
+
Интерпретаторы отвечают за создание конвейера выполнения запроса из `AST`. Есть простые интерпретаторы, такие как `InterpreterExistsQuery` и `InterpreterDropQuery` или более сложный `InterpreterSelectQuery`. Конвейер выполнения запроса представляет собой комбинацию входных и выходных потоков блоков. Например, результатом интерпретации `SELECT` запроса является `IBlockInputStream` для чтения результирующего набора данных; результат интерпретации `INSERT` запроса - это `IBlockOutputStream`, для записи данных, предназначенных для вставки; результат интерпретации `INSERT SELECT` запроса - это `IBlockInputStream`, который возвращает пустой результирующий набор при первом чтении, но копирует данные из `SELECT` к `INSERT`.
-
+
`InterpreterSelectQuery` использует `ExpressionAnalyzer` и `ExpressionActions` механизмы для анализа запросов и преобразований. Именно здесь выполняется большинство оптимизаций запросов на основе правил. `ExpressionAnalyzer` написан довольно грязно и должен быть переписан: различные преобразования запросов и оптимизации должны быть извлечены в отдельные классы, чтобы позволить модульные преобразования или запросы.
## Функции {#functions}
@@ -162,9 +162,9 @@ ClickHouse имеет сильную типизацию, поэтому нет
Сервера в кластере в основном независимы. Вы можете создать `Распределенную` (`Distributed`) таблицу на одном или всех серверах в кластере. Такая таблица сама по себе не хранит данные - она только предоставляет возможность "просмотра" всех локальных таблиц на нескольких узлах кластера. При выполнении `SELECT` распределенная таблица переписывает запрос, выбирает удаленные узлы в соответствии с настройками балансировки нагрузки и отправляет им запрос. Распределенная таблица просит удаленные сервера обработать запрос до той стадии, когда промежуточные результаты с разных серверов могут быть объединены. Затем он получает промежуточные результаты и объединяет их. Распределенная таблица пытается возложить как можно больше работы на удаленные серверы и сократить объем промежуточных данных, передаваемых по сети.
-Ситуация усложняется, при использовании подзапросы в случае IN или JOIN, когда каждый из них использует таблицу `Distributed`. Есть разные стратегии для выполнения таких запросов.
+Ситуация усложняется, при использовании подзапросов в случае `IN` или `JOIN`, когда каждый из них использует таблицу `Distributed`. Есть разные стратегии для выполнения таких запросов.
-Глобального плана выполнения распределенных запросов не существует. Каждый узел имеет собственный локальный план для своей части работы. У нас есть простое однонаправленное выполнение распределенных запросов: мы отправляем запросы на удаленные узлы и затем объединяем результаты. Но это невозможно для сложных запросов GROUP BY высокой кардинальности или запросов с большим числом временных данных в JOIN: в таких случаях нам необходимо перераспределить («reshuffle») данные между серверами, что требует дополнительной координации. ClickHouse не поддерживает выполнение запросов такого рода, и нам нужно работать над этим.
+Глобального плана выполнения распределенных запросов не существует. Каждый узел имеет собственный локальный план для своей части работы. У нас есть простое однонаправленное выполнение распределенных запросов: мы отправляем запросы на удаленные узлы и затем объединяем результаты. Но это невозможно для сложных запросов `GROUP BY` высокой кардинальности или запросов с большим числом временных данных в `JOIN`: в таких случаях нам необходимо перераспределить («reshuffle») данные между серверами, что требует дополнительной координации. ClickHouse не поддерживает выполнение запросов такого рода, и нам нужно работать над этим.
## Merge Tree {#merge-tree}
@@ -190,7 +190,7 @@ ClickHouse имеет сильную типизацию, поэтому нет
Репликация использует асинхронную multi-master схему. Вы можете вставить данные в любую реплику, которая имеет открытую сессию в `ZooKeeper`, и данные реплицируются на все другие реплики асинхронно. Поскольку ClickHouse не поддерживает UPDATE, репликация исключает конфликты (conflict-free replication). Поскольку подтверждение вставок кворумом не реализовано, только что вставленные данные могут быть потеряны в случае сбоя одного узла.
-Метаданные для репликации хранятся в `ZooKeeper`. Существует журнал репликации, в котором перечислены действия, которые необходимо выполнить. Среди этих действий: получить часть (get the part); объединить части (merge parts); удалить партицию (drop a partition) и так далее. Каждая реплика копирует журнал репликации в свою очередь, а затем выполняет действия из очереди. Например, при вставке в журнале создается действие «получить часть» (get the part), и каждая реплика загружает эту часть. Слияния координируются между репликами, чтобы получить идентичные до байта результаты. Все части объединяются одинаково на всех репликах. Одна из реплик-лидеров инициирует новое слияние кусков первой и записывает действия «слияния частей» в журнал. Несколько реплик (или все) могут быть лидерами одновременно. Реплике можно запретить быть лидером с помощью `merge_tree` настройки `replicated_can_become_leader`.
+Метаданные для репликации хранятся в `ZooKeeper`. Существует журнал репликации, в котором перечислены действия, которые необходимо выполнить. Среди этих действий: получить часть (get the part); объединить части (merge parts); удалить партицию (drop a partition) и так далее. Каждая реплика копирует журнал репликации в свою очередь, а затем выполняет действия из очереди. Например, при вставке в журнале создается действие «получить часть» (get the part), и каждая реплика загружает эту часть. Слияния координируются между репликами, чтобы получить идентичные до байта результаты. Все части объединяются одинаково на всех репликах. Одна из реплик-лидеров инициирует новое слияние кусков первой и записывает действия «слияния частей» в журнал. Несколько реплик (или все) могут быть лидерами одновременно. Реплике можно запретить быть лидером с помощью `merge_tree` настройки `replicated_can_become_leader`.
Репликация является физической: между узлами передаются только сжатые части, а не запросы. Слияния обрабатываются на каждой реплике независимо, в большинстве случаев, чтобы снизить затраты на сеть, во избежание усиления роли сети. Крупные объединенные части отправляются по сети только в случае значительной задержки репликации.
diff --git a/docs/ru/development/developer-instruction.md b/docs/ru/development/developer-instruction.md
index 9ddb17b7212..463d38a44fb 100644
--- a/docs/ru/development/developer-instruction.md
+++ b/docs/ru/development/developer-instruction.md
@@ -7,15 +7,15 @@ toc_title: "Инструкция для разработчиков"
Сборка ClickHouse поддерживается на Linux, FreeBSD, Mac OS X.
-# Если вы используете Windows {#esli-vy-ispolzuete-windows}
+## Если вы используете Windows {#esli-vy-ispolzuete-windows}
Если вы используете Windows, вам потребуется создать виртуальную машину с Ubuntu. Для работы с виртуальной машиной, установите VirtualBox. Скачать Ubuntu можно на сайте: https://www.ubuntu.com/#download Создайте виртуальную машину из полученного образа. Выделите для неё не менее 4 GB оперативной памяти. Для запуска терминала в Ubuntu, найдите в меню программу со словом terminal (gnome-terminal, konsole или что-то в этом роде) или нажмите Ctrl+Alt+T.
-# Если вы используете 32-битную систему {#esli-vy-ispolzuete-32-bitnuiu-sistemu}
+## Если вы используете 32-битную систему {#esli-vy-ispolzuete-32-bitnuiu-sistemu}
ClickHouse не работает и не собирается на 32-битных системах. Получите доступ к 64-битной системе и продолжайте.
-# Создание репозитория на GitHub {#sozdanie-repozitoriia-na-github}
+## Создание репозитория на GitHub {#sozdanie-repozitoriia-na-github}
Для работы с репозиторием ClickHouse, вам потребуется аккаунт на GitHub. Наверное, он у вас уже есть.
@@ -34,7 +34,7 @@ ClickHouse не работает и не собирается на 32-битны
Подробное руководство по использованию Git: https://git-scm.com/book/ru/v2
-# Клонирование репозитория на рабочую машину {#klonirovanie-repozitoriia-na-rabochuiu-mashinu}
+## Клонирование репозитория на рабочую машину {#klonirovanie-repozitoriia-na-rabochuiu-mashinu}
Затем вам потребуется загрузить исходники для работы на свой компьютер. Это называется «клонирование репозитория», потому что создаёт на вашем компьютере локальную копию репозитория, с которой вы будете работать.
@@ -78,7 +78,7 @@ ClickHouse не работает и не собирается на 32-битны
После этого, вы сможете добавлять в свой репозиторий обновления из репозитория Яндекса с помощью команды `git pull upstream master`.
-## Работа с сабмодулями Git {#rabota-s-sabmoduliami-git}
+### Работа с сабмодулями Git {#rabota-s-sabmoduliami-git}
Работа с сабмодулями git может быть достаточно болезненной. Следующие команды позволят содержать их в порядке:
@@ -110,7 +110,7 @@ The next commands would help you to reset all submodules to the initial state (!
git submodule foreach git submodule foreach git reset --hard
git submodule foreach git submodule foreach git clean -xfd
-# Система сборки {#sistema-sborki}
+## Система сборки {#sistema-sborki}
ClickHouse использует систему сборки CMake и Ninja.
@@ -130,25 +130,25 @@ Ninja - система запуска сборочных задач.
Проверьте версию CMake: `cmake --version`. Если версия меньше 3.3, то установите новую версию с сайта https://cmake.org/download/
-# Необязательные внешние библиотеки {#neobiazatelnye-vneshnie-biblioteki}
+## Необязательные внешние библиотеки {#neobiazatelnye-vneshnie-biblioteki}
ClickHouse использует для сборки некоторое количество внешних библиотек. Но ни одну из них не требуется отдельно устанавливать, так как они собираются вместе с ClickHouse, из исходников, которые расположены в submodules. Посмотреть набор этих библиотек можно в директории contrib.
-# Компилятор C++ {#kompiliator-c}
+## Компилятор C++ {#kompiliator-c}
-В качестве компилятора C++ поддерживается GCC начиная с версии 9 или Clang начиная с версии 8.
+В качестве компилятора C++ поддерживается Clang начиная с версии 11.
-Официальные сборки от Яндекса, на данный момент, используют GCC, так как он генерирует слегка более производительный машинный код (разница в среднем до нескольких процентов по нашим бенчмаркам). Clang обычно более удобен для разработки. Впрочем, наша среда continuous integration проверяет около десятка вариантов сборки.
+Впрочем, наша среда continuous integration проверяет около десятка вариантов сборки, включая gcc, но сборка с помощью gcc непригодна для использования в продакшене.
-Для установки GCC под Ubuntu, выполните: `sudo apt install gcc g++`.
+On Ubuntu/Debian you can use the automatic installation script (check [official webpage](https://apt.llvm.org/))
-Проверьте версию gcc: `gcc --version`. Если версия меньше 10, то следуйте инструкции: https://clickhouse.tech/docs/ru/development/build/#install-gcc-10.
+```bash
+sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
+```
Сборка под Mac OS X поддерживается только для компилятора Clang. Чтобы установить его выполните `brew install llvm`
-Если вы решили использовать Clang, вы также можете установить `libc++` и `lld`, если вы знаете, что это такое. При желании, установите `ccache`.
-
-# Процесс сборки {#protsess-sborki}
+## Процесс сборки {#protsess-sborki}
Теперь вы готовы к сборке ClickHouse. Для размещения собранных файлов, рекомендуется создать отдельную директорию build внутри директории ClickHouse:
@@ -158,14 +158,7 @@ ClickHouse использует для сборки некоторое коли
Вы можете иметь несколько разных директорий (build_release, build_debug) для разных вариантов сборки.
Находясь в директории build, выполните конфигурацию сборки с помощью CMake.
-Перед первым запуском необходимо выставить переменные окружения, отвечающие за выбор компилятора (в данном примере это - gcc версии 9).
-
-Linux:
-
- export CC=gcc-10 CXX=g++-10
- cmake ..
-
-Mac OS X:
+Перед первым запуском необходимо выставить переменные окружения, отвечающие за выбор компилятора.
export CC=clang CXX=clang++
cmake ..
@@ -206,7 +199,7 @@ Mac OS X:
ls -l programs/clickhouse
-# Запуск собранной версии ClickHouse {#zapusk-sobrannoi-versii-clickhouse}
+## Запуск собранной версии ClickHouse {#zapusk-sobrannoi-versii-clickhouse}
Для запуска сервера из под текущего пользователя, с выводом логов в терминал и с использованием примеров конфигурационных файлов, расположенных в исходниках, перейдите в директорию `ClickHouse/programs/server/` (эта директория находится не в директории build) и выполните:
@@ -233,7 +226,7 @@ Mac OS X:
sudo service clickhouse-server stop
sudo -u clickhouse ClickHouse/build/programs/clickhouse server --config-file /etc/clickhouse-server/config.xml
-# Среда разработки {#sreda-razrabotki}
+## Среда разработки {#sreda-razrabotki}
Если вы не знаете, какую среду разработки использовать, то рекомендуется использовать CLion. CLion является платным ПО, но его можно использовать бесплатно в течение пробного периода. Также он бесплатен для учащихся. CLion можно использовать как под Linux, так и под Mac OS X.
@@ -243,7 +236,7 @@ Mac OS X:
На всякий случай заметим, что CLion самостоятельно создаёт свою build директорию, самостоятельно выбирает тип сборки debug по-умолчанию, для конфигурации использует встроенную в CLion версию CMake вместо установленного вами, а для запуска задач использует make вместо ninja. Это нормально, просто имейте это ввиду, чтобы не возникало путаницы.
-# Написание кода {#napisanie-koda}
+## Написание кода {#napisanie-koda}
Описание архитектуры ClickHouse: https://clickhouse.tech/docs/ru/development/architecture/
@@ -253,7 +246,7 @@ Mac OS X:
Список задач: https://github.com/ClickHouse/ClickHouse/issues?q=is%3Aopen+is%3Aissue+label%3A%22easy+task%22
-# Тестовые данные {#testovye-dannye}
+## Тестовые данные {#testovye-dannye}
Разработка ClickHouse часто требует загрузки реалистичных наборов данных. Особенно это важно для тестирования производительности. Специально для вас мы подготовили набор данных, представляющий собой анонимизированные данные Яндекс.Метрики. Загрузка этих данных потребует ещё 3 GB места на диске. Для выполнения большинства задач разработки, загружать эти данные не обязательно.
@@ -274,7 +267,7 @@ Mac OS X:
clickhouse-client --max_insert_block_size 100000 --query "INSERT INTO test.hits FORMAT TSV" < hits_v1.tsv
clickhouse-client --max_insert_block_size 100000 --query "INSERT INTO test.visits FORMAT TSV" < visits_v1.tsv
-# Создание Pull Request {#sozdanie-pull-request}
+## Создание Pull Request {#sozdanie-pull-request}
Откройте свой форк репозитория в интерфейсе GitHub. Если вы вели разработку в бранче, выберите этот бранч. На странице будет доступна кнопка «Pull request». По сути, это означает «создать заявку на принятие моих изменений в основной репозиторий».
diff --git a/docs/ru/development/style.md b/docs/ru/development/style.md
index f08ecc3c4c7..de29e629ceb 100644
--- a/docs/ru/development/style.md
+++ b/docs/ru/development/style.md
@@ -747,7 +747,7 @@ The dictionary is configured incorrectly.
Есть два основных варианта проверки на такие ошибки:
* Исключение с кодом `LOGICAL_ERROR`. Его можно использовать для важных проверок, которые делаются в том числе в релизной сборке.
-* `assert`. Такие условия не проверяются в релизной сборке, можно использовать для тяжёлых и опциональных проверок.
+* `assert`. Такие условия не проверяются в релизной сборке, можно использовать для тяжёлых и опциональных проверок.
Пример сообщения, у которого должен быть код `LOGICAL_ERROR`:
`Block header is inconsistent with Chunk in ICompicatedProcessor::munge(). It is a bug!`
@@ -780,7 +780,7 @@ The dictionary is configured incorrectly.
**2.** Язык - C++20 (см. список доступных [C++20 фич](https://en.cppreference.com/w/cpp/compiler_support#C.2B.2B20_features)).
-**3.** Компилятор - `gcc`. На данный момент (август 2020), код собирается версией 9.3. (Также код может быть собран `clang` версий 10 и 9)
+**3.** Компилятор - `clang`. На данный момент (апрель 2021), код собирается версией 11. (Также код может быть собран `gcc` версии 10, но такая сборка не тестируется и непригодна для продакшена).
Используется стандартная библиотека (реализация `libc++`).
diff --git a/docs/ru/engines/database-engines/atomic.md b/docs/ru/engines/database-engines/atomic.md
index a371301fd2e..8c75be3d93b 100644
--- a/docs/ru/engines/database-engines/atomic.md
+++ b/docs/ru/engines/database-engines/atomic.md
@@ -3,15 +3,52 @@ toc_priority: 32
toc_title: Atomic
---
-
# Atomic {#atomic}
-Поддерживает неблокирующие запросы `DROP` и `RENAME TABLE` и запросы `EXCHANGE TABLES t1 AND t2`. Движок `Atomic` используется по умолчанию.
+Поддерживает неблокирующие запросы [DROP TABLE](#drop-detach-table) и [RENAME TABLE](#rename-table) и атомарные запросы [EXCHANGE TABLES t1 AND t](#exchange-tables). Движок `Atomic` используется по умолчанию.
## Создание БД {#creating-a-database}
-```sql
-CREATE DATABASE test ENGINE = Atomic;
+``` sql
+ CREATE DATABASE test[ ENGINE = Atomic];
```
-[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/database-engines/atomic/)
+## Особенности и рекомендации {#specifics-and-recommendations}
+
+### UUID {#table-uuid}
+
+Каждая таблица в базе данных `Atomic` имеет уникальный [UUID](../../sql-reference/data-types/uuid.md) и хранит данные в папке `/clickhouse_path/store/xxx/xxxyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy/`, где `xxxyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy` - это UUID таблицы.
+Обычно UUID генерируется автоматически, но пользователь также может явно указать UUID в момент создания таблицы (однако это не рекомендуется). Для отображения UUID в запросе `SHOW CREATE` вы можете использовать настройку [show_table_uuid_in_table_create_query_if_not_nil](../../operations/settings/settings.md#show_table_uuid_in_table_create_query_if_not_nil). Результат выполнения в таком случае будет иметь вид:
+
+```sql
+CREATE TABLE name UUID '28f1c61c-2970-457a-bffe-454156ddcfef' (n UInt64) ENGINE = ...;
+```
+### RENAME TABLE {#rename-table}
+
+Запросы `RENAME` выполняются без изменения UUID и перемещения табличных данных. Эти запросы не ожидают завершения использующих таблицу запросов и будут выполнены мгновенно.
+
+### DROP/DETACH TABLE {#drop-detach-table}
+
+При выполнении запроса `DROP TABLE` никакие данные не удаляются. Таблица помечается как удаленная, метаданные перемещаются в папку `/clickhouse_path/metadata_dropped/` и база данных уведомляет фоновый поток. Задержка перед окончательным удалением данных задается настройкой [database_atomic_delay_before_drop_table_sec](../../operations/server-configuration-parameters/settings.md#database_atomic_delay_before_drop_table_sec).
+Вы можете задать синхронный режим, определяя модификатор `SYNC`. Используйте для этого настройку [database_atomic_wait_for_drop_and_detach_synchronously](../../operations/settings/settings.md#database_atomic_wait_for_drop_and_detach_synchronously). В этом случае запрос `DROP` ждет завершения `SELECT`, `INSERT` и других запросов, которые используют таблицу. Таблица будет фактически удалена, когда она не будет использоваться.
+
+### EXCHANGE TABLES {#exchange-tables}
+
+Запрос `EXCHANGE` меняет местами две таблицы атомарно. Вместо неатомарной операции:
+
+```sql
+RENAME TABLE new_table TO tmp, old_table TO new_table, tmp TO old_table;
+```
+вы можете использовать один атомарный запрос:
+
+``` sql
+EXCHANGE TABLES new_table AND old_table;
+```
+
+### ReplicatedMergeTree in Atomic Database {#replicatedmergetree-in-atomic-database}
+
+Для таблиц [ReplicatedMergeTree](../table-engines/mergetree-family/replication.md#table_engines-replication) рекомендуется не указывать параметры движка - путь в ZooKeeper и имя реплики. В этом случае будут использоваться параметры конфигурации: [default_replica_path](../../operations/server-configuration-parameters/settings.md#default_replica_path) и [default_replica_name](../../operations/server-configuration-parameters/settings.md#default_replica_name). Если вы хотите определить параметры движка явно, рекомендуется использовать макрос {uuid}. Это удобно, так как автоматически генерируются уникальные пути для каждой таблицы в ZooKeeper.
+
+## Смотрите также
+
+- Системная таблица [system.databases](../../operations/system-tables/databases.md).
diff --git a/docs/ru/engines/table-engines/index.md b/docs/ru/engines/table-engines/index.md
index a364a3cb972..b17b2124250 100644
--- a/docs/ru/engines/table-engines/index.md
+++ b/docs/ru/engines/table-engines/index.md
@@ -48,6 +48,14 @@ toc_title: "Введение"
Движки семейства:
+- [Kafka](integrations/kafka.md#kafka)
+- [MySQL](integrations/mysql.md#mysql)
+- [ODBC](integrations/odbc.md#table-engine-odbc)
+- [JDBC](integrations/jdbc.md#table-engine-jdbc)
+- [S3](integrations/s3.md#table-engine-s3)
+
+### Специальные движки {#spetsialnye-dvizhki}
+
- [ODBC](../../engines/table-engines/integrations/odbc.md)
- [JDBC](../../engines/table-engines/integrations/jdbc.md)
- [MySQL](../../engines/table-engines/integrations/mysql.md)
@@ -84,4 +92,3 @@ toc_title: "Введение"
Чтобы получить данные из виртуального столбца, необходимо указать его название в запросе `SELECT`. `SELECT *` не отображает данные из виртуальных столбцов.
При создании таблицы со столбцом, имя которого совпадает с именем одного из виртуальных столбцов таблицы, виртуальный столбец становится недоступным. Не делайте так. Чтобы помочь избежать конфликтов, имена виртуальных столбцов обычно предваряются подчеркиванием.
-
diff --git a/docs/ru/engines/table-engines/integrations/postgresql.md b/docs/ru/engines/table-engines/integrations/postgresql.md
index 8964b1dbf02..cb8e38ae5c9 100644
--- a/docs/ru/engines/table-engines/integrations/postgresql.md
+++ b/docs/ru/engines/table-engines/integrations/postgresql.md
@@ -22,7 +22,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
Структура таблицы может отличаться от исходной структуры таблицы PostgreSQL:
-- Имена столбцов должны быть такими же, как в исходной таблице MySQL, но вы можете использовать только некоторые из этих столбцов и в любом порядке.
+- Имена столбцов должны быть такими же, как в исходной таблице PostgreSQL, но вы можете использовать только некоторые из этих столбцов и в любом порядке.
- Типы столбцов могут отличаться от типов в исходной таблице PostgreSQL. ClickHouse пытается [приводить](../../../sql-reference/functions/type-conversion-functions.md#type_conversion_function-cast) values to the ClickHouse data types.
- Настройка `external_table_functions_use_nulls` определяет как обрабатывать Nullable столбцы. По умолчанию 1, если 0 - табличная функция не будет делать nullable столбцы и будет вместо null выставлять значения по умолчанию для скалярного типа. Это также применимо для null значений внутри массивов.
@@ -94,10 +94,10 @@ postgres=# INSERT INTO test (int_id, str, "float") VALUES (1,'test',2);
INSERT 0 1
postgresql> SELECT * FROM test;
- int_id | int_nullable | float | str | float_nullable
---------+--------------+-------+------+----------------
- 1 | | 2 | test |
-(1 row)
+ int_id | int_nullable | float | str | float_nullable
+ --------+--------------+-------+------+----------------
+ 1 | | 2 | test |
+ (1 row)
```
Таблица в ClickHouse, получение данных из PostgreSQL таблицы, созданной выше:
diff --git a/docs/ru/engines/table-engines/integrations/s3.md b/docs/ru/engines/table-engines/integrations/s3.md
index fa10e8ebc34..216db98077c 100644
--- a/docs/ru/engines/table-engines/integrations/s3.md
+++ b/docs/ru/engines/table-engines/integrations/s3.md
@@ -19,7 +19,7 @@ ENGINE = S3(path, [aws_access_key_id, aws_secret_access_key,] format, structure,
- `path` — URL-адрес бакета с указанием пути к файлу. Поддерживает следующие подстановочные знаки в режиме "только чтение": `*`, `?`, `{abc,def}` и `{N..M}` где `N`, `M` — числа, `'abc'`, `'def'` — строки. Подробнее смотри [ниже](#wildcards-in-path).
- `format` — [формат](../../../interfaces/formats.md#formats) файла.
- `structure` — структура таблицы в формате `'column1_name column1_type, column2_name column2_type, ...'`.
-- `compression` — тип сжатия. Возможные значения: none, gzip/gz, brotli/br, xz/LZMA, zstd/zst. Необязательный параметр. Если не указано, то тип сжатия определяется автоматически по расширению файла.
+- `compression` — тип сжатия. Возможные значения: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. Необязательный параметр. Если не указано, то тип сжатия определяется автоматически по расширению файла.
**Пример**
@@ -73,17 +73,17 @@ SELECT * FROM s3_engine_table LIMIT 2;
Соображение безопасности: если злонамеренный пользователь попробует указать произвольные URL-адреса S3, параметр `s3_max_redirects` должен быть установлен в ноль, чтобы избежать атак [SSRF] (https://en.wikipedia.org/wiki/Server-side_request_forgery). Как альтернатива, в конфигурации сервера должен быть указан `remote_host_filter`.
-## Настройки конечных точек {#endpoint-settings}
+## Настройки точки приема запроса {#endpoint-settings}
-Для конечной точки (которая соответствует точному префиксу URL-адреса) в конфигурационном файле могут быть заданы следующие настройки:
+Для точки приема запроса (которая соответствует точному префиксу URL-адреса) в конфигурационном файле могут быть заданы следующие настройки:
Обязательная настройка:
-- `endpoint` — указывает префикс конечной точки.
+- `endpoint` — указывает префикс точки приема запроса.
Необязательные настройки:
-- `access_key_id` и `secret_access_key` — указывают учетные данные для использования с данной конечной точкой.
-- `use_environment_credentials` — если `true`, S3-клиент будет пытаться получить учетные данные из переменных среды и метаданных Amazon EC2 для данной конечной точки. Значение по умолчанию - `false`.
-- `header` — добавляет указанный HTTP-заголовок к запросу на заданную конечную точку. Может быть определен несколько раз.
+- `access_key_id` и `secret_access_key` — указывают учетные данные для использования с данной точкой приема запроса.
+- `use_environment_credentials` — если `true`, S3-клиент будет пытаться получить учетные данные из переменных среды и метаданных Amazon EC2 для данной точки приема запроса. Значение по умолчанию - `false`.
+- `header` — добавляет указанный HTTP-заголовок к запросу на заданную точку приема запроса. Может быть определен несколько раз.
- `server_side_encryption_customer_key_base64` — устанавливает необходимые заголовки для доступа к объектам S3 с шифрованием SSE-C.
**Пример**
@@ -133,8 +133,7 @@ CREATE TABLE table_with_asterisk (name String, value UInt32)
ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/{some,another}_prefix/*', 'CSV');
```
-!!! warning "Warning"
- Если список файлов содержит диапазоны чисел с ведущими нулями, используйте конструкцию с фигурными скобками для каждой цифры отдельно или используйте `?`.
+Если список файлов содержит диапазоны чисел с ведущими нулями, используйте конструкцию с фигурными скобками для каждой цифры отдельно или используйте `?`.
4. Создание таблицы из файлов с именами `file-000.csv`, `file-001.csv`, … , `file-999.csv`:
@@ -145,6 +144,3 @@ ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/big_prefix/file-
**Смотрите также**
- [Табличная функция S3](../../../sql-reference/table-functions/s3.md)
-
-[Оригинальная статья](https://clickhouse.tech/docs/ru/engines/table-engines/integrations/s3/)
-
diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md
index 7d7641a417d..b8bd259167a 100644
--- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md
+++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md
@@ -753,7 +753,8 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd'
Необязательные параметры:
-- `use_environment_credentials` — признак, нужно ли считывать учетные данные AWS из переменных окружения `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` и `AWS_SESSION_TOKEN`, если они есть. Значение по умолчанию: `false`.
+- `use_environment_credentials` — признак, нужно ли считывать учетные данные AWS из сетевого окружения, а также из переменных окружения `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` и `AWS_SESSION_TOKEN`, если они есть. Значение по умолчанию: `false`.
+- `use_insecure_imds_request` — признак, нужно ли использовать менее безопасное соединение при выполнении запроса к IMDS при получении учётных данных из метаданных Amazon EC2. Значение по умолчанию: `false`.
- `proxy` — конфигурация прокси-сервера для конечной точки S3. Каждый элемент `uri` внутри блока `proxy` должен содержать URL прокси-сервера.
- `connect_timeout_ms` — таймаут подключения к сокету в миллисекундах. Значение по умолчанию: 10 секунд.
- `request_timeout_ms` — таймаут выполнения запроса в миллисекундах. Значение по умолчанию: 5 секунд.
diff --git a/docs/ru/getting-started/example-datasets/cell-towers.md b/docs/ru/getting-started/example-datasets/cell-towers.md
new file mode 100644
index 00000000000..a5524248019
--- /dev/null
+++ b/docs/ru/getting-started/example-datasets/cell-towers.md
@@ -0,0 +1,128 @@
+---
+toc_priority: 21
+toc_title: Вышки сотовой связи
+---
+
+# Вышки сотовой связи {#cell-towers}
+
+Источник этого набора данных (dataset) - самая большая в мире открытая база данных о сотовых вышках - [OpenCellid](https://www.opencellid.org/). К 2021-му году здесь накопилось более, чем 40 миллионов записей о сотовых вышках (GSM, LTE, UMTS, и т.д.) по всему миру с их географическими координатами и метаданными (код страны, сети, и т.д.).
+
+OpenCelliD Project имеет лицензию Creative Commons Attribution-ShareAlike 4.0 International License, и мы распространяем снэпшот набора данных по условиям этой же лицензии. После авторизации можно загрузить последнюю версию набора данных.
+
+## Как получить набор данных {#get-the-dataset}
+
+1. Загрузите снэпшот набора данных за февраль 2021 [отсюда](https://datasets.clickhouse.tech/cell_towers.csv.xz) (729 MB).
+
+2. Если нужно, проверьте полноту и целостность при помощи команды:
+
+```
+md5sum cell_towers.csv.xz
+8cf986f4a0d9f12c6f384a0e9192c908 cell_towers.csv.xz
+```
+
+3. Распакуйте набор данных при помощи команды:
+
+```
+xz -d cell_towers.csv.xz
+```
+
+4. Создайте таблицу:
+
+```
+CREATE TABLE cell_towers
+(
+ radio Enum8('' = 0, 'CDMA' = 1, 'GSM' = 2, 'LTE' = 3, 'NR' = 4, 'UMTS' = 5),
+ mcc UInt16,
+ net UInt16,
+ area UInt16,
+ cell UInt64,
+ unit Int16,
+ lon Float64,
+ lat Float64,
+ range UInt32,
+ samples UInt32,
+ changeable UInt8,
+ created DateTime,
+ updated DateTime,
+ averageSignal UInt8
+)
+ENGINE = MergeTree ORDER BY (radio, mcc, net, created);
+```
+
+5. Вставьте данные:
+```
+clickhouse-client --query "INSERT INTO cell_towers FORMAT CSVWithNames" < cell_towers.csv
+```
+
+## Примеры {#examples}
+
+1. Количество вышек по типам:
+
+```
+SELECT radio, count() AS c FROM cell_towers GROUP BY radio ORDER BY c DESC
+
+┌─radio─┬────────c─┐
+│ UMTS │ 20686487 │
+│ LTE │ 12101148 │
+│ GSM │ 9931312 │
+│ CDMA │ 556344 │
+│ NR │ 867 │
+└───────┴──────────┘
+
+5 rows in set. Elapsed: 0.011 sec. Processed 43.28 million rows, 43.28 MB (3.83 billion rows/s., 3.83 GB/s.)
+```
+
+2. Количество вышек по [мобильному коду страны (MCC)](https://ru.wikipedia.org/wiki/Mobile_Country_Code):
+
+```
+SELECT mcc, count() FROM cell_towers GROUP BY mcc ORDER BY count() DESC LIMIT 10
+
+┌─mcc─┬─count()─┐
+│ 310 │ 5024650 │
+│ 262 │ 2622423 │
+│ 250 │ 1953176 │
+│ 208 │ 1891187 │
+│ 724 │ 1836150 │
+│ 404 │ 1729151 │
+│ 234 │ 1618924 │
+│ 510 │ 1353998 │
+│ 440 │ 1343355 │
+│ 311 │ 1332798 │
+└─────┴─────────┘
+
+10 rows in set. Elapsed: 0.019 sec. Processed 43.28 million rows, 86.55 MB (2.33 billion rows/s., 4.65 GB/s.)
+```
+
+Можно увидеть, что по количеству вышек лидируют следующие страны: США, Германия, Россия.
+
+Вы также можете создать [внешний словарь](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md) в ClickHouse для того, чтобы расшифровать эти значения.
+
+## Пример использования {#use-case}
+
+Рассмотрим применение функции `pointInPolygon`.
+
+1. Создаем таблицу, в которой будем хранить многоугольники:
+
+```
+CREATE TEMPORARY TABLE moscow (polygon Array(Tuple(Float64, Float64)));
+```
+
+2. Очертания Москвы выглядят приблизительно так ("Новая Москва" в них не включена):
+
+```
+INSERT INTO moscow VALUES ([(37.84172564285271, 55.78000432402266), (37.8381207618713, 55.775874525970494), (37.83979446823122, 55.775626746008065), (37.84243326983639, 55.77446586811748), (37.84262672750849, 55.771974101091104), (37.84153238623039, 55.77114545193181), (37.841124690460184, 55.76722010265554), (37.84239076983644, 55.76654891107098), (37.842283558197025, 55.76258709833121), (37.8421759312134, 55.758073999993734), (37.84198330422974, 55.75381499999371), (37.8416827275085, 55.749277102484484), (37.84157576190186, 55.74794544108413), (37.83897929098507, 55.74525257875241), (37.83739676451868, 55.74404373042019), (37.838732481460525, 55.74298009816793), (37.841183997352545, 55.743060321833575), (37.84097476190185, 55.73938799999373), (37.84048155819702, 55.73570799999372), (37.840095812164286, 55.73228210777237), (37.83983814285274, 55.73080491981639), (37.83846476321406, 55.729799917464675), (37.83835745269769, 55.72919751082619), (37.838636380279524, 55.72859509486539), (37.8395161005249, 55.727705075632784), (37.83897964285276, 55.722727886185154), (37.83862557539366, 55.72034817326636), (37.83559735744853, 55.71944437307499), (37.835370708803126, 55.71831419154461), (37.83738169402022, 55.71765218986692), (37.83823396494291, 55.71691750159089), (37.838056931213345, 55.71547311301385), (37.836812846557606, 55.71221445615604), (37.83522525396725, 55.709331054395555), (37.83269301586908, 55.70953687463627), (37.829667367706236, 55.70903403789297), (37.83311126588435, 55.70552351822608), (37.83058993121339, 55.70041317726053), (37.82983872750851, 55.69883771404813), (37.82934501586913, 55.69718947487017), (37.828926414016685, 55.69504441658371), (37.82876530422971, 55.69287499999378), (37.82894754100031, 55.690759754047335), (37.827697554878185, 55.68951421135665), (37.82447346292115, 55.68965045405069), (37.83136543914793, 55.68322046195302), (37.833554015869154, 55.67814012759211), (37.83544184655761, 55.67295011628339), (37.837480388885474, 55.6672498719639), (37.838960677246064, 55.66316274139358), (37.83926093121332, 55.66046999999383), (37.839025050262435, 55.65869897264431), (37.83670784390257, 55.65794084879904), (37.835656529083245, 55.65694309303843), (37.83704060449217, 55.65689306460552), (37.83696819873806, 55.65550363526252), (37.83760389616388, 55.65487847246661), (37.83687972750851, 55.65356745541324), (37.83515216004943, 55.65155951234079), (37.83312418518067, 55.64979413590619), (37.82801726983639, 55.64640836412121), (37.820614174591, 55.64164525405531), (37.818908190475426, 55.6421883258084), (37.81717543386075, 55.64112490388471), (37.81690987037274, 55.63916106913107), (37.815099354492155, 55.637925371757085), (37.808769150787356, 55.633798276884455), (37.80100123544311, 55.62873670012244), (37.79598013491824, 55.62554336109055), (37.78634567724606, 55.62033499605651), (37.78334147619623, 55.618768681480326), (37.77746201055901, 55.619855533402706), (37.77527329626457, 55.61909966711279), (37.77801986242668, 55.618770300976294), (37.778212973541216, 55.617257701952106), (37.77784818518065, 55.61574504433011), (37.77016867724609, 55.61148576294007), (37.760191219573976, 55.60599579539028), (37.75338926983641, 55.60227892751446), (37.746329965606634, 55.59920577639331), (37.73939925396728, 55.59631430313617), (37.73273665739439, 55.5935318803559), (37.7299954450912, 55.59350760316188), (37.7268679946899, 55.59469840523759), (37.72626726983634, 55.59229549697373), (37.7262673598022, 55.59081598950582), (37.71897193121335, 55.5877595845419), (37.70871550793456, 55.58393177431724), (37.700497489410374, 55.580917323756644), (37.69204305026244, 55.57778089778455), (37.68544477378839, 55.57815154690915), (37.68391050793454, 55.57472945079756), (37.678803592590306, 55.57328235936491), (37.6743402539673, 55.57255251445782), (37.66813862698363, 55.57216388774464), (37.617927457672096, 55.57505691895805), (37.60443099999999, 55.5757737568051), (37.599683515869145, 55.57749105910326), (37.59754177842709, 55.57796291823627), (37.59625834786988, 55.57906686095235), (37.59501783265684, 55.57746616444403), (37.593090671936025, 55.57671634534502), (37.587018007904, 55.577944600233785), (37.578692203704804, 55.57982895000019), (37.57327546607398, 55.58116294118248), (37.57385012109279, 55.581550362779), (37.57399562266922, 55.5820107079112), (37.5735356072979, 55.58226289171689), (37.57290393054962, 55.582393529795155), (37.57037722355653, 55.581919415056234), (37.5592298306885, 55.584471614867844), (37.54189249206543, 55.58867650795186), (37.5297256269836, 55.59158133551745), (37.517837865081766, 55.59443656218868), (37.51200186508174, 55.59635625174229), (37.506808949737554, 55.59907823904434), (37.49820432275389, 55.6062944994944), (37.494406071441674, 55.60967103463367), (37.494760001358024, 55.61066689753365), (37.49397137107085, 55.61220931698269), (37.49016528606031, 55.613417718449064), (37.48773249206542, 55.61530616333343), (37.47921386508177, 55.622640129112334), (37.470652153442394, 55.62993723476164), (37.46273446298218, 55.6368075123157), (37.46350692265317, 55.64068225239439), (37.46050283203121, 55.640794546982576), (37.457627470916734, 55.64118904154646), (37.450718034393326, 55.64690488145138), (37.44239252645875, 55.65397824729769), (37.434587576721185, 55.66053543155961), (37.43582144975277, 55.661693766520735), (37.43576786245721, 55.662755031737014), (37.430982915344174, 55.664610641628116), (37.428547447097685, 55.66778515273695), (37.42945134592044, 55.668633314343566), (37.42859571562949, 55.66948145750025), (37.4262836402282, 55.670813882451405), (37.418709037048295, 55.6811141674414), (37.41922139651101, 55.68235377885389), (37.419218771842885, 55.68359335082235), (37.417196501327446, 55.684375235224735), (37.41607020370478, 55.68540557585352), (37.415640857147146, 55.68686637150793), (37.414632153442334, 55.68903015131686), (37.413344899475064, 55.690896881757396), (37.41171432275391, 55.69264232162232), (37.40948282275393, 55.69455101638112), (37.40703674603271, 55.69638690385348), (37.39607169577025, 55.70451821283731), (37.38952706878662, 55.70942491932811), (37.387778313491815, 55.71149057784176), (37.39049275399779, 55.71419814298992), (37.385557272491454, 55.7155489617061), (37.38388335714726, 55.71849856042102), (37.378368238098155, 55.7292763261685), (37.37763597123337, 55.730845879211614), (37.37890062088197, 55.73167906388319), (37.37750451918789, 55.734703664681774), (37.375610832015965, 55.734851959522246), (37.3723813571472, 55.74105626086403), (37.37014935714723, 55.746115620904355), (37.36944173016362, 55.750883999993725), (37.36975304365541, 55.76335905525834), (37.37244070571134, 55.76432079697595), (37.3724259757175, 55.76636979670426), (37.369922155757884, 55.76735417953104), (37.369892695770275, 55.76823419316575), (37.370214730163575, 55.782312184391266), (37.370493611114505, 55.78436801120489), (37.37120164550783, 55.78596427165359), (37.37284851456452, 55.7874378183096), (37.37608325135799, 55.7886695054807), (37.3764587460632, 55.78947647305964), (37.37530000265506, 55.79146512926804), (37.38235915344241, 55.79899647809345), (37.384344043655396, 55.80113596939471), (37.38594269577028, 55.80322699999366), (37.38711208598329, 55.804919036911976), (37.3880239841309, 55.806610999993666), (37.38928977249147, 55.81001864976979), (37.39038389947512, 55.81348641242801), (37.39235781481933, 55.81983538336746), (37.393709457672124, 55.82417822811877), (37.394685720901464, 55.82792275755836), (37.39557615344238, 55.830447148154136), (37.39844478226658, 55.83167107969975), (37.40019761214057, 55.83151823557964), (37.400398790382326, 55.83264967594742), (37.39659544313046, 55.83322180909622), (37.39667059524539, 55.83402792148566), (37.39682089947515, 55.83638877400216), (37.39643489154053, 55.83861656112751), (37.3955338994751, 55.84072348043264), (37.392680272491454, 55.84502158126453), (37.39241188227847, 55.84659117913199), (37.392529730163616, 55.84816071336481), (37.39486835714723, 55.85288092980303), (37.39873052645878, 55.859893456073635), (37.40272161111449, 55.86441833633205), (37.40697072750854, 55.867579567544375), (37.410007082016016, 55.868369880337), (37.4120992989502, 55.86920843741314), (37.412668021163924, 55.87055369615854), (37.41482461111453, 55.87170587948249), (37.41862266137694, 55.873183961039565), (37.42413732540892, 55.874879126654704), (37.4312182698669, 55.875614937236705), (37.43111093783558, 55.8762723478417), (37.43332105622856, 55.87706546369396), (37.43385747619623, 55.87790681284802), (37.441303050262405, 55.88027084462084), (37.44747234260555, 55.87942070143253), (37.44716141796871, 55.88072960917233), (37.44769797085568, 55.88121221323979), (37.45204320500181, 55.882080694420715), (37.45673176190186, 55.882346110794586), (37.463383999999984, 55.88252729504517), (37.46682797486874, 55.88294937719063), (37.470014457672086, 55.88361266759345), (37.47751410450743, 55.88546991372396), (37.47860317658232, 55.88534929207307), (37.48165826025772, 55.882563306475106), (37.48316434442331, 55.8815803226785), (37.483831555817645, 55.882427612793315), (37.483182967125686, 55.88372791409729), (37.483092277908824, 55.88495581062434), (37.4855716508179, 55.8875561994203), (37.486440636245746, 55.887827444039566), (37.49014203439328, 55.88897899871799), (37.493210285705544, 55.890208937135604), (37.497512451065035, 55.891342397444696), (37.49780744510645, 55.89174030252967), (37.49940333499519, 55.89239745507079), (37.50018383334346, 55.89339220941865), (37.52421672750851, 55.903869074155224), (37.52977457672118, 55.90564076517974), (37.53503220370484, 55.90661661218259), (37.54042858064267, 55.90714113744566), (37.54320461007303, 55.905645048442985), (37.545686966066306, 55.906608607018505), (37.54743976120755, 55.90788552162358), (37.55796999999999, 55.90901557907218), (37.572711542327866, 55.91059395704873), (37.57942799999998, 55.91073854155573), (37.58502865872187, 55.91009969268444), (37.58739968913264, 55.90794809960554), (37.59131567193598, 55.908713267595054), (37.612687423278814, 55.902866854295375), (37.62348079629517, 55.90041967242986), (37.635797880950896, 55.898141151686396), (37.649487626983664, 55.89639275532968), (37.65619302513125, 55.89572360207488), (37.66294133862307, 55.895295577183965), (37.66874564418033, 55.89505457604897), (37.67375601586915, 55.89254677027454), (37.67744661901856, 55.8947775867987), (37.688347, 55.89450045676125), (37.69480554232789, 55.89422926332761), (37.70107096560668, 55.89322256101114), (37.705962965606716, 55.891763491662616), (37.711885134918205, 55.889110234998974), (37.71682005026245, 55.886577568759876), (37.7199315476074, 55.88458159806678), (37.72234560316464, 55.882281005794134), (37.72364385977171, 55.8809452036196), (37.725371142837474, 55.8809722706006), (37.727870902099546, 55.88037213862385), (37.73394330422971, 55.877941504088696), (37.745339592590376, 55.87208120378722), (37.75525267724611, 55.86703807949492), (37.76919976190188, 55.859821640197474), (37.827835219574, 55.82962968399116), (37.83341438888553, 55.82575289922351), (37.83652584655761, 55.82188784027888), (37.83809213491821, 55.81612575504693), (37.83605359521481, 55.81460347077685), (37.83632178569025, 55.81276696067908), (37.838623105812026, 55.811486181656385), (37.83912198147584, 55.807329380532785), (37.839079078033414, 55.80510270463816), (37.83965844708251, 55.79940712529036), (37.840581150787344, 55.79131399999368), (37.84172564285271, 55.78000432402266)]);
+```
+
+3. Проверяем, сколько сотовых вышек находится в Москве:
+
+```
+SELECT count() FROM cell_towers WHERE pointInPolygon((lon, lat), (SELECT * FROM moscow))
+
+┌─count()─┐
+│ 310463 │
+└─────────┘
+
+1 rows in set. Elapsed: 0.067 sec. Processed 43.28 million rows, 692.42 MB (645.83 million rows/s., 10.33 GB/s.)
+```
+
+Вы можете протестировать другие запросы с помощью интерактивного ресурса [Playground](https://gh-api.clickhouse.tech/play?user=play). Например, [вот так](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIG1jYywgY291bnQoKSBGUk9NIGNlbGxfdG93ZXJzIEdST1VQIEJZIG1jYyBPUkRFUiBCWSBjb3VudCgpIERFU0M=). Однако, обратите внимание, что здесь нельзя создавать временные таблицы.
diff --git a/docs/ru/getting-started/example-datasets/index.md b/docs/ru/getting-started/example-datasets/index.md
index f590300adda..756b3a75dee 100644
--- a/docs/ru/getting-started/example-datasets/index.md
+++ b/docs/ru/getting-started/example-datasets/index.md
@@ -16,4 +16,5 @@ toc_title: "Введение"
- [AMPLab Big Data Benchmark](amplab-benchmark.md)
- [Данные о такси в Нью-Йорке](nyc-taxi.md)
- [OnTime](ontime.md)
+- [Вышки сотовой связи](../../getting-started/example-datasets/cell-towers.md)
diff --git a/docs/ru/guides/apply-catboost-model.md b/docs/ru/guides/apply-catboost-model.md
index 11964c57fc7..db2be63692f 100644
--- a/docs/ru/guides/apply-catboost-model.md
+++ b/docs/ru/guides/apply-catboost-model.md
@@ -158,7 +158,9 @@ FROM amazon_train
/home/catboost/data/libcatboostmodel.so
/home/catboost/models/*_model.xml
```
-
+!!! note "Примечание"
+ Вы можете позднее изменить путь к конфигурации модели CatBoost без перезагрузки сервера.
+
## 4. Запустите вывод модели из SQL {#run-model-inference}
Для тестирования модели запустите клиент ClickHouse `$ clickhouse client`.
diff --git a/docs/ru/interfaces/cli.md b/docs/ru/interfaces/cli.md
index 96ec36be79f..277b73a6d36 100644
--- a/docs/ru/interfaces/cli.md
+++ b/docs/ru/interfaces/cli.md
@@ -121,6 +121,7 @@ $ clickhouse-client --param_tbl="numbers" --param_db="system" --param_col="numbe
- `--user, -u` — имя пользователя, по умолчанию — ‘default’.
- `--password` — пароль, по умолчанию — пустая строка.
- `--query, -q` — запрос для выполнения, при использовании в неинтерактивном режиме.
+- `--queries-file, -qf` - путь к файлу с запросами для выполнения. Необходимо указать только одну из опций: `query` или `queries-file`.
- `--database, -d` — выбрать текущую БД. Без указания значение берется из настроек сервера (по умолчанию — БД ‘default’).
- `--multiline, -m` — если указано — разрешить многострочные запросы, не отправлять запрос по нажатию Enter.
- `--multiquery, -n` — если указано — разрешить выполнять несколько запросов, разделённых точкой с запятой.
@@ -130,6 +131,7 @@ $ clickhouse-client --param_tbl="numbers" --param_db="system" --param_col="numbe
- `--stacktrace` — если указано, в случае исключения, выводить также его стек-трейс.
- `--config-file` — имя конфигурационного файла.
- `--secure` — если указано, будет использован безопасный канал.
+- `--history_file` - путь к файлу с историей команд.
- `--param_` — значение параметра для [запроса с параметрами](#cli-queries-with-parameters).
Начиная с версии 20.5, в `clickhouse-client` есть автоматическая подсветка синтаксиса (включена всегда).
diff --git a/docs/ru/interfaces/third-party/gui.md b/docs/ru/interfaces/third-party/gui.md
index f913a0ff2cc..156f7130bc5 100644
--- a/docs/ru/interfaces/third-party/gui.md
+++ b/docs/ru/interfaces/third-party/gui.md
@@ -166,4 +166,19 @@ toc_title: "Визуальные интерфейсы от сторонних р
[Как сконфигурировать ClickHouse в Looker.](https://docs.looker.com/setup-and-management/database-config/clickhouse)
-[Original article](https://clickhouse.tech/docs/ru/interfaces/third-party/gui/)
+### SeekTable {#seektable}
+
+[SeekTable](https://www.seektable.com) — это аналитический инструмент для самостоятельного анализа и обработки данных бизнес-аналитики. Он доступен как в виде облачного сервиса, так и в виде локальной версии. Отчеты из SeekTable могут быть встроены в любое веб-приложение.
+
+Основные возможности:
+
+- Удобный конструктор отчетов.
+- Гибкая настройка отчетов SQL и создание запросов для специфичных отчетов.
+- Интегрируется с ClickHouse, используя собственную точку приема запроса TCP/IP или интерфейс HTTP(S) (два разных драйвера).
+- Поддерживает всю мощь диалекта ClickHouse SQL для построения запросов по различным измерениям и показателям.
+- [WEB-API](https://www.seektable.com/help/web-api-integration) для автоматизированной генерации отчетов.
+- Процесс разработки отчетов поддерживает [резервное копирование/восстановление данных](https://www.seektable.com/help/self-hosted-backup-restore); конфигурация моделей данных (кубов) / отчетов представляет собой удобочитаемый XML-файл, который может храниться в системе контроля версий.
+
+SeekTable [бесплатен](https://www.seektable.com/help/cloud-pricing) для личного/индивидуального использования.
+
+[Как сконфигурировать подключение ClickHouse в SeekTable.](https://www.seektable.com/help/clickhouse-pivot-table)
diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md
index 109146d27f4..be9e2deab74 100644
--- a/docs/ru/operations/server-configuration-parameters/settings.md
+++ b/docs/ru/operations/server-configuration-parameters/settings.md
@@ -101,6 +101,12 @@ ClickHouse проверяет условия для `min_part_size` и `min_part
```
+## database_atomic_delay_before_drop_table_sec {#database_atomic_delay_before_drop_table_sec}
+
+Устанавливает задержку перед удалением табличных данных, в секундах. Если запрос имеет идентификатор `SYNC`, эта настройка игнорируется.
+
+Значение по умолчанию: `480` (8 минут).
+
## default\_database {#default-database}
База данных по умолчанию.
@@ -285,7 +291,7 @@ ClickHouse проверяет условия для `min_part_size` и `min_part
## interserver_http_host {#interserver-http-host}
-Имя хоста, которое могут использовать другие серверы для обращения к этому.
+Имя хоста, которое могут использовать другие серверы для обращения к этому хосту.
Если не указано, то определяется аналогично команде `hostname -f`.
@@ -297,11 +303,36 @@ ClickHouse проверяет условия для `min_part_size` и `min_part
example.yandex.ru
```
+## interserver_https_port {#interserver-https-port}
+
+Порт для обмена данными между репликами ClickHouse по протоколу `HTTPS`.
+
+**Пример**
+
+``` xml
+9010
+```
+
+## interserver_https_host {#interserver-https-host}
+
+Имя хоста, которое могут использовать другие реплики для обращения к нему по протоколу `HTTPS`.
+
+**Пример**
+
+``` xml
+example.yandex.ru
+```
+
+
+
## interserver_http_credentials {#server-settings-interserver-http-credentials}
Имя пользователя и пароль, использующиеся для аутентификации при [репликации](../../operations/server-configuration-parameters/settings.md) движками Replicated\*. Это имя пользователя и пароль используются только для взаимодействия между репликами кластера и никак не связаны с аутентификацией клиентов ClickHouse. Сервер проверяет совпадение имени и пароля для соединяющихся с ним реплик, а также использует это же имя и пароль для соединения с другими репликами. Соответственно, эти имя и пароль должны быть прописаны одинаковыми для всех реплик кластера.
По умолчанию аутентификация не используется.
+!!! note "Примечание"
+ Эти учетные данные являются общими для обмена данными по протоколам `HTTP` и `HTTPS`.
+
Раздел содержит следующие параметры:
- `user` — имя пользователя.
diff --git a/docs/ru/operations/settings/merge-tree-settings.md b/docs/ru/operations/settings/merge-tree-settings.md
index bfc0b0a2644..f9093d379e3 100644
--- a/docs/ru/operations/settings/merge-tree-settings.md
+++ b/docs/ru/operations/settings/merge-tree-settings.md
@@ -55,6 +55,26 @@ Eсли число кусков в партиции превышает знач
ClickHouse искусственно выполняет `INSERT` дольше (добавляет ‘sleep’), чтобы фоновый механизм слияния успевал слиять куски быстрее, чем они добавляются.
+## inactive_parts_to_throw_insert {#inactive-parts-to-throw-insert}
+
+Если число неактивных кусков в партиции превышает значение `inactive_parts_to_throw_insert`, `INSERT` прерывается с исключением «Too many inactive parts (N). Parts cleaning are processing significantly slower than inserts».
+
+Возможные значения:
+
+- Положительное целое число.
+
+Значение по умолчанию: 0 (не ограничено).
+
+## inactive_parts_to_delay_insert {#inactive-parts-to-delay-insert}
+
+Если число неактивных кусков в партиции больше или равно значению `inactive_parts_to_delay_insert`, `INSERT` искусственно замедляется. Это полезно, когда сервер не может быстро очистить неактивные куски.
+
+Возможные значения:
+
+- Положительное целое число.
+
+Значение по умолчанию: 0 (не ограничено).
+
## max_delay_to_insert {#max-delay-to-insert}
Величина в секундах, которая используется для расчета задержки `INSERT`, если число кусков в партиции превышает значение [parts_to_delay_insert](#parts-to-delay-insert).
diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md
index d10ac2ab317..4951be49629 100644
--- a/docs/ru/operations/settings/settings.md
+++ b/docs/ru/operations/settings/settings.md
@@ -844,8 +844,6 @@ SELECT type, query FROM system.query_log WHERE log_comment = 'log_comment test'
Значение по умолчанию: количество процессорных ядер без учёта Hyper-Threading.
-Если на сервере обычно исполняется менее одного запроса SELECT одновременно, то выставите этот параметр в значение чуть меньше количества реальных процессорных ядер.
-
Для запросов, которые быстро завершаются из-за LIMIT-а, имеет смысл выставить max_threads поменьше. Например, если нужное количество записей находится в каждом блоке, то при max_threads = 8 будет считано 8 блоков, хотя достаточно было прочитать один.
Чем меньше `max_threads`, тем меньше будет использоваться оперативки.
@@ -2690,6 +2688,28 @@ SELECT * FROM test2;
Значение по умолчанию: `0`.
+## database_atomic_wait_for_drop_and_detach_synchronously {#database_atomic_wait_for_drop_and_detach_synchronously}
+
+Добавляет модификатор `SYNC` ко всем запросам `DROP` и `DETACH`.
+
+Возможные значения:
+
+- 0 — Запросы будут выполняться с задержкой.
+- 1 — Запросы будут выполняться без задержки.
+
+Значение по умолчанию: `0`.
+
+## show_table_uuid_in_table_create_query_if_not_nil {#show_table_uuid_in_table_create_query_if_not_nil}
+
+Устанавливает отображение запроса `SHOW TABLE`.
+
+Возможные значения:
+
+- 0 — Запрос будет отображаться без UUID таблицы.
+- 1 — Запрос будет отображаться с UUID таблицы.
+
+Значение по умолчанию: `0`.
+
## allow_experimental_live_view {#allow-experimental-live-view}
Включает экспериментальную возможность использования [LIVE-представлений](../../sql-reference/statements/create/view.md#live-view).
@@ -2724,4 +2744,15 @@ SELECT * FROM test2;
Значение по умолчанию: `60`.
+## check_query_single_value_result {#check_query_single_value_result}
+
+Определяет уровень детализации результата для запросов [CHECK TABLE](../../sql-reference/statements/check-table.md#checking-mergetree-tables) для таблиц семейства `MergeTree`.
+
+Возможные значения:
+
+- 0 — запрос возвращает статус каждого куска данных таблицы.
+- 1 — запрос возвращает статус таблицы в целом.
+
+Значение по умолчанию: `0`.
+
[Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings/)
diff --git a/docs/ru/operations/system-tables/columns.md b/docs/ru/operations/system-tables/columns.md
index af4cff85439..b8a0aef2299 100644
--- a/docs/ru/operations/system-tables/columns.md
+++ b/docs/ru/operations/system-tables/columns.md
@@ -4,7 +4,9 @@
С помощью этой таблицы можно получить информацию аналогично запросу [DESCRIBE TABLE](../../sql-reference/statements/misc.md#misc-describe-table), но для многих таблиц сразу.
-Таблица `system.columns` содержит столбцы (тип столбца указан в скобках):
+Колонки [временных таблиц](../../sql-reference/statements/create/table.md#temporary-tables) содержатся в `system.columns` только в тех сессиях, в которых эти таблицы были созданы. Поле `database` у таких колонок пустое.
+
+Cтолбцы:
- `database` ([String](../../sql-reference/data-types/string.md)) — имя базы данных.
- `table` ([String](../../sql-reference/data-types/string.md)) — имя таблицы.
@@ -23,3 +25,46 @@
- `is_in_sampling_key` ([UInt8](../../sql-reference/data-types/int-uint.md)) — флаг, показывающий включение столбца в ключ выборки.
- `compression_codec` ([String](../../sql-reference/data-types/string.md)) — имя кодека сжатия.
+**Пример**
+
+```sql
+SELECT * FROM system.columns LIMIT 2 FORMAT Vertical;
+```
+
+```text
+Row 1:
+──────
+database: system
+table: aggregate_function_combinators
+name: name
+type: String
+default_kind:
+default_expression:
+data_compressed_bytes: 0
+data_uncompressed_bytes: 0
+marks_bytes: 0
+comment:
+is_in_partition_key: 0
+is_in_sorting_key: 0
+is_in_primary_key: 0
+is_in_sampling_key: 0
+compression_codec:
+
+Row 2:
+──────
+database: system
+table: aggregate_function_combinators
+name: is_internal
+type: UInt8
+default_kind:
+default_expression:
+data_compressed_bytes: 0
+data_uncompressed_bytes: 0
+marks_bytes: 0
+comment:
+is_in_partition_key: 0
+is_in_sorting_key: 0
+is_in_primary_key: 0
+is_in_sampling_key: 0
+compression_codec:
+```
diff --git a/docs/ru/operations/system-tables/replication_queue.md b/docs/ru/operations/system-tables/replication_queue.md
index 56e8c695a21..2f9d80be16f 100644
--- a/docs/ru/operations/system-tables/replication_queue.md
+++ b/docs/ru/operations/system-tables/replication_queue.md
@@ -14,7 +14,17 @@
- `node_name` ([String](../../sql-reference/data-types/string.md)) — имя узла в ZooKeeper.
-- `type` ([String](../../sql-reference/data-types/string.md)) — тип задачи в очереди: `GET_PARTS`, `MERGE_PARTS`, `DETACH_PARTS`, `DROP_PARTS` или `MUTATE_PARTS`.
+- `type` ([String](../../sql-reference/data-types/string.md)) — тип задачи в очереди:
+
+ - `GET_PART` — скачать кусок с другой реплики.
+ - `ATTACH_PART` — присоединить кусок. Задача может быть выполнена и с куском из нашей собственной реплики (если он находится в папке `detached`). Эта задача практически идентична задаче `GET_PART`, лишь немного оптимизирована.
+ - `MERGE_PARTS` — выполнить слияние кусков.
+ - `DROP_RANGE` — удалить куски в партициях из указнного диапазона.
+ - `CLEAR_COLUMN` — удалить указанный столбец из указанной партиции. Примечание: не используется с 20.4.
+ - `CLEAR_INDEX` — удалить указанный индекс из указанной партиции. Примечание: не используется с 20.4.
+ - `REPLACE_RANGE` — удалить указанный диапазон кусков и заменить их на новые.
+ - `MUTATE_PART` — применить одну или несколько мутаций к куску.
+ - `ALTER_METADATA` — применить изменения структуры таблицы в результате запросов с выражением `ALTER`.
- `create_time` ([Datetime](../../sql-reference/data-types/datetime.md)) — дата и время отправки задачи на выполнение.
@@ -77,4 +87,3 @@ last_postpone_time: 1970-01-01 03:00:00
**Смотрите также**
- [Управление таблицами ReplicatedMergeTree](../../sql-reference/statements/system.md#query-language-system-replicated)
-
diff --git a/docs/ru/operations/system-tables/tables.md b/docs/ru/operations/system-tables/tables.md
index 42e55b1f6b7..11bb6a9eda2 100644
--- a/docs/ru/operations/system-tables/tables.md
+++ b/docs/ru/operations/system-tables/tables.md
@@ -1,39 +1,94 @@
# system.tables {#system-tables}
-Содержит метаданные каждой таблицы, о которой знает сервер. Отсоединённые таблицы не отображаются в `system.tables`.
+Содержит метаданные каждой таблицы, о которой знает сервер.
-Эта таблица содержит следующие столбцы (тип столбца показан в скобках):
+Отсоединённые таблицы ([DETACH](../../sql-reference/statements/detach.md)) не отображаются в `system.tables`.
-- `database String` — имя базы данных, в которой находится таблица.
-- `name` (String) — имя таблицы.
-- `engine` (String) — движок таблицы (без параметров).
-- `is_temporary` (UInt8) — флаг, указывающий на то, временная это таблица или нет.
-- `data_path` (String) — путь к данным таблицы в файловой системе.
-- `metadata_path` (String) — путь к табличным метаданным в файловой системе.
-- `metadata_modification_time` (DateTime) — время последней модификации табличных метаданных.
-- `dependencies_database` (Array(String)) — зависимости базы данных.
-- `dependencies_table` (Array(String)) — табличные зависимости (таблицы [MaterializedView](../../engines/table-engines/special/materializedview.md), созданные на базе текущей таблицы).
-- `create_table_query` (String) — запрос, которым создавалась таблица.
-- `engine_full` (String) — параметры табличного движка.
-- `partition_key` (String) — ключ партиционирования таблицы.
-- `sorting_key` (String) — ключ сортировки таблицы.
-- `primary_key` (String) - первичный ключ таблицы.
-- `sampling_key` (String) — ключ сэмплирования таблицы.
-- `storage_policy` (String) - политика хранения данных:
+Информация о [временных таблицах](../../sql-reference/statements/create/table.md#temporary-tables) содержится в `system.tables` только в тех сессиях, в которых эти таблицы были созданы. Поле `database` у таких таблиц пустое, а флаг `is_temporary` включен.
+
+Столбцы:
+
+- `database` ([String](../../sql-reference/data-types/string.md)) — имя базы данных, в которой находится таблица.
+- `name` ([String](../../sql-reference/data-types/string.md)) — имя таблицы.
+- `engine` ([String](../../sql-reference/data-types/string.md)) — движок таблицы (без параметров).
+- `is_temporary` ([UInt8](../../sql-reference/data-types/int-uint.md)) — флаг, указывающий на то, временная это таблица или нет.
+- `data_path` ([String](../../sql-reference/data-types/string.md)) — путь к данным таблицы в файловой системе.
+- `metadata_path` ([String](../../sql-reference/data-types/string.md)) — путь к табличным метаданным в файловой системе.
+- `metadata_modification_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — время последней модификации табличных метаданных.
+- `dependencies_database` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) — зависимости базы данных.
+- `dependencies_table` ([Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md))) — табличные зависимости (таблицы [MaterializedView](../../engines/table-engines/special/materializedview.md), созданные на базе текущей таблицы).
+- `create_table_query` ([String](../../sql-reference/data-types/string.md)) — запрос, при помощи которого создавалась таблица.
+- `engine_full` ([String](../../sql-reference/data-types/string.md)) — параметры табличного движка.
+- `partition_key` ([String](../../sql-reference/data-types/string.md)) — ключ партиционирования таблицы.
+- `sorting_key` ([String](../../sql-reference/data-types/string.md)) — ключ сортировки таблицы.
+- `primary_key` ([String](../../sql-reference/data-types/string.md)) - первичный ключ таблицы.
+- `sampling_key` ([String](../../sql-reference/data-types/string.md)) — ключ сэмплирования таблицы.
+- `storage_policy` ([String](../../sql-reference/data-types/string.md)) - политика хранения данных:
- [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes)
- [Distributed](../../engines/table-engines/special/distributed.md#distributed)
-- `total_rows` (Nullable(UInt64)) - общее количество строк, если есть возможность быстро определить точное количество строк в таблице, в противном случае `Null` (включая базовую таблицу `Buffer`).
+- `total_rows` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) - общее количество строк, если есть возможность быстро определить точное количество строк в таблице, в противном случае `NULL` (включая базовую таблицу `Buffer`).
-- `total_bytes` (Nullable(UInt64)) - общее количество байт, если можно быстро определить точное количество байт для таблицы на накопителе, в противном случае `Null` (**не включает** в себя никакого базового хранилища).
+- `total_bytes` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) - общее количество байт, если можно быстро определить точное количество байт для таблицы на накопителе, в противном случае `NULL` (не включает в себя никакого базового хранилища).
- Если таблица хранит данные на диске, возвращает используемое пространство на диске (т. е. сжатое).
- Если таблица хранит данные в памяти, возвращает приблизительное количество используемых байт в памяти.
-- `lifetime_rows` (Nullable(UInt64)) - общее количество строк, добавленных оператором `INSERT` с момента запуска сервера (только для таблиц `Buffer`).
+- `lifetime_rows` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) - общее количество строк, добавленных оператором `INSERT` с момента запуска сервера (только для таблиц `Buffer`).
-- `lifetime_bytes` (Nullable(UInt64)) - общее количество байт, добавленных оператором `INSERT` с момента запуска сервера (только для таблиц `Buffer`).
+- `lifetime_bytes` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md))) - общее количество байт, добавленных оператором `INSERT` с момента запуска сервера (только для таблиц `Buffer`).
Таблица `system.tables` используется при выполнении запроса `SHOW TABLES`.
+**Пример**
+
+```sql
+SELECT * FROM system.tables LIMIT 2 FORMAT Vertical;
+```
+
+```text
+Row 1:
+──────
+database: system
+name: aggregate_function_combinators
+uuid: 00000000-0000-0000-0000-000000000000
+engine: SystemAggregateFunctionCombinators
+is_temporary: 0
+data_paths: []
+metadata_path: /var/lib/clickhouse/metadata/system/aggregate_function_combinators.sql
+metadata_modification_time: 1970-01-01 03:00:00
+dependencies_database: []
+dependencies_table: []
+create_table_query:
+engine_full:
+partition_key:
+sorting_key:
+primary_key:
+sampling_key:
+storage_policy:
+total_rows: ᴺᵁᴸᴸ
+total_bytes: ᴺᵁᴸᴸ
+
+Row 2:
+──────
+database: system
+name: asynchronous_metrics
+uuid: 00000000-0000-0000-0000-000000000000
+engine: SystemAsynchronousMetrics
+is_temporary: 0
+data_paths: []
+metadata_path: /var/lib/clickhouse/metadata/system/asynchronous_metrics.sql
+metadata_modification_time: 1970-01-01 03:00:00
+dependencies_database: []
+dependencies_table: []
+create_table_query:
+engine_full:
+partition_key:
+sorting_key:
+primary_key:
+sampling_key:
+storage_policy:
+total_rows: ᴺᵁᴸᴸ
+total_bytes: ᴺᵁᴸᴸ
+```
diff --git a/docs/ru/operations/system-tables/trace_log.md b/docs/ru/operations/system-tables/trace_log.md
index 3d22e4eabfd..6d8130c1d00 100644
--- a/docs/ru/operations/system-tables/trace_log.md
+++ b/docs/ru/operations/system-tables/trace_log.md
@@ -18,10 +18,12 @@ ClickHouse создает эту таблицу когда утсановлен
Во время соединения с сервером через `clickhouse-client`, вы видите строку похожую на `Connected to ClickHouse server version 19.18.1 revision 54429.`. Это поле содержит номер после `revision`, но не содержит строку после `version`.
-- `timer_type`([Enum8](../../sql-reference/data-types/enum.md)) — тип таймера:
+- `trace_type`([Enum8](../../sql-reference/data-types/enum.md)) — тип трассировки:
- - `Real` означает wall-clock время.
- - `CPU` означает относительное CPU время.
+ - `Real` — сбор трассировок стека адресов вызова по времени wall-clock.
+ - `CPU` — сбор трассировок стека адресов вызова по времени CPU.
+ - `Memory` — сбор выделенной памяти, когда ее размер превышает относительный инкремент.
+ - `MemorySample` — сбор случайно выделенной памяти.
- `thread_number`([UInt32](../../sql-reference/data-types/int-uint.md)) — идентификатор треда.
diff --git a/docs/ru/operations/update.md b/docs/ru/operations/update.md
index 5c187ed1604..a3e87b52ede 100644
--- a/docs/ru/operations/update.md
+++ b/docs/ru/operations/update.md
@@ -3,7 +3,7 @@ toc_priority: 47
toc_title: "Обновление ClickHouse"
---
-# Обновление ClickHouse {#obnovlenie-clickhouse}
+# Обновление ClickHouse {#clickhouse-upgrade}
Если ClickHouse установлен с помощью deb-пакетов, выполните следующие команды на сервере:
@@ -15,4 +15,17 @@ $ sudo service clickhouse-server restart
Если ClickHouse установлен не из рекомендуемых deb-пакетов, используйте соответствующий метод обновления.
-ClickHouse не поддерживает распределенное обновление. Операция должна выполняться последовательно на каждом отдельном сервере. Не обновляйте все серверы в кластере одновременно, иначе кластер становится недоступным в течение некоторого времени.
+!!! note "Примечание"
+ Вы можете обновить сразу несколько серверов, кроме случая, когда все реплики одного шарда отключены.
+
+Обновление ClickHouse до определенной версии:
+
+**Пример**
+
+`xx.yy.a.b` — это номер текущей стабильной версии. Последнюю стабильную версию можно узнать [здесь](https://github.com/ClickHouse/ClickHouse/releases)
+
+```bash
+$ sudo apt-get update
+$ sudo apt-get install clickhouse-server=xx.yy.a.b clickhouse-client=xx.yy.a.b clickhouse-common-static=xx.yy.a.b
+$ sudo service clickhouse-server restart
+```
diff --git a/docs/ru/sql-reference/aggregate-functions/combinators.md b/docs/ru/sql-reference/aggregate-functions/combinators.md
index eb52fa9bc75..74f9d1c1c05 100644
--- a/docs/ru/sql-reference/aggregate-functions/combinators.md
+++ b/docs/ru/sql-reference/aggregate-functions/combinators.md
@@ -27,6 +27,40 @@ toc_title: "Комбинаторы агрегатных функций"
Комбинаторы -If и -Array можно сочетать. При этом, должен сначала идти Array, а потом If. Примеры: `uniqArrayIf(arr, cond)`, `quantilesTimingArrayIf(level1, level2)(arr, cond)`. Из-за такого порядка получается, что аргумент cond не должен быть массивом.
+## -SimpleState {#agg-functions-combinator-simplestate}
+
+При использовании этого комбинатора агрегатная функция возвращает то же значение, но типа [SimpleAggregateFunction(...)](../../sql-reference/data-types/simpleaggregatefunction.md). Текущее значение функции может храниться в таблице для последующей работы с таблицами семейства [AggregatingMergeTree](../../engines/table-engines/mergetree-family/aggregatingmergetree.md).
+
+**Синтаксис**
+
+``` sql
+SimpleState(x)
+```
+
+**Аргументы**
+
+- `x` — параметры агрегатной функции.
+
+**Возвращаемое значение**
+
+Значение агрегатной функции типа `SimpleAggregateFunction(...)`.
+
+**Пример**
+
+Запрос:
+
+``` sql
+WITH anySimpleState(number) AS c SELECT toTypeName(c), c FROM numbers(1);
+```
+
+Результат:
+
+``` text
+┌─toTypeName(c)────────────────────────┬─c─┐
+│ SimpleAggregateFunction(any, UInt64) │ 0 │
+└──────────────────────────────────────┴───┘
+```
+
## -State {#state}
В случае применения этого комбинатора, агрегатная функция возвращает не готовое значение (например, в случае функции [uniq](reference/uniq.md#agg_function-uniq) — количество уникальных значений), а промежуточное состояние агрегации (например, в случае функции `uniq` — хэш-таблицу для расчёта количества уникальных значений), которое имеет тип `AggregateFunction(...)` и может использоваться для дальнейшей обработки или может быть сохранено в таблицу для последующей доагрегации.
@@ -247,4 +281,3 @@ FROM people
│ [3,2] │ [11.5,12.949999809265137] │
└────────┴───────────────────────────┘
```
-
diff --git a/docs/ru/sql-reference/aggregate-functions/reference/uniqhll12.md b/docs/ru/sql-reference/aggregate-functions/reference/uniqhll12.md
index 4002cc06383..7a421d419ae 100644
--- a/docs/ru/sql-reference/aggregate-functions/reference/uniqhll12.md
+++ b/docs/ru/sql-reference/aggregate-functions/reference/uniqhll12.md
@@ -26,7 +26,7 @@ uniqHLL12(x[, ...])
- Использует алгоритм HyperLogLog для аппроксимации числа различных значений аргументов.
- Используется 212 5-битовых ячеек. Размер состояния чуть больше 2.5 КБ. Результат не точный (ошибка до ~10%) для небольших множеств (<10K элементов). Однако для множеств большой кардинальности (10K - 100M) результат довольно точен (ошибка до ~1.6%). Начиная с 100M ошибка оценки будет только расти и для множеств огромной кардинальности (1B+ элементов) функция возвращает результат с очень большой неточностью.
+ Используется 2^12 5-битовых ячеек. Размер состояния чуть больше 2.5 КБ. Результат не точный (ошибка до ~10%) для небольших множеств (<10K элементов). Однако для множеств большой кардинальности (10K - 100M) результат довольно точен (ошибка до ~1.6%). Начиная с 100M ошибка оценки будет только расти и для множеств огромной кардинальности (1B+ элементов) функция возвращает результат с очень большой неточностью.
- Результат детерминирован (не зависит от порядка выполнения запроса).
diff --git a/docs/ru/sql-reference/data-types/datetime64.md b/docs/ru/sql-reference/data-types/datetime64.md
index 6576bf9dc0d..3a08da75bb7 100644
--- a/docs/ru/sql-reference/data-types/datetime64.md
+++ b/docs/ru/sql-reference/data-types/datetime64.md
@@ -7,9 +7,9 @@ toc_title: DateTime64
Позволяет хранить момент времени, который может быть представлен как календарная дата и время, с заданной суб-секундной точностью.
-Размер тика/точность: 10-precision секунд, где precision - целочисленный параметр типа.
+Размер тика (точность, precision): 10-precision секунд, где precision - целочисленный параметр.
-Синтаксис:
+**Синтаксис:**
``` sql
DateTime64(precision, [timezone])
@@ -17,9 +17,11 @@ DateTime64(precision, [timezone])
Данные хранятся в виде количества ‘тиков’, прошедших с момента начала эпохи (1970-01-01 00:00:00 UTC), в Int64. Размер тика определяется параметром precision. Дополнительно, тип `DateTime64` позволяет хранить часовой пояс, единый для всей колонки, который влияет на то, как будут отображаться значения типа `DateTime64` в текстовом виде и как будут парситься значения заданные в виде строк (‘2020-01-01 05:00:01.000’). Часовой пояс не хранится в строках таблицы (выборки), а хранится в метаданных колонки. Подробнее см. [DateTime](datetime.md).
-## Пример {#primer}
+Поддерживаются значения от 1 января 1925 г. и до 31 декабря 2283 г.
-**1.** Создание таблицы с столбцом типа `DateTime64` и вставка данных в неё:
+## Примеры {#examples}
+
+1. Создание таблицы со столбцом типа `DateTime64` и вставка данных в неё:
``` sql
CREATE TABLE dt
@@ -27,15 +29,15 @@ CREATE TABLE dt
`timestamp` DateTime64(3, 'Europe/Moscow'),
`event_id` UInt8
)
-ENGINE = TinyLog
+ENGINE = TinyLog;
```
``` sql
-INSERT INTO dt Values (1546300800000, 1), ('2019-01-01 00:00:00', 2)
+INSERT INTO dt Values (1546300800000, 1), ('2019-01-01 00:00:00', 2);
```
``` sql
-SELECT * FROM dt
+SELECT * FROM dt;
```
``` text
@@ -46,12 +48,12 @@ SELECT * FROM dt
```
- При вставке даты-времени как числа (аналогично ‘Unix timestamp’), время трактуется как UTC. Unix timestamp `1546300800` в часовом поясе `Europe/London (UTC+0)` представляет время `'2019-01-01 00:00:00'`. Однако, столбец `timestamp` имеет тип `DateTime('Europe/Moscow (UTC+3)')`, так что при выводе в виде строки время отобразится как `2019-01-01 03:00:00`.
-- При вставке даты-времени в виде строки, время трактуется соответственно часовому поясу установленному для колонки. `'2019-01-01 00:00:00'` трактуется как время по Москве (и в базу сохраняется `'2018-12-31 21:00:00'` в виде Unix Timestamp)
+- При вставке даты-времени в виде строки, время трактуется соответственно часовому поясу установленному для колонки. `'2019-01-01 00:00:00'` трактуется как время по Москве (и в базу сохраняется `'2018-12-31 21:00:00'` в виде Unix Timestamp).
-**2.** Фильтрация по значениям даты-времени
+2. Фильтрация по значениям даты и времени
``` sql
-SELECT * FROM dt WHERE timestamp = toDateTime64('2019-01-01 00:00:00', 3, 'Europe/Moscow')
+SELECT * FROM dt WHERE timestamp = toDateTime64('2019-01-01 00:00:00', 3, 'Europe/Moscow');
```
``` text
@@ -60,12 +62,12 @@ SELECT * FROM dt WHERE timestamp = toDateTime64('2019-01-01 00:00:00', 3, 'Europ
└─────────────────────────┴──────────┘
```
-В отличие от типа `DateTime`, `DateTime64` не конвертируется из строк автоматически
+В отличие от типа `DateTime`, `DateTime64` не конвертируется из строк автоматически.
-**3.** Получение часового пояса для значения типа `DateTime64`:
+3. Получение часового пояса для значения типа `DateTime64`:
``` sql
-SELECT toDateTime64(now(), 3, 'Europe/Moscow') AS column, toTypeName(column) AS x
+SELECT toDateTime64(now(), 3, 'Europe/Moscow') AS column, toTypeName(column) AS x;
```
``` text
@@ -74,13 +76,13 @@ SELECT toDateTime64(now(), 3, 'Europe/Moscow') AS column, toTypeName(column) AS
└─────────────────────────┴────────────────────────────────┘
```
-**4.** Конвертация часовых поясов
+4. Конвертация часовых поясов
``` sql
SELECT
toDateTime64(timestamp, 3, 'Europe/London') as lon_time,
toDateTime64(timestamp, 3, 'Europe/Moscow') as mos_time
-FROM dt
+FROM dt;
```
``` text
@@ -90,7 +92,7 @@ FROM dt
└─────────────────────────┴─────────────────────────┘
```
-## See Also {#see-also}
+**See Also**
- [Функции преобразования типов](../../sql-reference/functions/type-conversion-functions.md)
- [Функции для работы с датой и временем](../../sql-reference/functions/date-time-functions.md)
diff --git a/docs/ru/sql-reference/data-types/simpleaggregatefunction.md b/docs/ru/sql-reference/data-types/simpleaggregatefunction.md
index 0948153362b..7b81c577762 100644
--- a/docs/ru/sql-reference/data-types/simpleaggregatefunction.md
+++ b/docs/ru/sql-reference/data-types/simpleaggregatefunction.md
@@ -3,6 +3,8 @@
Хранит только текущее значение агрегатной функции и не сохраняет ее полное состояние, как это делает [`AggregateFunction`](../../sql-reference/data-types/aggregatefunction.md). Такая оптимизация может быть применена к функциям, которые обладают следующим свойством: результат выполнения функции `f` к набору строк `S1 UNION ALL S2` может быть получен путем выполнения `f` к отдельным частям набора строк,
а затем повторного выполнения `f` к результатам: `f(S1 UNION ALL S2) = f(f(S1) UNION ALL f(S2))`. Это свойство гарантирует, что результатов частичной агрегации достаточно для вычисления комбинированной, поэтому хранить и обрабатывать какие-либо дополнительные данные не требуется.
+Чтобы получить промежуточное значение, обычно используются агрегатные функции с суффиксом [-SimpleState](../../sql-reference/aggregate-functions/combinators.md#agg-functions-combinator-simplestate).
+
Поддерживаются следующие агрегатные функции:
- [`any`](../../sql-reference/aggregate-functions/reference/any.md#agg_function-any)
@@ -15,10 +17,12 @@
- [`groupBitOr`](../../sql-reference/aggregate-functions/reference/groupbitor.md#groupbitor)
- [`groupBitXor`](../../sql-reference/aggregate-functions/reference/groupbitxor.md#groupbitxor)
- [`groupArrayArray`](../../sql-reference/aggregate-functions/reference/grouparray.md#agg_function-grouparray)
-- [`groupUniqArrayArray`](../../sql-reference/aggregate-functions/reference/groupuniqarray.md#groupuniqarray)
+- [`groupUniqArrayArray`](../../sql-reference/aggregate-functions/reference/groupuniqarray.md)
- [`sumMap`](../../sql-reference/aggregate-functions/reference/summap.md#agg_functions-summap)
- [`minMap`](../../sql-reference/aggregate-functions/reference/minmap.md#agg_functions-minmap)
- [`maxMap`](../../sql-reference/aggregate-functions/reference/maxmap.md#agg_functions-maxmap)
+- [`argMin`](../../sql-reference/aggregate-functions/reference/argmin.md)
+- [`argMax`](../../sql-reference/aggregate-functions/reference/argmax.md)
!!! note "Примечание"
Значения `SimpleAggregateFunction(func, Type)` отображаются и хранятся так же, как и `Type`, поэтому комбинаторы [-Merge](../../sql-reference/aggregate-functions/combinators.md#aggregate_functions_combinators-merge) и [-State](../../sql-reference/aggregate-functions/combinators.md#agg-functions-combinator-state) не требуются.
diff --git a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md
index 57f53390d1c..609ee225ce2 100644
--- a/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md
+++ b/docs/ru/sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md
@@ -3,7 +3,7 @@ toc_priority: 44
toc_title: "Ключ и поля словаря"
---
-# Ключ и поля словаря {#kliuch-i-polia-slovaria}
+# Ключ и поля словаря {#dictionary-key-and-fields}
Секция `` описывает ключ словаря и поля, доступные для запросов.
@@ -88,7 +88,7 @@ PRIMARY KEY Id
- `PRIMARY KEY` – имя столбца с ключами.
-### Составной ключ {#sostavnoi-kliuch}
+### Составной ключ {#composite-key}
Ключом может быть кортеж (`tuple`) из полей произвольных типов. В этом случае [layout](external-dicts-dict-layout.md) должен быть `complex_key_hashed` или `complex_key_cache`.
@@ -159,13 +159,12 @@ CREATE DICTIONARY somename (
| Тег | Описание | Обязательный |
|------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------|
| `name` | Имя столбца. | Да |
-| `type` | Тип данных ClickHouse.
ClickHouse пытается привести значение из словаря к заданному типу данных. Например, в случае MySQL, в таблице-источнике поле может быть `TEXT`, `VARCHAR`, `BLOB`, но загружено может быть как `String`. [Nullable](../../../sql-reference/data-types/nullable.md) не поддерживается. | Да |
-| `null_value` | Значение по умолчанию для несуществующего элемента.
В примере это пустая строка. Нельзя указать значение `NULL`. | Да |
+| `type` | Тип данных ClickHouse.
ClickHouse пытается привести значение из словаря к заданному типу данных. Например, в случае MySQL, в таблице-источнике поле может быть `TEXT`, `VARCHAR`, `BLOB`, но загружено может быть как `String`.
[Nullable](../../../sql-reference/data-types/nullable.md) в настоящее время поддерживается для словарей [Flat](external-dicts-dict-layout.md#flat), [Hashed](external-dicts-dict-layout.md#dicts-external_dicts_dict_layout-hashed), [ComplexKeyHashed](external-dicts-dict-layout.md#complex-key-hashed), [Direct](external-dicts-dict-layout.md#direct), [ComplexKeyDirect](external-dicts-dict-layout.md#complex-key-direct), [RangeHashed](external-dicts-dict-layout.md#range-hashed), [Polygon](external-dicts-dict-polygon.md). Для словарей [Cache](external-dicts-dict-layout.md#cache), [ComplexKeyCache](external-dicts-dict-layout.md#complex-key-cache), [SSDCache](external-dicts-dict-layout.md#ssd-cache), [SSDComplexKeyCache](external-dicts-dict-layout.md#complex-key-ssd-cache) и [IPTrie](external-dicts-dict-layout.md#ip-trie) `Nullable`-типы не поддерживаются. | Да |
+| `null_value` | Значение по умолчанию для несуществующего элемента.
В примере это пустая строка. Значение [NULL](../../syntax.md#null-literal) можно указывать только для типов `Nullable` (см. предыдущую строку с описанием типов). | Да |
| `expression` | [Выражение](../../syntax.md#syntax-expressions), которое ClickHouse выполняет со значением.
Выражением может быть имя столбца в удаленной SQL базе. Таким образом, вы можете использовать его для создания псевдонима удаленного столбца.
Значение по умолчанию: нет выражения. | Нет |
-| `hierarchical` | Если `true`, то атрибут содержит ключ предка для текущего элемента. Смотрите [Иерархические словари](external-dicts-dict-hierarchical.md).
Default value: `false`. | No |
+| `hierarchical` | Если `true`, то атрибут содержит ключ предка для текущего элемента. Смотрите [Иерархические словари](external-dicts-dict-hierarchical.md).
Значение по умолчанию: `false`. | Нет |
| `is_object_id` | Признак того, что запрос выполняется к документу MongoDB по `ObjectID`.
Значение по умолчанию: `false`. | Нет |
-## Смотрите также {#smotrite-takzhe}
+**Смотрите также**
- [Функции для работы с внешними словарями](../../../sql-reference/functions/ext-dict-functions.md).
-
diff --git a/docs/ru/sql-reference/dictionaries/index.md b/docs/ru/sql-reference/dictionaries/index.md
index bd432497be8..59c7518d0c5 100644
--- a/docs/ru/sql-reference/dictionaries/index.md
+++ b/docs/ru/sql-reference/dictionaries/index.md
@@ -10,8 +10,6 @@ toc_title: "Введение"
ClickHouse поддерживает специальные функции для работы со словарями, которые можно использовать в запросах. Проще и эффективнее использовать словари с помощью функций, чем `JOIN` с таблицами-справочниками.
-В словаре нельзя хранить значения [NULL](../../sql-reference/syntax.md#null-literal).
-
ClickHouse поддерживает:
- [Встроенные словари](internal-dicts.md#internal_dicts) со специфическим [набором функций](../../sql-reference/dictionaries/external-dictionaries/index.md).
diff --git a/docs/ru/sql-reference/functions/bitmap-functions.md b/docs/ru/sql-reference/functions/bitmap-functions.md
index ddae2f3eb40..3da729664d0 100644
--- a/docs/ru/sql-reference/functions/bitmap-functions.md
+++ b/docs/ru/sql-reference/functions/bitmap-functions.md
@@ -25,7 +25,7 @@ SELECT bitmapBuild([1, 2, 3, 4, 5]) AS res, toTypeName(res);
``` text
┌─res─┬─toTypeName(bitmapBuild([1, 2, 3, 4, 5]))─────┐
-│ │ AggregateFunction(groupBitmap, UInt8) │
+│ │ AggregateFunction(groupBitmap, UInt8) │
└─────┴──────────────────────────────────────────────┘
```
diff --git a/docs/ru/sql-reference/functions/hash-functions.md b/docs/ru/sql-reference/functions/hash-functions.md
index 2efff9c3727..07c741e0588 100644
--- a/docs/ru/sql-reference/functions/hash-functions.md
+++ b/docs/ru/sql-reference/functions/hash-functions.md
@@ -430,7 +430,7 @@ murmurHash3_128( expr )
**Аргументы**
-- `expr` — [выражение](../syntax.md#syntax-expressions), возвращающее значение типа[String](../../sql-reference/functions/hash-functions.md).
+- `expr` — [выражение](../syntax.md#syntax-expressions), возвращающее значение типа [String](../../sql-reference/functions/hash-functions.md).
**Возвращаемое значение**
@@ -439,13 +439,13 @@ murmurHash3_128( expr )
**Пример**
``` sql
-SELECT murmurHash3_128('example_string') AS MurmurHash3, toTypeName(MurmurHash3) AS type;
+SELECT hex(murmurHash3_128('example_string')) AS MurmurHash3, toTypeName(MurmurHash3) AS type;
```
``` text
-┌─MurmurHash3──────┬─type────────────┐
-│ 6�1�4"S5KT�~~q │ FixedString(16) │
-└──────────────────┴─────────────────┘
+┌─MurmurHash3──────────────────────┬─type───┐
+│ 368A1A311CB7342253354B548E7E7E71 │ String │
+└──────────────────────────────────┴────────┘
```
## xxHash32, xxHash64 {#hash-functions-xxhash32-xxhash64}
diff --git a/docs/ru/sql-reference/functions/json-functions.md b/docs/ru/sql-reference/functions/json-functions.md
index 5d419d26981..4de487c03ad 100644
--- a/docs/ru/sql-reference/functions/json-functions.md
+++ b/docs/ru/sql-reference/functions/json-functions.md
@@ -16,51 +16,65 @@ toc_title: JSON
## visitParamHas(params, name) {#visitparamhasparams-name}
-Проверить наличие поля с именем name.
+Проверяет наличие поля с именем `name`.
+
+Алиас: `simpleJSONHas`.
## visitParamExtractUInt(params, name) {#visitparamextractuintparams-name}
-Распарсить UInt64 из значения поля с именем name. Если поле строковое - попытаться распарсить число из начала строки. Если такого поля нет, или если оно есть, но содержит не число, то вернуть 0.
+Пытается выделить число типа UInt64 из значения поля с именем `name`. Если поле строковое, пытается выделить число из начала строки. Если такого поля нет, или если оно есть, но содержит не число, то возвращает 0.
+
+Алиас: `simpleJSONExtractUInt`.
## visitParamExtractInt(params, name) {#visitparamextractintparams-name}
Аналогично для Int64.
+Алиас: `simpleJSONExtractInt`.
+
## visitParamExtractFloat(params, name) {#visitparamextractfloatparams-name}
Аналогично для Float64.
+Алиас: `simpleJSONExtractFloat`.
+
## visitParamExtractBool(params, name) {#visitparamextractboolparams-name}
-Распарсить значение true/false. Результат - UInt8.
+Пытается выделить значение true/false. Результат — UInt8.
+
+Алиас: `simpleJSONExtractBool`.
## visitParamExtractRaw(params, name) {#visitparamextractrawparams-name}
-Вернуть значение поля, включая разделители.
+Возвращает значение поля, включая разделители.
+
+Алиас: `simpleJSONExtractRaw`.
Примеры:
``` sql
-visitParamExtractRaw('{"abc":"\\n\\u0000"}', 'abc') = '"\\n\\u0000"'
-visitParamExtractRaw('{"abc":{"def":[1,2,3]}}', 'abc') = '{"def":[1,2,3]}'
+visitParamExtractRaw('{"abc":"\\n\\u0000"}', 'abc') = '"\\n\\u0000"';
+visitParamExtractRaw('{"abc":{"def":[1,2,3]}}', 'abc') = '{"def":[1,2,3]}';
```
## visitParamExtractString(params, name) {#visitparamextractstringparams-name}
-Распарсить строку в двойных кавычках. У значения убирается экранирование. Если убрать экранированные символы не удалось, то возвращается пустая строка.
+Разбирает строку в двойных кавычках. У значения убирается экранирование. Если убрать экранированные символы не удалось, то возвращается пустая строка.
+
+Алиас: `simpleJSONExtractString`.
Примеры:
``` sql
-visitParamExtractString('{"abc":"\\n\\u0000"}', 'abc') = '\n\0'
-visitParamExtractString('{"abc":"\\u263a"}', 'abc') = '☺'
-visitParamExtractString('{"abc":"\\u263"}', 'abc') = ''
-visitParamExtractString('{"abc":"hello}', 'abc') = ''
+visitParamExtractString('{"abc":"\\n\\u0000"}', 'abc') = '\n\0';
+visitParamExtractString('{"abc":"\\u263a"}', 'abc') = '☺';
+visitParamExtractString('{"abc":"\\u263"}', 'abc') = '';
+visitParamExtractString('{"abc":"hello}', 'abc') = '';
```
-На данный момент, не поддерживаются записанные в формате `\uXXXX\uYYYY` кодовые точки не из basic multilingual plane (они переводятся не в UTF-8, а в CESU-8).
+На данный момент не поддерживаются записанные в формате `\uXXXX\uYYYY` кодовые точки не из basic multilingual plane (они переводятся не в UTF-8, а в CESU-8).
-Следующие функции используют [simdjson](https://github.com/lemire/simdjson) который разработан под более сложные требования для разбора JSON. Упомянутое выше предположение 2 по-прежнему применимо.
+Следующие функции используют [simdjson](https://github.com/lemire/simdjson), который разработан под более сложные требования для разбора JSON. Упомянутое выше допущение 2 по-прежнему применимо.
## isValidJSON(json) {#isvalidjsonjson}
@@ -292,4 +306,3 @@ SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello"
│ [('d','"hello"'),('f','"world"')] │
└───────────────────────────────────────────────────────────────────────────────────────────────────────┘
```
-
diff --git a/docs/ru/sql-reference/functions/other-functions.md b/docs/ru/sql-reference/functions/other-functions.md
index f9b3e5c3e68..84bbc6af968 100644
--- a/docs/ru/sql-reference/functions/other-functions.md
+++ b/docs/ru/sql-reference/functions/other-functions.md
@@ -1133,6 +1133,111 @@ SELECT defaultValueOfTypeName('Nullable(Int8)')
└──────────────────────────────────────────┘
```
+## indexHint {#indexhint}
+Возвращает все данные из диапазона, в который попадают данные, соответствующие указанному выражению.
+Переданное выражение не будет вычислено. Выбор диапазона производится по индексу.
+Индекс в ClickHouse разреженный, при чтении диапазона в ответ попадают «лишние» соседние данные.
+
+**Синтаксис**
+
+```sql
+SELECT * FROM table WHERE indexHint()
+```
+
+**Возвращаемое значение**
+
+Возвращает диапазон индекса, в котором выполняется заданное условие.
+
+Тип: [Uint8](https://clickhouse.yandex/docs/ru/data_types/int_uint/#diapazony-uint).
+
+**Пример**
+
+Рассмотрим пример с использованием тестовых данных таблицы [ontime](../../getting-started/example-datasets/ontime.md).
+
+Исходная таблица:
+
+```sql
+SELECT count() FROM ontime
+```
+
+```text
+┌─count()─┐
+│ 4276457 │
+└─────────┘
+```
+
+В таблице есть индексы по полям `(FlightDate, (Year, FlightDate))`.
+
+Выполним выборку по дате, где индекс не используется.
+
+Запрос:
+
+```sql
+SELECT FlightDate AS k, count() FROM ontime GROUP BY k ORDER BY k
+```
+
+ClickHouse обработал всю таблицу (`Processed 4.28 million rows`).
+
+Результат:
+
+```text
+┌──────────k─┬─count()─┐
+│ 2017-01-01 │ 13970 │
+│ 2017-01-02 │ 15882 │
+........................
+│ 2017-09-28 │ 16411 │
+│ 2017-09-29 │ 16384 │
+│ 2017-09-30 │ 12520 │
+└────────────┴─────────┘
+```
+
+Для подключения индекса выбираем конкретную дату.
+
+Запрос:
+
+```sql
+SELECT FlightDate AS k, count() FROM ontime WHERE k = '2017-09-15' GROUP BY k ORDER BY k
+```
+
+При использовании индекса ClickHouse обработал значительно меньшее количество строк (`Processed 32.74 thousand rows`).
+
+Результат:
+
+```text
+┌──────────k─┬─count()─┐
+│ 2017-09-15 │ 16428 │
+└────────────┴─────────┘
+```
+
+Передадим в функцию `indexHint` выражение `k = '2017-09-15'`.
+
+Запрос:
+
+```sql
+SELECT
+ FlightDate AS k,
+ count()
+FROM ontime
+WHERE indexHint(k = '2017-09-15')
+GROUP BY k
+ORDER BY k ASC
+```
+
+ClickHouse применил индекс по аналогии с примером выше (`Processed 32.74 thousand rows`).
+Выражение `k = '2017-09-15'` не используется при формировании результата.
+Функция `indexHint` позволяет увидеть соседние данные.
+
+Результат:
+
+```text
+┌──────────k─┬─count()─┐
+│ 2017-09-14 │ 7071 │
+│ 2017-09-15 │ 16428 │
+│ 2017-09-16 │ 1077 │
+│ 2017-09-30 │ 8167 │
+└────────────┴─────────┘
+```
+
## replicate {#other-functions-replicate}
Создает массив, заполненный одним значением.
diff --git a/docs/ru/sql-reference/functions/string-functions.md b/docs/ru/sql-reference/functions/string-functions.md
index 6ef7dc01b6a..04af599c09a 100644
--- a/docs/ru/sql-reference/functions/string-functions.md
+++ b/docs/ru/sql-reference/functions/string-functions.md
@@ -645,3 +645,66 @@ SELECT decodeXMLComponent('< Σ >');
- [Мнемоники в HTML](https://ru.wikipedia.org/wiki/%D0%9C%D0%BD%D0%B5%D0%BC%D0%BE%D0%BD%D0%B8%D0%BA%D0%B8_%D0%B2_HTML)
+
+
+## extractTextFromHTML {#extracttextfromhtml}
+
+Функция для извлечения текста из HTML или XHTML.
+Она не соответствует всем HTML, XML или XHTML стандартам на 100%, но ее реализация достаточно точная и быстрая. Правила обработки следующие:
+
+1. Комментарии удаляются. Пример: ``. Комментарий должен оканчиваться символами `-->`. Вложенные комментарии недопустимы.
+Примечание: конструкции наподобие `` и `` не являются допустимыми комментариями в HTML, но они будут удалены согласно другим правилам.
+2. Содержимое CDATA вставляется дословно. Примечание: формат CDATA специфичен для XML/XHTML. Но он обрабатывается всегда по принципу "наилучшего возможного результата".
+3. Элементы `script` и `style` удаляются вместе со всем содержимым. Примечание: предполагается, что закрывающий тег не может появиться внутри содержимого. Например, в JS строковый литерал должен быть экранирован как `"<\/script>"`.
+Примечание: комментарии и CDATA возможны внутри `script` или `style` - тогда закрывающие теги не ищутся внутри CDATA. Пример: `]]>`. Но они ищутся внутри комментариев. Иногда возникают сложные случаи: ` var y = "-->"; alert(x + y);`
+Примечание: `script` и `style` могут быть названиями пространств имен XML - тогда они не обрабатываются как обычные элементы `script` или `style`. Пример: `Hello`.
+Примечание: пробелы возможны после имени закрывающего тега: ``, но не перед ним: `< / script>`.
+4. Другие теги или элементы, подобные тегам, удаляются, а их внутреннее содержимое остается. Пример: `.`
+Примечание: ожидается, что такой HTML является недопустимым: ``
+Примечание: функция также удаляет подобные тегам элементы: `<>`, ``, и т. д.
+Примечание: если встречается тег без завершающего символа `>`, то удаляется этот тег и весь следующий за ним текст: `world`, `Helloworld` — в HTML нет пробелов, но функция вставляет их. Также следует учитывать такие варианты написания: `Helloworld
`, `Hello
world`. Подобные результаты выполнения функции могут использоваться для анализа данных, например, для преобразования HTML-текста в набор используемых слов.
+7. Также обратите внимание, что правильная обработка пробелов требует поддержки `` и свойств CSS `display` и `white-space`.
+
+**Синтаксис**
+
+``` sql
+extractTextFromHTML(x)
+```
+
+**Аргументы**
+
+- `x` — текст для обработки. [String](../../sql-reference/data-types/string.md).
+
+**Возвращаемое значение**
+
+- Извлеченный текст.
+
+Тип: [String](../../sql-reference/data-types/string.md).
+
+**Пример**
+
+Первый пример содержит несколько тегов и комментарий. На этом примере также видно, как обрабатываются пробелы.
+Второй пример показывает обработку `CDATA` и тега `script`.
+В третьем примере текст выделяется из полного HTML ответа, полученного с помощью функции [url](../../sql-reference/table-functions/url.md).
+
+Запрос:
+
+``` sql
+SELECT extractTextFromHTML(' A text withtags.
');
+SELECT extractTextFromHTML('CDATA]]> ');
+SELECT extractTextFromHTML(html) FROM url('http://www.donothingfor2minutes.com/', RawBLOB, 'html String');
+```
+
+Результат:
+
+``` text
+A text with tags .
+The content within CDATA
+Do Nothing for 2 Minutes 2:00
+```
diff --git a/docs/ru/sql-reference/statements/alter/column.md b/docs/ru/sql-reference/statements/alter/column.md
index 87fc1c78cd0..158ab2e7385 100644
--- a/docs/ru/sql-reference/statements/alter/column.md
+++ b/docs/ru/sql-reference/statements/alter/column.md
@@ -63,6 +63,9 @@ DROP COLUMN [IF EXISTS] name
Запрос удаляет данные из файловой системы. Так как это представляет собой удаление целых файлов, запрос выполняется почти мгновенно.
+!!! warning "Предупреждение"
+ Вы не можете удалить столбец, используемый в [материализованном представлениии](../../../sql-reference/statements/create/view.md#materialized). В противном случае будет ошибка.
+
Пример:
``` sql
@@ -155,7 +158,7 @@ ALTER TABLE table_name MODIFY column_name REMOVE property;
ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL;
```
-## Смотрите также
+**Смотрите также**
- [REMOVE TTL](ttl.md).
diff --git a/docs/ru/sql-reference/statements/alter/partition.md b/docs/ru/sql-reference/statements/alter/partition.md
index 3e7b069b066..02a87406e86 100644
--- a/docs/ru/sql-reference/statements/alter/partition.md
+++ b/docs/ru/sql-reference/statements/alter/partition.md
@@ -38,7 +38,7 @@ ALTER TABLE mt DETACH PART 'all_2_2_0';
После того как запрос будет выполнен, вы сможете производить любые операции с данными в директории `detached`. Например, можно удалить их из файловой системы.
-Запрос реплицируется — данные будут перенесены в директорию `detached` и забыты на всех репликах. Обратите внимание, запрос может быть отправлен только на реплику-лидер. Чтобы узнать, является ли реплика лидером, выполните запрос `SELECT` к системной таблице [system.replicas](../../../operations/system-tables/replicas.md#system_tables-replicas). Либо можно выполнить запрос `DETACH` на всех репликах — тогда на всех репликах, кроме реплики-лидера, запрос вернет ошибку.
+Запрос реплицируется — данные будут перенесены в директорию `detached` и забыты на всех репликах. Обратите внимание, запрос может быть отправлен только на реплику-лидер. Чтобы узнать, является ли реплика лидером, выполните запрос `SELECT` к системной таблице [system.replicas](../../../operations/system-tables/replicas.md#system_tables-replicas). Либо можно выполнить запрос `DETACH` на всех репликах — тогда на всех репликах, кроме реплик-лидеров (поскольку допускается несколько лидеров), запрос вернет ошибку.
## DROP PARTITION\|PART {#alter_drop-partition}
@@ -83,9 +83,13 @@ ALTER TABLE visits ATTACH PART 201901_2_2_0;
Как корректно задать имя партиции или куска, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr).
-Этот запрос реплицируется. Реплика-иницатор проверяет, есть ли данные в директории `detached`. Если данные есть, то запрос проверяет их целостность. В случае успеха данные добавляются в таблицу. Все остальные реплики загружают данные с реплики-инициатора запроса.
+Этот запрос реплицируется. Реплика-иницатор проверяет, есть ли данные в директории `detached`.
+Если данные есть, то запрос проверяет их целостность. В случае успеха данные добавляются в таблицу.
-Это означает, что вы можете разместить данные в директории `detached` на одной реплике и с помощью запроса `ALTER ... ATTACH` добавить их в таблицу на всех репликах.
+Если реплика, не являющаяся инициатором запроса, получив команду присоединения, находит кусок с правильными контрольными суммами в своей собственной папке `detached`, она присоединяет данные, не скачивая их с других реплик.
+Если нет куска с правильными контрольными суммами, данные загружаются из любой реплики, имеющей этот кусок.
+
+Вы можете поместить данные в директорию `detached` на одной реплике и с помощью запроса `ALTER ... ATTACH` добавить их в таблицу на всех репликах.
## ATTACH PARTITION FROM {#alter_attach-partition-from}
@@ -93,7 +97,8 @@ ALTER TABLE visits ATTACH PART 201901_2_2_0;
ALTER TABLE table2 ATTACH PARTITION partition_expr FROM table1
```
-Копирует партицию из таблицы `table1` в таблицу `table2` и добавляет к существующим данным `table2`. Данные из `table1` не удаляются.
+Копирует партицию из таблицы `table1` в таблицу `table2`.
+Обратите внимание, что данные не удаляются ни из `table1`, ни из `table2`.
Следует иметь в виду:
@@ -305,4 +310,3 @@ OPTIMIZE TABLE table_not_partitioned PARTITION tuple() FINAL;
`IN PARTITION` указывает на партицию, для которой применяются выражения [UPDATE](../../../sql-reference/statements/alter/update.md#alter-table-update-statements) или [DELETE](../../../sql-reference/statements/alter/delete.md#alter-mutations) в результате запроса `ALTER TABLE`. Новые куски создаются только в указанной партиции. Таким образом, `IN PARTITION` помогает снизить нагрузку, когда таблица разбита на множество партиций, а вам нужно обновить данные лишь точечно.
Примеры запросов `ALTER ... PARTITION` можно посмотреть в тестах: [`00502_custom_partitioning_local`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_local.sql) и [`00502_custom_partitioning_replicated_zookeeper`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.sql).
-
diff --git a/docs/ru/sql-reference/statements/alter/ttl.md b/docs/ru/sql-reference/statements/alter/ttl.md
index e949c992bbe..2a2d10b69de 100644
--- a/docs/ru/sql-reference/statements/alter/ttl.md
+++ b/docs/ru/sql-reference/statements/alter/ttl.md
@@ -82,4 +82,4 @@ SELECT * FROM table_with_ttl;
### Смотрите также
- Подробнее о [свойстве TTL](../../../engines/table-engines/mergetree-family/mergetree.md#mergetree-column-ttl).
-
+- Изменить столбец [с TTL](../../../sql-reference/statements/alter/column.md#alter_modify-column).
\ No newline at end of file
diff --git a/docs/ru/sql-reference/statements/alter/user.md b/docs/ru/sql-reference/statements/alter/user.md
index 604eff9de15..53d090f8eab 100644
--- a/docs/ru/sql-reference/statements/alter/user.md
+++ b/docs/ru/sql-reference/statements/alter/user.md
@@ -12,10 +12,10 @@ toc_title: USER
``` sql
ALTER USER [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1]
[, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...]
- [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}]
- [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
+ [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']}]
+ [[ADD | DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
[DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ]
- [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
+ [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY | WRITABLE] | PROFILE 'profile_name'] [,...]
```
Для выполнения `ALTER USER` необходима привилегия [ALTER USER](../grant.md#grant-access-management).
diff --git a/docs/ru/sql-reference/statements/attach.md b/docs/ru/sql-reference/statements/attach.md
index 18ec47d05c8..b135507b818 100644
--- a/docs/ru/sql-reference/statements/attach.md
+++ b/docs/ru/sql-reference/statements/attach.md
@@ -5,12 +5,14 @@ toc_title: ATTACH
# ATTACH Statement {#attach}
-Запрос полностью аналогичен запросу `CREATE`, но:
+Выполняет подключение таблицы, например, при перемещении базы данных на другой сервер.
-- вместо слова `CREATE` используется слово `ATTACH`;
-- запрос не создаёт данные на диске, а предполагает, что данные уже лежат в соответствующих местах, и всего лишь добавляет информацию о таблице на сервер. После выполнения запроса `ATTACH` сервер будет знать о существовании таблицы.
+Запрос не создаёт данные на диске, а предполагает, что данные уже лежат в соответствующих местах, и всего лишь добавляет информацию о таблице на сервер. После выполнения запроса `ATTACH` сервер будет знать о существовании таблицы.
-Если таблица перед этим была отключена ([DETACH](../../sql-reference/statements/detach.md)), т.е. её структура известна, можно использовать сокращенную форму записи без определения структуры.
+Если таблица перед этим была отключена при помощи ([DETACH](../../sql-reference/statements/detach.md)), т.е. её структура известна, можно использовать сокращенную форму записи без определения структуры.
+
+## Варианты синтаксиса {#syntax-forms}
+### Присоединение существующей таблицы {#attach-existing-table}
``` sql
ATTACH TABLE [IF NOT EXISTS] [db.]name [ON CLUSTER cluster]
@@ -20,4 +22,38 @@ ATTACH TABLE [IF NOT EXISTS] [db.]name [ON CLUSTER cluster]
Если таблица была отключена перманентно, она не будет подключена обратно во время старта сервера, так что нужно явно использовать запрос `ATTACH`, чтобы подключить ее.
+### Создание новой таблицы и присоединение данных {#create-new-table-and-attach-data}
+**С указанием пути к табличным данным**
+
+```sql
+ATTACH TABLE name FROM 'path/to/data/' (col1 Type1, ...)
+```
+
+Cоздает новую таблицу с указанной структурой и присоединяет табличные данные из соответствующего каталога в `user_files`.
+
+**Пример**
+
+Запрос:
+
+```sql
+DROP TABLE IF EXISTS test;
+INSERT INTO TABLE FUNCTION file('01188_attach/test/data.TSV', 'TSV', 's String, n UInt8') VALUES ('test', 42);
+ATTACH TABLE test FROM '01188_attach/test' (s String, n UInt8) ENGINE = File(TSV);
+SELECT * FROM test;
+```
+Результат:
+
+```sql
+┌─s────┬──n─┐
+│ test │ 42 │
+└──────┴────┘
+```
+
+**С указанием UUID таблицы** (Только для баз данных `Atomic`)
+
+```sql
+ATTACH TABLE name UUID '' (col1 Type1, ...)
+```
+
+Cоздает новую таблицу с указанной структурой и присоединяет данные из таблицы с указанным UUID.
diff --git a/docs/ru/sql-reference/statements/check-table.md b/docs/ru/sql-reference/statements/check-table.md
index 10336f821d0..9592c1a5bc2 100644
--- a/docs/ru/sql-reference/statements/check-table.md
+++ b/docs/ru/sql-reference/statements/check-table.md
@@ -29,9 +29,36 @@ CHECK TABLE [db.]name
В движках `*Log` не предусмотрено автоматическое восстановление данных после сбоя. Используйте запрос `CHECK TABLE`, чтобы своевременно выявлять повреждение данных.
-Для движков из семейства `MergeTree` запрос `CHECK TABLE` показывает статус проверки для каждого отдельного куска данных таблицы на локальном сервере.
+## Проверка таблиц семейства MergeTree {#checking-mergetree-tables}
-**Что делать, если данные повреждены**
+Для таблиц семейства `MergeTree` если [check_query_single_value_result](../../operations/settings/settings.md#check_query_single_value_result) = 0, запрос `CHECK TABLE` возвращает статус каждого куска данных таблицы на локальном сервере.
+
+```sql
+SET check_query_single_value_result = 0;
+CHECK TABLE test_table;
+```
+
+```text
+┌─part_path─┬─is_passed─┬─message─┐
+│ all_1_4_1 │ 1 │ │
+│ all_1_4_2 │ 1 │ │
+└───────────┴───────────┴─────────┘
+```
+
+Если `check_query_single_value_result` = 0, запрос `CHECK TABLE` возвращает статус таблицы в целом.
+
+```sql
+SET check_query_single_value_result = 1;
+CHECK TABLE test_table;
+```
+
+```text
+┌─result─┐
+│ 1 │
+└────────┘
+```
+
+## Что делать, если данные повреждены {#if-data-is-corrupted}
В этом случае можно скопировать оставшиеся неповрежденные данные в другую таблицу. Для этого:
diff --git a/docs/ru/sql-reference/statements/create/row-policy.md b/docs/ru/sql-reference/statements/create/row-policy.md
index 88709598906..6fe1dc45815 100644
--- a/docs/ru/sql-reference/statements/create/row-policy.md
+++ b/docs/ru/sql-reference/statements/create/row-policy.md
@@ -5,7 +5,7 @@ toc_title: "Политика доступа"
# CREATE ROW POLICY {#create-row-policy-statement}
-Создает [фильтры для строк](../../../operations/access-rights.md#row-policy-management), которые пользователь может прочесть из таблицы.
+Создает [политики доступа к строкам](../../../operations/access-rights.md#row-policy-management), т.е. фильтры, которые определяют, какие строки пользователь может читать из таблицы.
Синтаксис:
@@ -13,33 +13,74 @@ toc_title: "Политика доступа"
CREATE [ROW] POLICY [IF NOT EXISTS | OR REPLACE] policy_name1 [ON CLUSTER cluster_name1] ON [db1.]table1
[, policy_name2 [ON CLUSTER cluster_name2] ON [db2.]table2 ...]
[AS {PERMISSIVE | RESTRICTIVE}]
- [FOR SELECT]
- [USING condition]
+ [FOR SELECT] USING condition
[TO {role [,...] | ALL | ALL EXCEPT role [,...]}]
```
-Секция `ON CLUSTER` позволяет создавать фильтры для строк на кластере, см. [Распределенные DDL запросы](../../../sql-reference/distributed-ddl.md).
+## Секция USING {#create-row-policy-using}
-## Секция AS {#create-row-policy-as}
-
-С помощью данной секции можно создать политику разрешения или ограничения.
-
-Политика разрешения предоставляет доступ к строкам. Разрешительные политики, которые применяются к одной таблице, объединяются с помощью логического оператора `OR`. Политики являются разрешительными по умолчанию.
-
-Политика ограничения запрещает доступ к строкам. Ограничительные политики, которые применяются к одной таблице, объединяются логическим оператором `AND`.
-
-Ограничительные политики применяются к строкам, прошедшим фильтр разрешительной политики. Если вы не зададите разрешительные политики, пользователь не сможет обращаться ни к каким строкам из таблицы.
+Секция `USING` указывает условие для фильтрации строк. Пользователь может видеть строку, если это условие, вычисленное для строки, дает ненулевой результат.
## Секция TO {#create-row-policy-to}
-В секции `TO` вы можете перечислить как роли, так и пользователей. Например, `CREATE ROW POLICY ... TO accountant, john@localhost`.
+В секции `TO` перечисляются пользователи и роли, для которых должна действовать политика. Например, `CREATE ROW POLICY ... TO accountant, john@localhost`.
Ключевым словом `ALL` обозначаются все пользователи, включая текущего. Ключевые слова `ALL EXCEPT` позволяют исключить пользователей из списка всех пользователей. Например, `CREATE ROW POLICY ... TO ALL EXCEPT accountant, john@localhost`
+!!! note "Note"
+ Если для таблицы не задано ни одной политики доступа к строкам, то любой пользователь может выполнить команду SELECT и получить все строки таблицы. Если определить хотя бы одну политику для таблицы, до доступ к строкам будет управляться этими политиками, причем для всех пользователей (даже для тех, для кого политики не определялись). Например, следующая политика
+
+ `CREATE ROW POLICY pol1 ON mydb.table1 USING b=1 TO mira, peter`
+
+ запретит пользователям `mira` и `peter` видеть строки с `b != 1`, и еще запретит всем остальным пользователям (например, пользователю `paul`) видеть какие-либо строки вообще из таблицы `mydb.table1`.
+
+ Если это нежелательно, такое поведение можно исправить, определив дополнительную политику:
+
+ `CREATE ROW POLICY pol2 ON mydb.table1 USING 1 TO ALL EXCEPT mira, peter`
+
+## Секция AS {#create-row-policy-as}
+
+Может быть одновременно активно более одной политики для одной и той же таблицы и одного и того же пользователя. Поэтому нам нужен способ комбинировать политики.
+
+По умолчанию политики комбинируются с использованием логического оператора `OR`. Например, политики:
+
+``` sql
+CREATE ROW POLICY pol1 ON mydb.table1 USING b=1 TO mira, peter
+CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 TO peter, antonio
+```
+
+разрешат пользователю с именем `peter` видеть строки, для которых будет верно `b=1` или `c=2`.
+
+Секция `AS` указывает, как политики должны комбинироваться с другими политиками. Политики могут быть или разрешительными (`PERMISSIVE`), или ограничительными (`RESTRICTIVE`). По умолчанию политики создаются разрешительными (`PERMISSIVE`); такие политики комбинируются с использованием логического оператора `OR`.
+
+Ограничительные (`RESTRICTIVE`) политики комбинируются с использованием логического оператора `AND`.
+
+Общая формула выглядит так:
+
+```
+строка_видима = (одна или больше permissive-политик дала ненулевой результат проверки условия) И
+ (все restrictive-политики дали ненулевой результат проверки условия)
+```
+
+Например, политики
+
+``` sql
+CREATE ROW POLICY pol1 ON mydb.table1 USING b=1 TO mira, peter
+CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 AS RESTRICTIVE TO peter, antonio
+```
+
+разрешат пользователю с именем `peter` видеть только те строки, для которых будет одновременно `b=1` и `c=2`.
+
+## Секция ON CLUSTER {#create-row-policy-on-cluster}
+
+Секция `ON CLUSTER` позволяет создавать политики на кластере, см. [Распределенные DDL запросы](../../../sql-reference/distributed-ddl.md).
+
## Примеры
-`CREATE ROW POLICY filter ON mydb.mytable FOR SELECT USING a<1000 TO accountant, john@localhost`
+`CREATE ROW POLICY filter1 ON mydb.mytable USING a<1000 TO accountant, john@localhost`
-`CREATE ROW POLICY filter ON mydb.mytable FOR SELECT USING a<1000 TO ALL EXCEPT mira`
+`CREATE ROW POLICY filter2 ON mydb.mytable USING a<1000 AND b=5 TO ALL EXCEPT mira`
+
+`CREATE ROW POLICY filter3 ON mydb.mytable USING 1 TO admin`
\ No newline at end of file
diff --git a/docs/ru/sql-reference/statements/create/table.md b/docs/ru/sql-reference/statements/create/table.md
index b998435bcd8..1ccd0a600f3 100644
--- a/docs/ru/sql-reference/statements/create/table.md
+++ b/docs/ru/sql-reference/statements/create/table.md
@@ -46,15 +46,32 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name AS table_function()
### Из запроса SELECT {#from-select-query}
``` sql
-CREATE TABLE [IF NOT EXISTS] [db.]table_name ENGINE = engine AS SELECT ...
+CREATE TABLE [IF NOT EXISTS] [db.]table_name[(name1 [type1], name2 [type2], ...)] ENGINE = engine AS SELECT ...
```
-Создаёт таблицу со структурой, как результат запроса `SELECT`, с движком engine, и заполняет её данными из SELECT-а.
+Создаёт таблицу со структурой, как результат запроса `SELECT`, с движком `engine`, и заполняет её данными из `SELECT`. Также вы можете явно задать описание столбцов.
-Во всех случаях, если указано `IF NOT EXISTS`, то запрос не будет возвращать ошибку, если таблица уже существует. В этом случае, запрос будет ничего не делать.
+Если таблица уже существует и указано `IF NOT EXISTS`, то запрос ничего не делает.
После секции `ENGINE` в запросе могут использоваться и другие секции в зависимости от движка. Подробную документацию по созданию таблиц смотрите в описаниях [движков таблиц](../../../engines/table-engines/index.md#table_engines).
+**Пример**
+
+Запрос:
+
+``` sql
+CREATE TABLE t1 (x String) ENGINE = Memory AS SELECT 1;
+SELECT x, toTypeName(x) FROM t1;
+```
+
+Результат:
+
+```text
+┌─x─┬─toTypeName(x)─┐
+│ 1 │ String │
+└───┴───────────────┘
+```
+
## Модификатор NULL или NOT NULL {#null-modifiers}
Модификатор `NULL` или `NOT NULL`, указанный после типа данных в определении столбца, позволяет или не позволяет типу данных быть [Nullable](../../../sql-reference/data-types/nullable.md#data_type-nullable).
@@ -230,7 +247,7 @@ CREATE TABLE codec_example
)
ENGINE = MergeTree()
```
-## Временные таблицы {#vremennye-tablitsy}
+## Временные таблицы {#temporary-tables}
ClickHouse поддерживает временные таблицы со следующими характеристиками:
diff --git a/docs/ru/sql-reference/statements/create/user.md b/docs/ru/sql-reference/statements/create/user.md
index 68277d67052..a487d1ac593 100644
--- a/docs/ru/sql-reference/statements/create/user.md
+++ b/docs/ru/sql-reference/statements/create/user.md
@@ -9,15 +9,17 @@ toc_title: "Пользователь"
Синтаксис:
-```sql
+``` sql
CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1]
[, name2 [ON CLUSTER cluster_name2] ...]
- [IDENTIFIED [WITH {NO_PASSWORD|PLAINTEXT_PASSWORD|SHA256_PASSWORD|SHA256_HASH|DOUBLE_SHA1_PASSWORD|DOUBLE_SHA1_HASH}] BY {'password'|'hash'}]
+ [NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']}]
[HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
[DEFAULT ROLE role [,...]]
- [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...]
+ [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY | WRITABLE] | PROFILE 'profile_name'] [,...]
```
+`ON CLUSTER` позволяет создавать пользователей в кластере, см. [Распределенные DDL](../../../sql-reference/distributed-ddl.md).
+
## Идентификация
Существует несколько способов идентификации пользователя:
@@ -28,6 +30,8 @@ CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1]
- `IDENTIFIED WITH sha256_hash BY 'hash'`
- `IDENTIFIED WITH double_sha1_password BY 'qwerty'`
- `IDENTIFIED WITH double_sha1_hash BY 'hash'`
+- `IDENTIFIED WITH ldap SERVER 'server_name'`
+- `IDENTIFIED WITH kerberos` or `IDENTIFIED WITH kerberos REALM 'realm'`
## Пользовательский хост
diff --git a/docs/ru/sql-reference/statements/grant.md b/docs/ru/sql-reference/statements/grant.md
index 7b2d26902ef..093e6eb3b93 100644
--- a/docs/ru/sql-reference/statements/grant.md
+++ b/docs/ru/sql-reference/statements/grant.md
@@ -93,7 +93,7 @@ GRANT SELECT(x,y) ON db.table TO john WITH GRANT OPTION
- `ALTER ADD CONSTRAINT`
- `ALTER DROP CONSTRAINT`
- `ALTER TTL`
- - `ALTER MATERIALIZE TTL`
+ - `ALTER MATERIALIZE TTL`
- `ALTER SETTINGS`
- `ALTER MOVE PARTITION`
- `ALTER FETCH PARTITION`
@@ -104,9 +104,9 @@ GRANT SELECT(x,y) ON db.table TO john WITH GRANT OPTION
- [CREATE](#grant-create)
- `CREATE DATABASE`
- `CREATE TABLE`
+ - `CREATE TEMPORARY TABLE`
- `CREATE VIEW`
- `CREATE DICTIONARY`
- - `CREATE TEMPORARY TABLE`
- [DROP](#grant-drop)
- `DROP DATABASE`
- `DROP TABLE`
@@ -152,7 +152,7 @@ GRANT SELECT(x,y) ON db.table TO john WITH GRANT OPTION
- `SYSTEM RELOAD`
- `SYSTEM RELOAD CONFIG`
- `SYSTEM RELOAD DICTIONARY`
- - `SYSTEM RELOAD EMBEDDED DICTIONARIES`
+ - `SYSTEM RELOAD EMBEDDED DICTIONARIES`
- `SYSTEM MERGES`
- `SYSTEM TTL MERGES`
- `SYSTEM FETCHES`
@@ -279,7 +279,7 @@ GRANT INSERT(x,y) ON db.table TO john
- `ALTER ADD CONSTRAINT`. Уровень: `TABLE`. Алиасы: `ADD CONSTRAINT`
- `ALTER DROP CONSTRAINT`. Уровень: `TABLE`. Алиасы: `DROP CONSTRAINT`
- `ALTER TTL`. Уровень: `TABLE`. Алиасы: `ALTER MODIFY TTL`, `MODIFY TTL`
- - `ALTER MATERIALIZE TTL`. Уровень: `TABLE`. Алиасы: `MATERIALIZE TTL`
+ - `ALTER MATERIALIZE TTL`. Уровень: `TABLE`. Алиасы: `MATERIALIZE TTL`
- `ALTER SETTINGS`. Уровень: `TABLE`. Алиасы: `ALTER SETTING`, `ALTER MODIFY SETTING`, `MODIFY SETTING`
- `ALTER MOVE PARTITION`. Уровень: `TABLE`. Алиасы: `ALTER MOVE PART`, `MOVE PARTITION`, `MOVE PART`
- `ALTER FETCH PARTITION`. Уровень: `TABLE`. Алиасы: `FETCH PARTITION`
@@ -307,9 +307,9 @@ GRANT INSERT(x,y) ON db.table TO john
- `CREATE`. Уровень: `GROUP`
- `CREATE DATABASE`. Уровень: `DATABASE`
- `CREATE TABLE`. Уровень: `TABLE`
+ - `CREATE TEMPORARY TABLE`. Уровень: `GLOBAL`
- `CREATE VIEW`. Уровень: `VIEW`
- `CREATE DICTIONARY`. Уровень: `DICTIONARY`
- - `CREATE TEMPORARY TABLE`. Уровень: `GLOBAL`
**Дополнительно**
@@ -407,7 +407,7 @@ GRANT INSERT(x,y) ON db.table TO john
- `SYSTEM RELOAD`. Уровень: `GROUP`
- `SYSTEM RELOAD CONFIG`. Уровень: `GLOBAL`. Алиасы: `RELOAD CONFIG`
- `SYSTEM RELOAD DICTIONARY`. Уровень: `GLOBAL`. Алиасы: `SYSTEM RELOAD DICTIONARIES`, `RELOAD DICTIONARY`, `RELOAD DICTIONARIES`
- - `SYSTEM RELOAD EMBEDDED DICTIONARIES`. Уровень: `GLOBAL`. Алиасы: `RELOAD EMBEDDED DICTIONARIES`
+ - `SYSTEM RELOAD EMBEDDED DICTIONARIES`. Уровень: `GLOBAL`. Алиасы: `RELOAD EMBEDDED DICTIONARIES`
- `SYSTEM MERGES`. Уровень: `TABLE`. Алиасы: `SYSTEM STOP MERGES`, `SYSTEM START MERGES`, `STOP MERGES`, `START MERGES`
- `SYSTEM TTL MERGES`. Уровень: `TABLE`. Алиасы: `SYSTEM STOP TTL MERGES`, `SYSTEM START TTL MERGES`, `STOP TTL MERGES`, `START TTL MERGES`
- `SYSTEM FETCHES`. Уровень: `TABLE`. Алиасы: `SYSTEM STOP FETCHES`, `SYSTEM START FETCHES`, `STOP FETCHES`, `START FETCHES`
diff --git a/docs/ru/sql-reference/statements/optimize.md b/docs/ru/sql-reference/statements/optimize.md
index 44101910a6c..e1a9d613537 100644
--- a/docs/ru/sql-reference/statements/optimize.md
+++ b/docs/ru/sql-reference/statements/optimize.md
@@ -5,19 +5,83 @@ toc_title: OPTIMIZE
# OPTIMIZE {#misc_operations-optimize}
-``` sql
-OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION ID 'partition_id'] [FINAL] [DEDUPLICATE]
-```
-
-Запрос пытается запустить внеплановый мёрж кусков данных для таблиц семейства [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md). Другие движки таблиц не поддерживаются.
-
-Если `OPTIMIZE` применяется к таблицам семейства [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md), ClickHouse создаёт задачу на мёрж и ожидает её исполнения на всех узлах (если активирована настройка `replication_alter_partitions_sync`).
-
-- Если `OPTIMIZE` не выполняет мёрж по любой причине, ClickHouse не оповещает об этом клиента. Чтобы включить оповещения, используйте настройку [optimize_throw_if_noop](../../operations/settings/settings.md#setting-optimize_throw_if_noop).
-- Если указать `PARTITION`, то оптимизация выполняется только для указанной партиции. [Как задавать имя партиции в запросах](alter/index.md#alter-how-to-specify-part-expr).
-- Если указать `FINAL`, то оптимизация выполняется даже в том случае, если все данные уже лежат в одном куске. Кроме того, слияние является принудительным, даже если выполняются параллельные слияния.
-- Если указать `DEDUPLICATE`, то произойдет схлопывание полностью одинаковых строк (сравниваются значения во всех колонках), имеет смысл только для движка MergeTree.
+Запрос пытается запустить внеплановое слияние кусков данных для таблиц.
!!! warning "Внимание"
- Запрос `OPTIMIZE` не может устранить причину появления ошибки «Too many parts».
-
+ `OPTIMIZE` не устраняет причину появления ошибки `Too many parts`.
+
+**Синтаксис**
+
+``` sql
+OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION ID 'partition_id'] [FINAL] [DEDUPLICATE [BY expression]]
+```
+
+Может применяться к таблицам семейства [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md), [MaterializedView](../../engines/table-engines/special/materializedview.md) и [Buffer](../../engines/table-engines/special/buffer.md). Другие движки таблиц не поддерживаются.
+
+Если запрос `OPTIMIZE` применяется к таблицам семейства [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md), ClickHouse создаёт задачу на слияние и ожидает её исполнения на всех узлах (если активирована настройка `replication_alter_partitions_sync`).
+
+- По умолчанию, если запросу `OPTIMIZE` не удалось выполнить слияние, то
+ClickHouse не оповещает клиента. Чтобы включить оповещения, используйте настройку [optimize_throw_if_noop](../../operations/settings/settings.md#setting-optimize_throw_if_noop).
+- Если указать `PARTITION`, то оптимизация выполняется только для указанной партиции. [Как задавать имя партиции в запросах](alter/index.md#alter-how-to-specify-part-expr).
+- Если указать `FINAL`, то оптимизация выполняется даже в том случае, если все данные уже лежат в одном куске данных. Кроме того, слияние является принудительным, даже если выполняются параллельные слияния.
+- Если указать `DEDUPLICATE`, то произойдет схлопывание полностью одинаковых строк (сравниваются значения во всех столбцах), имеет смысл только для движка MergeTree.
+
+## Выражение BY {#by-expression}
+
+Чтобы выполнить дедупликацию по произвольному набору столбцов, вы можете явно указать список столбцов или использовать любую комбинацию подстановки [`*`](../../sql-reference/statements/select/index.md#asterisk), выражений [`COLUMNS`](../../sql-reference/statements/select/index.md#columns-expression) и [`EXCEPT`](../../sql-reference/statements/select/index.md#except-modifier).
+
+ Список столбцов для дедупликации должен включать все столбцы, указанные в условиях сортировки (первичный ключ и ключ сортировки), а также в условиях партиционирования (ключ партиционирования).
+
+ !!! note "Примечание"
+ Обратите внимание, что символ подстановки `*` обрабатывается так же, как и в запросах `SELECT`: столбцы `MATERIALIZED` и `ALIAS` не включаются в результат.
+ Если указать пустой список или выражение, которое возвращает пустой список, или дедуплицировать столбец по псевдониму (`ALIAS`), то сервер вернет ошибку.
+
+
+**Примеры**
+
+Рассмотрим таблицу:
+
+``` sql
+CREATE TABLE example (
+ primary_key Int32,
+ secondary_key Int32,
+ value UInt32,
+ partition_key UInt32,
+ materialized_value UInt32 MATERIALIZED 12345,
+ aliased_value UInt32 ALIAS 2,
+ PRIMARY KEY primary_key
+) ENGINE=MergeTree
+PARTITION BY partition_key;
+```
+
+Прежний способ дедупликации, когда учитываются все столбцы. Строка удаляется только в том случае, если все значения во всех столбцах равны соответствующим значениям в предыдущей строке.
+
+``` sql
+OPTIMIZE TABLE example FINAL DEDUPLICATE;
+```
+
+Дедупликация по всем столбцам, кроме `ALIAS` и `MATERIALIZED`: `primary_key`, `secondary_key`, `value`, `partition_key` и `materialized_value`.
+
+
+``` sql
+OPTIMIZE TABLE example FINAL DEDUPLICATE BY *;
+```
+
+Дедупликация по всем столбцам, кроме `ALIAS`, `MATERIALIZED` и `materialized_value`: столбцы `primary_key`, `secondary_key`, `value` и `partition_key`.
+
+
+``` sql
+OPTIMIZE TABLE example FINAL DEDUPLICATE BY * EXCEPT materialized_value;
+```
+
+Дедупликация по столбцам `primary_key`, `secondary_key` и `partition_key`.
+
+``` sql
+OPTIMIZE TABLE example FINAL DEDUPLICATE BY primary_key, secondary_key, partition_key;
+```
+
+Дедупликация по любому столбцу, соответствующему регулярному выражению: столбцам `primary_key`, `secondary_key` и `partition_key`.
+
+``` sql
+OPTIMIZE TABLE example FINAL DEDUPLICATE BY COLUMNS('.*_key');
+```
diff --git a/docs/ru/sql-reference/statements/rename.md b/docs/ru/sql-reference/statements/rename.md
index 104918c1a73..192426dbafa 100644
--- a/docs/ru/sql-reference/statements/rename.md
+++ b/docs/ru/sql-reference/statements/rename.md
@@ -3,8 +3,16 @@ toc_priority: 48
toc_title: RENAME
---
-# RENAME {#misc_operations-rename}
+# RENAME Statement {#misc_operations-rename}
+## RENAME DATABASE {#misc_operations-rename_database}
+Переименование базы данных
+
+```
+RENAME DATABASE atomic_database1 TO atomic_database2 [ON CLUSTER cluster]
+```
+
+## RENAME TABLE {#misc_operations-rename_table}
Переименовывает одну или несколько таблиц.
``` sql
@@ -12,5 +20,3 @@ RENAME TABLE [db11.]name11 TO [db12.]name12, [db21.]name21 TO [db22.]name22, ...
```
Переименовывание таблицы является лёгкой операцией. Если вы указали после `TO` другую базу данных, то таблица будет перенесена в эту базу данных. При этом, директории с базами данных должны быть расположены в одной файловой системе (иначе возвращается ошибка). В случае переименования нескольких таблиц в одном запросе — это неатомарная операция, может выполнится частично, запросы в других сессиях могут получить ошибку `Table ... doesn't exist...`.
-
-
diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md
index ab68033d4f3..f0f9b77b5ba 100644
--- a/docs/ru/sql-reference/statements/system.md
+++ b/docs/ru/sql-reference/statements/system.md
@@ -204,6 +204,7 @@ SYSTEM STOP MOVES [[db.]merge_tree_family_table_name]
ClickHouse может управлять фоновыми процессами связанными c репликацией в таблицах семейства [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replacingmergetree.md).
### STOP FETCHES {#query_language-system-stop-fetches}
+
Позволяет остановить фоновые процессы синхронизации новыми вставленными кусками данных с другими репликами в кластере для таблиц семейства `ReplicatedMergeTree`:
Всегда возвращает `Ok.` вне зависимости от типа таблицы и даже если таблица или база данных не существет.
@@ -212,6 +213,7 @@ SYSTEM STOP FETCHES [[db.]replicated_merge_tree_family_table_name]
```
### START FETCHES {#query_language-system-start-fetches}
+
Позволяет запустить фоновые процессы синхронизации новыми вставленными кусками данных с другими репликами в кластере для таблиц семейства `ReplicatedMergeTree`:
Всегда возвращает `Ok.` вне зависимости от типа таблицы и даже если таблица или база данных не существет.
@@ -220,6 +222,7 @@ SYSTEM START FETCHES [[db.]replicated_merge_tree_family_table_name]
```
### STOP REPLICATED SENDS {#query_language-system-start-replicated-sends}
+
Позволяет остановить фоновые процессы отсылки новых вставленных кусков данных другим репликам в кластере для таблиц семейства `ReplicatedMergeTree`:
``` sql
@@ -227,6 +230,7 @@ SYSTEM STOP REPLICATED SENDS [[db.]replicated_merge_tree_family_table_name]
```
### START REPLICATED SENDS {#query_language-system-start-replicated-sends}
+
Позволяет запустить фоновые процессы отсылки новых вставленных кусков данных другим репликам в кластере для таблиц семейства `ReplicatedMergeTree`:
``` sql
@@ -234,6 +238,7 @@ SYSTEM START REPLICATED SENDS [[db.]replicated_merge_tree_family_table_name]
```
### STOP REPLICATION QUEUES {#query_language-system-stop-replication-queues}
+
Останавливает фоновые процессы разбора заданий из очереди репликации которая хранится в Zookeeper для таблиц семейства `ReplicatedMergeTree`. Возможные типы заданий - merges, fetches, mutation, DDL запросы с ON CLUSTER:
``` sql
@@ -241,6 +246,7 @@ SYSTEM STOP REPLICATION QUEUES [[db.]replicated_merge_tree_family_table_name]
```
### START REPLICATION QUEUES {#query_language-system-start-replication-queues}
+
Запускает фоновые процессы разбора заданий из очереди репликации которая хранится в Zookeeper для таблиц семейства `ReplicatedMergeTree`. Возможные типы заданий - merges, fetches, mutation, DDL запросы с ON CLUSTER:
``` sql
@@ -248,20 +254,24 @@ SYSTEM START REPLICATION QUEUES [[db.]replicated_merge_tree_family_table_name]
```
### SYNC REPLICA {#query_language-system-sync-replica}
+
Ждет когда таблица семейства `ReplicatedMergeTree` будет синхронизирована с другими репликами в кластере, будет работать до достижения `receive_timeout`, если синхронизация для таблицы отключена в настоящий момент времени:
``` sql
SYSTEM SYNC REPLICA [db.]replicated_merge_tree_family_table_name
```
+После выполнения этого запроса таблица `[db.]replicated_merge_tree_family_table_name` синхронизирует команды из общего реплицированного лога в свою собственную очередь репликации. Затем запрос ждет, пока реплика не обработает все синхронизированные команды.
+
### RESTART REPLICA {#query_language-system-restart-replica}
-Реинициализация состояния Zookeeper сессий для таблицы семейства `ReplicatedMergeTree`, сравнивает текущее состояние с тем что хранится в Zookeeper как источник правды и добавляет задачи Zookeeper очередь если необходимо
-Инициализация очереди репликации на основе данных ZooKeeper, происходит так же как при attach table. На короткое время таблица станет недоступной для любых операций.
+
+Реинициализация состояния Zookeeper-сессий для таблицы семейства `ReplicatedMergeTree`. Сравнивает текущее состояние с тем, что хранится в Zookeeper, как источник правды, и добавляет задачи в очередь репликации в Zookeeper, если необходимо.
+Инициализация очереди репликации на основе данных ZooKeeper происходит так же, как при attach table. На короткое время таблица станет недоступной для любых операций.
``` sql
SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name
```
### RESTART REPLICAS {#query_language-system-restart-replicas}
-Реинициализация состояния Zookeeper сессий для всех `ReplicatedMergeTree` таблиц, сравнивает текущее состояние с тем что хранится в Zookeeper как источник правды и добавляет задачи Zookeeper очередь если необходимо
+Реинициализация состояния ZooKeeper-сессий для всех `ReplicatedMergeTree` таблиц. Сравнивает текущее состояние реплики с тем, что хранится в ZooKeeper, как c источником правды, и добавляет задачи в очередь репликации в ZooKeeper, если необходимо.
diff --git a/docs/ru/sql-reference/table-functions/postgresql.md b/docs/ru/sql-reference/table-functions/postgresql.md
index 66637276726..2d8afe28f1e 100644
--- a/docs/ru/sql-reference/table-functions/postgresql.md
+++ b/docs/ru/sql-reference/table-functions/postgresql.md
@@ -65,10 +65,10 @@ postgres=# INSERT INTO test (int_id, str, "float") VALUES (1,'test',2);
INSERT 0 1
postgresql> SELECT * FROM test;
- int_id | int_nullable | float | str | float_nullable
---------+--------------+-------+------+----------------
- 1 | | 2 | test |
-(1 row)
+ int_id | int_nullable | float | str | float_nullable
+ --------+--------------+-------+------+----------------
+ 1 | | 2 | test |
+ (1 row)
```
Получение данных в ClickHouse:
diff --git a/docs/ru/sql-reference/table-functions/s3.md b/docs/ru/sql-reference/table-functions/s3.md
index 1d3fc8cfdb7..e062e59c67c 100644
--- a/docs/ru/sql-reference/table-functions/s3.md
+++ b/docs/ru/sql-reference/table-functions/s3.md
@@ -18,7 +18,7 @@ s3(path, [aws_access_key_id, aws_secret_access_key,] format, structure, [compres
- `path` — URL-адрес бакета с указанием пути к файлу. Поддерживает следующие подстановочные знаки в режиме "только чтение": `*, ?, {abc,def} и {N..M}` где `N, M` — числа, `'abc', 'def'` — строки. Подробнее смотри [здесь](../../engines/table-engines/integrations/s3.md#wildcards-in-path).
- `format` — [формат](../../interfaces/formats.md#formats) файла.
- `structure` — cтруктура таблицы. Формат `'column1_name column1_type, column2_name column2_type, ...'`.
-- `compression` — автоматически обнаруживает сжатие по расширению файла. Возможные значения: none, gzip/gz, brotli/br, xz/LZMA, zstd/zst. Необязательный параметр.
+- `compression` — автоматически обнаруживает сжатие по расширению файла. Возможные значения: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. Необязательный параметр.
**Возвращаемые значения**
diff --git a/docs/tools/single_page.py b/docs/tools/single_page.py
index b88df5a03cb..a1e650d3ad3 100644
--- a/docs/tools/single_page.py
+++ b/docs/tools/single_page.py
@@ -109,7 +109,8 @@ def build_single_page_version(lang, args, nav, cfg):
extra['single_page'] = True
extra['is_amp'] = False
- with open(os.path.join(args.docs_dir, lang, 'single.md'), 'w') as single_md:
+ single_md_path = os.path.join(args.docs_dir, lang, 'single.md')
+ with open(single_md_path, 'w') as single_md:
concatenate(lang, args.docs_dir, single_md, nav)
with util.temp_dir() as site_temp:
@@ -221,3 +222,7 @@ def build_single_page_version(lang, args, nav, cfg):
subprocess.check_call(' '.join(create_pdf_command), shell=True)
logging.info(f'Finished building single page version for {lang}')
+
+ if os.path.exists(single_md_path):
+ os.unlink(single_md_path)
+
\ No newline at end of file
diff --git a/docs/zh/development/build.md b/docs/zh/development/build.md
index 1aa5c1c97b7..01e0740bfa4 100644
--- a/docs/zh/development/build.md
+++ b/docs/zh/development/build.md
@@ -35,28 +35,12 @@ sudo apt-get install git cmake ninja-build
或cmake3而不是旧系统上的cmake。
或者在早期版本的系统中用 cmake3 替代 cmake
-## 安装 GCC 10 {#an-zhuang-gcc-10}
+## 安装 Clang
-有几种方法可以做到这一点。
+On Ubuntu/Debian you can use the automatic installation script (check [official webpage](https://apt.llvm.org/))
-### 安装 PPA 包 {#an-zhuang-ppa-bao}
-
-``` bash
-sudo apt-get install software-properties-common
-sudo apt-add-repository ppa:ubuntu-toolchain-r/test
-sudo apt-get update
-sudo apt-get install gcc-10 g++-10
-```
-
-### 源码安装 gcc {#yuan-ma-an-zhuang-gcc}
-
-请查看 [utils/ci/build-gcc-from-sources.sh](https://github.com/ClickHouse/ClickHouse/blob/master/utils/ci/build-gcc-from-sources.sh)
-
-## 使用 GCC 10 来编译 {#shi-yong-gcc-10-lai-bian-yi}
-
-``` bash
-export CC=gcc-10
-export CXX=g++-10
+```bash
+sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
```
## 拉取 ClickHouse 源码 {#la-qu-clickhouse-yuan-ma-1}
diff --git a/docs/zh/development/developer-instruction.md b/docs/zh/development/developer-instruction.md
index 53aab5dc086..04950c11521 100644
--- a/docs/zh/development/developer-instruction.md
+++ b/docs/zh/development/developer-instruction.md
@@ -123,17 +123,13 @@ ClickHouse使用多个外部库进行构建。大多数外部库不需要单独
# C++ 编译器 {#c-bian-yi-qi}
-GCC编译器从版本9开始,以及Clang版本\>=8都可支持构建ClickHouse。
+We support clang starting from version 11.
-Yandex官方当前使用GCC构建ClickHouse,因为它生成的机器代码性能较好(根据测评,最多可以相差几个百分点)。Clang通常可以更加便捷的开发。我们的持续集成(CI)平台会运行大约十二种构建组合的检查。
+On Ubuntu/Debian you can use the automatic installation script (check [official webpage](https://apt.llvm.org/))
-在Ubuntu上安装GCC,请执行:`sudo apt install gcc g++`
-
-请使用`gcc --version`查看gcc的版本。如果gcc版本低于9,请参考此处的指示:https://clickhouse.tech/docs/zh/development/build/#an-zhuang-gcc-10 。
-
-在Mac OS X上安装GCC,请执行:`brew install gcc`
-
-如果您决定使用Clang,还可以同时安装 `libc++`以及`lld`,前提是您也熟悉它们。此外,也推荐使用`ccache`。
+```bash
+sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
+```
# 构建的过程 {#gou-jian-de-guo-cheng}
@@ -146,7 +142,7 @@ Yandex官方当前使用GCC构建ClickHouse,因为它生成的机器代码性
在`build`目录下,通过运行CMake配置构建。 在第一次运行之前,请定义用于指定编译器的环境变量(本示例中为gcc 9 编译器)。
- export CC=gcc-10 CXX=g++-10
+ export CC=clang CXX=clang++
cmake ..
`CC`变量指代C的编译器(C Compiler的缩写),而`CXX`变量指代要使用哪个C++编译器进行编译。
diff --git a/docs/zh/development/style.md b/docs/zh/development/style.md
index c8e883920dd..bb9bfde7b9b 100644
--- a/docs/zh/development/style.md
+++ b/docs/zh/development/style.md
@@ -696,7 +696,7 @@ auto s = std::string{"Hello"};
**2.** 语言: C++20.
-**3.** 编译器: `gcc`。 此时(2020年08月),代码使用9.3版编译。(它也可以使用`clang 8` 编译)
+**3.** 编译器: `clang`。 此时(2021年03月),代码使用11版编译。(它也可以使用`gcc` 编译 but it is not suitable for production)
使用标准库 (`libc++`)。
diff --git a/docs/zh/sql-reference/functions/other-functions.md b/docs/zh/sql-reference/functions/other-functions.md
index b17a5e89332..c58c4bd1510 100644
--- a/docs/zh/sql-reference/functions/other-functions.md
+++ b/docs/zh/sql-reference/functions/other-functions.md
@@ -477,6 +477,103 @@ FROM
1 rows in set. Elapsed: 0.002 sec.
+
+## indexHint {#indexhint}
+输出符合索引选择范围内的所有数据,同时不实用参数中的表达式进行过滤。
+
+传递给函数的表达式参数将不会被计算,但ClickHouse使用参数中的表达式进行索引过滤。
+
+**返回值**
+
+- 1。
+
+**示例**
+
+这是一个包含[ontime](../../getting-started/example-datasets/ontime.md)测试数据集的测试表。
+
+```
+SELECT count() FROM ontime
+
+┌─count()─┐
+│ 4276457 │
+└─────────┘
+```
+
+该表使用`(FlightDate, (Year, FlightDate))`作为索引。
+
+对该表进行如下的查询:
+
+```
+:) SELECT FlightDate AS k, count() FROM ontime GROUP BY k ORDER BY k
+
+SELECT
+ FlightDate AS k,
+ count()
+FROM ontime
+GROUP BY k
+ORDER BY k ASC
+
+┌──────────k─┬─count()─┐
+│ 2017-01-01 │ 13970 │
+│ 2017-01-02 │ 15882 │
+........................
+│ 2017-09-28 │ 16411 │
+│ 2017-09-29 │ 16384 │
+│ 2017-09-30 │ 12520 │
+└────────────┴─────────┘
+
+273 rows in set. Elapsed: 0.072 sec. Processed 4.28 million rows, 8.55 MB (59.00 million rows/s., 118.01 MB/s.)
+```
+
+在这个查询中,由于没有使用索引,所以ClickHouse将处理整个表的所有数据(`Processed 4.28 million rows`)。使用下面的查询尝试使用索引进行查询:
+
+```
+:) SELECT FlightDate AS k, count() FROM ontime WHERE k = '2017-09-15' GROUP BY k ORDER BY k
+
+SELECT
+ FlightDate AS k,
+ count()
+FROM ontime
+WHERE k = '2017-09-15'
+GROUP BY k
+ORDER BY k ASC
+
+┌──────────k─┬─count()─┐
+│ 2017-09-15 │ 16428 │
+└────────────┴─────────┘
+
+1 rows in set. Elapsed: 0.014 sec. Processed 32.74 thousand rows, 65.49 KB (2.31 million rows/s., 4.63 MB/s.)
+```
+
+在最后一行的显示中,通过索引ClickHouse处理的行数明显减少(`Processed 32.74 thousand rows`)。
+
+现在将表达式`k = '2017-09-15'`传递给`indexHint`函数:
+
+```
+:) SELECT FlightDate AS k, count() FROM ontime WHERE indexHint(k = '2017-09-15') GROUP BY k ORDER BY k
+
+SELECT
+ FlightDate AS k,
+ count()
+FROM ontime
+WHERE indexHint(k = '2017-09-15')
+GROUP BY k
+ORDER BY k ASC
+
+┌──────────k─┬─count()─┐
+│ 2017-09-14 │ 7071 │
+│ 2017-09-15 │ 16428 │
+│ 2017-09-16 │ 1077 │
+│ 2017-09-30 │ 8167 │
+└────────────┴─────────┘
+
+4 rows in set. Elapsed: 0.004 sec. Processed 32.74 thousand rows, 65.49 KB (8.97 million rows/s., 17.94 MB/s.)
+```
+
+对于这个请求,根据ClickHouse显示ClickHouse与上一次相同的方式应用了索引(`Processed 32.74 thousand rows`)。但是,最终返回的结果集中并没有根据`k = '2017-09-15'`表达式进行过滤结果。
+
+由于ClickHouse中使用稀疏索引,因此在读取范围时(本示例中为相邻日期),"额外"的数据将包含在索引结果中。使用`indexHint`函数可以查看到它们。
+
## 复制 {#replicate}
使用单个值填充一个数组。
diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt
index c3600e5812a..ad3ff84d8bf 100644
--- a/programs/CMakeLists.txt
+++ b/programs/CMakeLists.txt
@@ -33,7 +33,14 @@ option (ENABLE_CLICKHOUSE_OBFUSCATOR "Table data obfuscator (convert real data t
${ENABLE_CLICKHOUSE_ALL})
# https://clickhouse.tech/docs/en/operations/utilities/odbc-bridge/
-option (ENABLE_CLICKHOUSE_ODBC_BRIDGE "HTTP-server working like a proxy to ODBC driver"
+if (ENABLE_ODBC)
+ option (ENABLE_CLICKHOUSE_ODBC_BRIDGE "HTTP-server working like a proxy to ODBC driver"
+ ${ENABLE_CLICKHOUSE_ALL})
+else ()
+ option (ENABLE_CLICKHOUSE_ODBC_BRIDGE "HTTP-server working like a proxy to ODBC driver" OFF)
+endif ()
+
+option (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE "HTTP-server working like a proxy to Library dictionary source"
${ENABLE_CLICKHOUSE_ALL})
# https://presentations.clickhouse.tech/matemarketing_2020/
@@ -109,6 +116,12 @@ else()
message(STATUS "ODBC bridge mode: OFF")
endif()
+if (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE)
+ message(STATUS "Library bridge mode: ON")
+else()
+ message(STATUS "Library bridge mode: OFF")
+endif()
+
if (ENABLE_CLICKHOUSE_INSTALL)
message(STATUS "ClickHouse install: ON")
else()
@@ -194,6 +207,10 @@ if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
add_subdirectory (odbc-bridge)
endif ()
+if (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE)
+ add_subdirectory (library-bridge)
+endif ()
+
if (CLICKHOUSE_ONE_SHARED)
add_library(clickhouse-lib SHARED ${CLICKHOUSE_SERVER_SOURCES} ${CLICKHOUSE_CLIENT_SOURCES} ${CLICKHOUSE_LOCAL_SOURCES} ${CLICKHOUSE_BENCHMARK_SOURCES} ${CLICKHOUSE_COPIER_SOURCES} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES} ${CLICKHOUSE_COMPRESSOR_SOURCES} ${CLICKHOUSE_FORMAT_SOURCES} ${CLICKHOUSE_OBFUSCATOR_SOURCES} ${CLICKHOUSE_GIT_IMPORT_SOURCES} ${CLICKHOUSE_ODBC_BRIDGE_SOURCES})
target_link_libraries(clickhouse-lib ${CLICKHOUSE_SERVER_LINK} ${CLICKHOUSE_CLIENT_LINK} ${CLICKHOUSE_LOCAL_LINK} ${CLICKHOUSE_BENCHMARK_LINK} ${CLICKHOUSE_COPIER_LINK} ${CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK} ${CLICKHOUSE_COMPRESSOR_LINK} ${CLICKHOUSE_FORMAT_LINK} ${CLICKHOUSE_OBFUSCATOR_LINK} ${CLICKHOUSE_GIT_IMPORT_LINK} ${CLICKHOUSE_ODBC_BRIDGE_LINK})
@@ -209,6 +226,10 @@ if (CLICKHOUSE_SPLIT_BINARY)
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-odbc-bridge)
endif ()
+ if (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE)
+ list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-library-bridge)
+ endif ()
+
set_target_properties(${CLICKHOUSE_ALL_TARGETS} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_ALL_TARGETS})
diff --git a/programs/benchmark/Benchmark.cpp b/programs/benchmark/Benchmark.cpp
index a0e2ea155ba..1d2b579db3a 100644
--- a/programs/benchmark/Benchmark.cpp
+++ b/programs/benchmark/Benchmark.cpp
@@ -95,8 +95,8 @@ public:
comparison_info_total.emplace_back(std::make_shared());
}
- global_context.makeGlobalContext();
- global_context.setSettings(settings);
+ global_context->makeGlobalContext();
+ global_context->setSettings(settings);
std::cerr << std::fixed << std::setprecision(3);
@@ -159,7 +159,7 @@ private:
bool print_stacktrace;
const Settings & settings;
SharedContextHolder shared_context;
- Context global_context;
+ ContextPtr global_context;
QueryProcessingStage::Enum query_processing_stage;
/// Don't execute new queries after timelimit or SIGINT or exception
diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp
index ca9976ac4a8..1aec3677b41 100644
--- a/programs/client/Client.cpp
+++ b/programs/client/Client.cpp
@@ -21,7 +21,7 @@
#include
#include
#include
-#include
+#include
#include
#include
#include
@@ -191,7 +191,7 @@ private:
bool has_vertical_output_suffix = false; /// Is \G present at the end of the query string?
SharedContextHolder shared_context = Context::createShared();
- Context context = Context::createGlobal(shared_context.get());
+ ContextPtr context = Context::createGlobal(shared_context.get());
/// Buffer that reads from stdin in batch mode.
ReadBufferFromFileDescriptor std_in {STDIN_FILENO};
@@ -274,20 +274,20 @@ private:
configReadClient(config(), home_path);
- context.setApplicationType(Context::ApplicationType::CLIENT);
- context.setQueryParameters(query_parameters);
+ context->setApplicationType(Context::ApplicationType::CLIENT);
+ context->setQueryParameters(query_parameters);
/// settings and limits could be specified in config file, but passed settings has higher priority
- for (const auto & setting : context.getSettingsRef().allUnchanged())
+ for (const auto & setting : context->getSettingsRef().allUnchanged())
{
const auto & name = setting.getName();
if (config().has(name))
- context.setSetting(name, config().getString(name));
+ context->setSetting(name, config().getString(name));
}
/// Set path for format schema files
if (config().has("format_schema_path"))
- context.setFormatSchemaPath(Poco::Path(config().getString("format_schema_path")).toString());
+ context->setFormatSchemaPath(Poco::Path(config().getString("format_schema_path")).toString());
/// Initialize query_id_formats if any
if (config().has("query_id_formats"))
@@ -527,7 +527,10 @@ private:
std::cerr << std::fixed << std::setprecision(3);
if (is_interactive)
+ {
+ clearTerminal();
showClientVersion();
+ }
is_default_format = !config().has("vertical") && !config().has("format");
if (config().has("vertical"))
@@ -535,15 +538,15 @@ private:
else
format = config().getString("format", is_interactive ? "PrettyCompact" : "TabSeparated");
- format_max_block_size = config().getInt("format_max_block_size", context.getSettingsRef().max_block_size);
+ format_max_block_size = config().getInt("format_max_block_size", context->getSettingsRef().max_block_size);
insert_format = "Values";
/// Setting value from cmd arg overrides one from config
- if (context.getSettingsRef().max_insert_block_size.changed)
- insert_format_max_block_size = context.getSettingsRef().max_insert_block_size;
+ if (context->getSettingsRef().max_insert_block_size.changed)
+ insert_format_max_block_size = context->getSettingsRef().max_insert_block_size;
else
- insert_format_max_block_size = config().getInt("insert_format_max_block_size", context.getSettingsRef().max_insert_block_size);
+ insert_format_max_block_size = config().getInt("insert_format_max_block_size", context->getSettingsRef().max_insert_block_size);
if (!is_interactive)
{
@@ -552,7 +555,7 @@ private:
ignore_error = config().getBool("ignore-error", false);
}
- ClientInfo & client_info = context.getClientInfo();
+ ClientInfo & client_info = context->getClientInfo();
client_info.setInitialQuery();
client_info.quota_key = config().getString("quota_key", "");
@@ -560,7 +563,7 @@ private:
/// Initialize DateLUT here to avoid counting time spent here as query execution time.
const auto local_tz = DateLUT::instance().getTimeZone();
- if (!context.getSettingsRef().use_client_time_zone)
+ if (!context->getSettingsRef().use_client_time_zone)
{
const auto & time_zone = connection->getServerTimezone(connection_parameters.timeouts);
if (!time_zone.empty())
@@ -735,7 +738,7 @@ private:
{
auto query_id = config().getString("query_id", "");
if (!query_id.empty())
- context.setCurrentQueryId(query_id);
+ context->setCurrentQueryId(query_id);
nonInteractive();
@@ -1035,7 +1038,7 @@ private:
{
Tokens tokens(this_query_begin, all_queries_end);
IParser::Pos token_iterator(tokens,
- context.getSettingsRef().max_parser_depth);
+ context->getSettingsRef().max_parser_depth);
if (!token_iterator.isValid())
{
break;
@@ -1084,7 +1087,7 @@ private:
if (ignore_error)
{
Tokens tokens(this_query_begin, all_queries_end);
- IParser::Pos token_iterator(tokens, context.getSettingsRef().max_parser_depth);
+ IParser::Pos token_iterator(tokens, context->getSettingsRef().max_parser_depth);
while (token_iterator->type != TokenType::Semicolon && token_iterator.isValid())
++token_iterator;
this_query_begin = token_iterator->end;
@@ -1130,7 +1133,7 @@ private:
// beneficial so that we see proper trailing comments in "echo" and
// server log.
adjustQueryEnd(this_query_end, all_queries_end,
- context.getSettingsRef().max_parser_depth);
+ context->getSettingsRef().max_parser_depth);
// full_query is the query + inline INSERT data + trailing comments
// (the latter is our best guess for now).
@@ -1170,7 +1173,7 @@ private:
{
this_query_end = insert_ast->end;
adjustQueryEnd(this_query_end, all_queries_end,
- context.getSettingsRef().max_parser_depth);
+ context->getSettingsRef().max_parser_depth);
}
// Now we know for sure where the query ends.
@@ -1287,7 +1290,7 @@ private:
// Prints changed settings to stderr. Useful for debugging fuzzing failures.
void printChangedSettings() const
{
- const auto & changes = context.getSettingsRef().changes();
+ const auto & changes = context->getSettingsRef().changes();
if (!changes.empty())
{
fmt::print(stderr, "Changed settings: ");
@@ -1587,11 +1590,11 @@ private:
if (is_interactive)
{
// Generate a new query_id
- context.setCurrentQueryId("");
+ context->setCurrentQueryId("");
for (const auto & query_id_format : query_id_formats)
{
writeString(query_id_format.first, std_out);
- writeString(fmt::format(query_id_format.second, fmt::arg("query_id", context.getCurrentQueryId())), std_out);
+ writeString(fmt::format(query_id_format.second, fmt::arg("query_id", context->getCurrentQueryId())), std_out);
writeChar('\n', std_out);
std_out.next();
}
@@ -1607,12 +1610,12 @@ private:
{
/// Temporarily apply query settings to context.
std::optional old_settings;
- SCOPE_EXIT({ if (old_settings) context.setSettings(*old_settings); });
+ SCOPE_EXIT_SAFE({ if (old_settings) context->setSettings(*old_settings); });
auto apply_query_settings = [&](const IAST & settings_ast)
{
if (!old_settings)
- old_settings.emplace(context.getSettingsRef());
- context.applySettingsChanges(settings_ast.as()->changes);
+ old_settings.emplace(context->getSettingsRef());
+ context->applySettingsChanges(settings_ast.as()->changes);
};
const auto * insert = parsed_query->as();
if (insert && insert->settings_ast)
@@ -1650,7 +1653,7 @@ private:
if (change.name == "profile")
current_profile = change.value.safeGet();
else
- context.applySettingChange(change);
+ context->applySettingChange(change);
}
}
@@ -1722,10 +1725,10 @@ private:
connection->sendQuery(
connection_parameters.timeouts,
query_to_send,
- context.getCurrentQueryId(),
+ context->getCurrentQueryId(),
query_processing_stage,
- &context.getSettingsRef(),
- &context.getClientInfo(),
+ &context->getSettingsRef(),
+ &context->getClientInfo(),
true);
sendExternalTables();
@@ -1763,10 +1766,10 @@ private:
connection->sendQuery(
connection_parameters.timeouts,
query_to_send,
- context.getCurrentQueryId(),
+ context->getCurrentQueryId(),
query_processing_stage,
- &context.getSettingsRef(),
- &context.getClientInfo(),
+ &context->getSettingsRef(),
+ &context->getClientInfo(),
true);
sendExternalTables();
@@ -1789,7 +1792,7 @@ private:
ParserQuery parser(end);
ASTPtr res;
- const auto & settings = context.getSettingsRef();
+ const auto & settings = context->getSettingsRef();
size_t max_length = 0;
if (!allow_multi_statements)
max_length = settings.max_query_size;
@@ -1877,7 +1880,7 @@ private:
current_format = insert->format;
}
- BlockInputStreamPtr block_input = context.getInputFormat(
+ BlockInputStreamPtr block_input = context->getInputFormat(
current_format, buf, sample, insert_format_max_block_size);
if (columns_description.hasDefaults())
@@ -2201,9 +2204,9 @@ private:
/// It is not clear how to write progress with parallel formatting. It may increase code complexity significantly.
if (!need_render_progress)
- block_out_stream = context.getOutputStreamParallelIfPossible(current_format, *out_buf, block);
+ block_out_stream = context->getOutputStreamParallelIfPossible(current_format, *out_buf, block);
else
- block_out_stream = context.getOutputStream(current_format, *out_buf, block);
+ block_out_stream = context->getOutputStream(current_format, *out_buf, block);
block_out_stream->writePrefix();
}
@@ -2467,6 +2470,17 @@ private:
std::cout << DBMS_NAME << " client version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl;
}
+ static void clearTerminal()
+ {
+ /// Clear from cursor until end of screen.
+ /// It is needed if garbage is left in terminal.
+ /// Show cursor. It can be left hidden by invocation of previous programs.
+ /// A test for this feature: perl -e 'print "x"x100000'; echo -ne '\033[0;0H\033[?25l'; clickhouse-client
+ std::cout <<
+ "\033[0J"
+ "\033[?25h";
+ }
+
public:
void init(int argc, char ** argv)
{
@@ -2696,12 +2710,12 @@ public:
}
}
- context.makeGlobalContext();
- context.setSettings(cmd_settings);
+ context->makeGlobalContext();
+ context->setSettings(cmd_settings);
/// Copy settings-related program options to config.
/// TODO: Is this code necessary?
- for (const auto & setting : context.getSettingsRef().all())
+ for (const auto & setting : context->getSettingsRef().all())
{
const auto & name = setting.getName();
if (options.count(name))
@@ -2793,7 +2807,7 @@ public:
{
std::string traceparent = options["opentelemetry-traceparent"].as();
std::string error;
- if (!context.getClientInfo().client_trace_context.parseTraceparentHeader(
+ if (!context->getClientInfo().client_trace_context.parseTraceparentHeader(
traceparent, error))
{
throw Exception(ErrorCodes::BAD_ARGUMENTS,
@@ -2804,7 +2818,7 @@ public:
if (options.count("opentelemetry-tracestate"))
{
- context.getClientInfo().client_trace_context.tracestate =
+ context->getClientInfo().client_trace_context.tracestate =
options["opentelemetry-tracestate"].as();
}
diff --git a/programs/client/ConnectionParameters.cpp b/programs/client/ConnectionParameters.cpp
index 19734dd5ffa..6faf43759df 100644
--- a/programs/client/ConnectionParameters.cpp
+++ b/programs/client/ConnectionParameters.cpp
@@ -7,6 +7,8 @@
#include
#include
#include
+#include
+#include
#include
#include
@@ -60,7 +62,9 @@ ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfigurati
#endif
}
- compression = config.getBool("compression", true) ? Protocol::Compression::Enable : Protocol::Compression::Disable;
+ /// By default compression is disabled if address looks like localhost.
+ compression = config.getBool("compression", !isLocalAddress(DNSResolver::instance().resolveHost(host)))
+ ? Protocol::Compression::Enable : Protocol::Compression::Disable;
timeouts = ConnectionTimeouts(
Poco::Timespan(config.getInt("connect_timeout", DBMS_DEFAULT_CONNECT_TIMEOUT_SEC), 0),
diff --git a/programs/client/QueryFuzzer.cpp b/programs/client/QueryFuzzer.cpp
index 0c8dc0731f9..6243e2c82ec 100644
--- a/programs/client/QueryFuzzer.cpp
+++ b/programs/client/QueryFuzzer.cpp
@@ -37,34 +37,33 @@ namespace ErrorCodes
Field QueryFuzzer::getRandomField(int type)
{
+ static constexpr Int64 bad_int64_values[]
+ = {-2, -1, 0, 1, 2, 3, 7, 10, 100, 255, 256, 257, 1023, 1024,
+ 1025, 65535, 65536, 65537, 1024 * 1024 - 1, 1024 * 1024,
+ 1024 * 1024 + 1, INT_MIN - 1ll, INT_MIN, INT_MIN + 1,
+ INT_MAX - 1, INT_MAX, INT_MAX + 1ll, INT64_MIN, INT64_MIN + 1,
+ INT64_MAX - 1, INT64_MAX};
switch (type)
{
case 0:
{
- static constexpr Int64 values[]
- = {-2, -1, 0, 1, 2, 3, 7, 10, 100, 255, 256, 257, 1023, 1024,
- 1025, 65535, 65536, 65537, 1024 * 1024 - 1, 1024 * 1024,
- 1024 * 1024 + 1, INT64_MIN, INT64_MAX};
- return values[fuzz_rand() % (sizeof(values) / sizeof(*values))];
+ return bad_int64_values[fuzz_rand() % (sizeof(bad_int64_values)
+ / sizeof(*bad_int64_values))];
}
case 1:
{
static constexpr float values[]
- = {NAN, INFINITY, -INFINITY, 0., 0.0001, 0.5, 0.9999,
- 1., 1.0001, 2., 10.0001, 100.0001, 1000.0001};
- return values[fuzz_rand() % (sizeof(values) / sizeof(*values))];
+ = {NAN, INFINITY, -INFINITY, 0., -0., 0.0001, 0.5, 0.9999,
+ 1., 1.0001, 2., 10.0001, 100.0001, 1000.0001, 1e10, 1e20,
+ FLT_MIN, FLT_MIN + FLT_EPSILON, FLT_MAX, FLT_MAX + FLT_EPSILON}; return values[fuzz_rand() % (sizeof(values) / sizeof(*values))];
}
case 2:
{
- static constexpr Int64 values[]
- = {-2, -1, 0, 1, 2, 3, 7, 10, 100, 255, 256, 257, 1023, 1024,
- 1025, 65535, 65536, 65537, 1024 * 1024 - 1, 1024 * 1024,
- 1024 * 1024 + 1, INT64_MIN, INT64_MAX};
static constexpr UInt64 scales[] = {0, 1, 2, 10};
return DecimalField(
- values[fuzz_rand() % (sizeof(values) / sizeof(*values))],
- scales[fuzz_rand() % (sizeof(scales) / sizeof(*scales))]
- );
+ bad_int64_values[fuzz_rand() % (sizeof(bad_int64_values)
+ / sizeof(*bad_int64_values))],
+ scales[fuzz_rand() % (sizeof(scales) / sizeof(*scales))]);
}
default:
assert(false);
diff --git a/programs/client/Suggest.cpp b/programs/client/Suggest.cpp
index dfa7048349e..8d4c0fdbd5a 100644
--- a/programs/client/Suggest.cpp
+++ b/programs/client/Suggest.cpp
@@ -108,14 +108,6 @@ void Suggest::loadImpl(Connection & connection, const ConnectionTimeouts & timeo
" UNION ALL "
"SELECT cluster FROM system.clusters"
" UNION ALL "
- "SELECT name FROM system.errors"
- " UNION ALL "
- "SELECT event FROM system.events"
- " UNION ALL "
- "SELECT metric FROM system.asynchronous_metrics"
- " UNION ALL "
- "SELECT metric FROM system.metrics"
- " UNION ALL "
"SELECT macro FROM system.macros"
" UNION ALL "
"SELECT policy_name FROM system.storage_policies"
@@ -139,17 +131,12 @@ void Suggest::loadImpl(Connection & connection, const ConnectionTimeouts & timeo
query << ") WHERE notEmpty(res)";
- Settings settings;
- /// To show all rows from:
- /// - system.errors
- /// - system.events
- settings.system_events_show_zero_values = true;
- fetch(connection, timeouts, query.str(), settings);
+ fetch(connection, timeouts, query.str());
}
-void Suggest::fetch(Connection & connection, const ConnectionTimeouts & timeouts, const std::string & query, Settings & settings)
+void Suggest::fetch(Connection & connection, const ConnectionTimeouts & timeouts, const std::string & query)
{
- connection.sendQuery(timeouts, query, "" /* query_id */, QueryProcessingStage::Complete, &settings);
+ connection.sendQuery(timeouts, query, "" /* query_id */, QueryProcessingStage::Complete);
while (true)
{
diff --git a/programs/client/Suggest.h b/programs/client/Suggest.h
index 0049bc08ebf..03332088cbe 100644
--- a/programs/client/Suggest.h
+++ b/programs/client/Suggest.h
@@ -33,7 +33,7 @@ public:
private:
void loadImpl(Connection & connection, const ConnectionTimeouts & timeouts, size_t suggestion_limit);
- void fetch(Connection & connection, const ConnectionTimeouts & timeouts, const std::string & query, Settings & settings);
+ void fetch(Connection & connection, const ConnectionTimeouts & timeouts, const std::string & query);
void fillWordsFromBlock(const Block & block);
/// Words are fetched asynchronously.
diff --git a/programs/config_tools.h.in b/programs/config_tools.h.in
index 7cb5a6d883a..abe9ef8c562 100644
--- a/programs/config_tools.h.in
+++ b/programs/config_tools.h.in
@@ -15,3 +15,4 @@
#cmakedefine01 ENABLE_CLICKHOUSE_GIT_IMPORT
#cmakedefine01 ENABLE_CLICKHOUSE_INSTALL
#cmakedefine01 ENABLE_CLICKHOUSE_ODBC_BRIDGE
+#cmakedefine01 ENABLE_CLICKHOUSE_LIBRARY_BRIDGE
diff --git a/programs/copier/ClusterCopier.cpp b/programs/copier/ClusterCopier.cpp
index bede40d65f5..aa9b359993e 100644
--- a/programs/copier/ClusterCopier.cpp
+++ b/programs/copier/ClusterCopier.cpp
@@ -22,7 +22,7 @@ namespace ErrorCodes
void ClusterCopier::init()
{
- auto zookeeper = context.getZooKeeper();
+ auto zookeeper = getContext()->getZooKeeper();
task_description_watch_callback = [this] (const Coordination::WatchResponse & response)
{
@@ -39,14 +39,14 @@ void ClusterCopier::init()
task_cluster_initial_config = task_cluster_current_config;
task_cluster->loadTasks(*task_cluster_initial_config);
- context.setClustersConfig(task_cluster_initial_config, task_cluster->clusters_prefix);
+ getContext()->setClustersConfig(task_cluster_initial_config, task_cluster->clusters_prefix);
/// Set up shards and their priority
task_cluster->random_engine.seed(task_cluster->random_device());
for (auto & task_table : task_cluster->table_tasks)
{
- task_table.cluster_pull = context.getCluster(task_table.cluster_pull_name);
- task_table.cluster_push = context.getCluster(task_table.cluster_push_name);
+ task_table.cluster_pull = getContext()->getCluster(task_table.cluster_pull_name);
+ task_table.cluster_push = getContext()->getCluster(task_table.cluster_push_name);
task_table.initShards(task_cluster->random_engine);
}
@@ -206,7 +206,7 @@ void ClusterCopier::uploadTaskDescription(const std::string & task_path, const s
if (task_config_str.empty())
return;
- auto zookeeper = context.getZooKeeper();
+ auto zookeeper = getContext()->getZooKeeper();
zookeeper->createAncestors(local_task_description_path);
auto code = zookeeper->tryCreate(local_task_description_path, task_config_str, zkutil::CreateMode::Persistent);
@@ -219,7 +219,7 @@ void ClusterCopier::uploadTaskDescription(const std::string & task_path, const s
void ClusterCopier::reloadTaskDescription()
{
- auto zookeeper = context.getZooKeeper();
+ auto zookeeper = getContext()->getZooKeeper();
task_description_watch_zookeeper = zookeeper;
String task_config_str;
@@ -235,7 +235,7 @@ void ClusterCopier::reloadTaskDescription()
/// Setup settings
task_cluster->reloadSettings(*config);
- context.setSettings(task_cluster->settings_common);
+ getContext()->setSettings(task_cluster->settings_common);
task_cluster_current_config = config;
task_description_current_stat = stat;
@@ -440,7 +440,7 @@ bool ClusterCopier::checkPartitionPieceIsDone(const TaskTable & task_table, cons
{
LOG_DEBUG(log, "Check that all shards processed partition {} piece {} successfully", partition_name, piece_number);
- auto zookeeper = context.getZooKeeper();
+ auto zookeeper = getContext()->getZooKeeper();
/// Collect all shards that contain partition piece number piece_number.
Strings piece_status_paths;
@@ -532,7 +532,7 @@ TaskStatus ClusterCopier::tryMoveAllPiecesToDestinationTable(const TaskTable & t
LOG_DEBUG(log, "Try to move {} to destination table", partition_name);
- auto zookeeper = context.getZooKeeper();
+ auto zookeeper = getContext()->getZooKeeper();
const auto current_partition_attach_is_active = task_table.getPartitionAttachIsActivePath(partition_name);
const auto current_partition_attach_is_done = task_table.getPartitionAttachIsDonePath(partition_name);
@@ -1095,7 +1095,7 @@ TaskStatus ClusterCopier::tryCreateDestinationTable(const ConnectionTimeouts & t
= rewriteCreateQueryStorage(task_shard->current_pull_table_create_query, task_table.table_push, task_table.engine_push_ast);
auto & create = create_query_push_ast->as();
create.if_not_exists = true;
- InterpreterCreateQuery::prepareOnClusterQuery(create, context, task_table.cluster_push_name);
+ InterpreterCreateQuery::prepareOnClusterQuery(create, getContext(), task_table.cluster_push_name);
String query = queryToString(create_query_push_ast);
LOG_DEBUG(log, "Create destination tables. Query: {}", query);
@@ -1211,7 +1211,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl(
auto split_table_for_current_piece = task_shard.list_of_split_tables_on_shard[current_piece_number];
- auto zookeeper = context.getZooKeeper();
+ auto zookeeper = getContext()->getZooKeeper();
const String piece_is_dirty_flag_path = partition_piece.getPartitionPieceIsDirtyPath();
const String piece_is_dirty_cleaned_path = partition_piece.getPartitionPieceIsCleanedPath();
@@ -1262,7 +1262,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl(
ParserQuery p_query(query.data() + query.size());
- const auto & settings = context.getSettingsRef();
+ const auto & settings = getContext()->getSettingsRef();
return parseQuery(p_query, query, settings.max_query_size, settings.max_parser_depth);
};
@@ -1366,10 +1366,10 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl(
ASTPtr query_select_ast = get_select_query(split_table_for_current_piece, "count()", /*enable_splitting*/ true);
UInt64 count;
{
- Context local_context = context;
+ auto local_context = Context::createCopy(context);
// Use pull (i.e. readonly) settings, but fetch data from destination servers
- local_context.setSettings(task_cluster->settings_pull);
- local_context.setSetting("skip_unavailable_shards", true);
+ local_context->setSettings(task_cluster->settings_pull);
+ local_context->setSetting("skip_unavailable_shards", true);
Block block = getBlockWithAllStreamData(InterpreterFactory::get(query_select_ast, local_context)->execute().getInputStream());
count = (block) ? block.safeGetByPosition(0).column->getUInt(0) : 0;
@@ -1468,7 +1468,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl(
query += "INSERT INTO " + getQuotedTable(split_table_for_current_piece) + " VALUES ";
ParserQuery p_query(query.data() + query.size());
- const auto & settings = context.getSettingsRef();
+ const auto & settings = getContext()->getSettingsRef();
query_insert_ast = parseQuery(p_query, query, settings.max_query_size, settings.max_parser_depth);
LOG_DEBUG(log, "Executing INSERT query: {}", query);
@@ -1476,18 +1476,18 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl(
try
{
- std::unique_ptr context_select = std::make_unique(context);
+ auto context_select = Context::createCopy(context);
context_select->setSettings(task_cluster->settings_pull);
- std::unique_ptr context_insert = std::make_unique(context);
+ auto context_insert = Context::createCopy(context);
context_insert->setSettings(task_cluster->settings_push);
/// Custom INSERT SELECT implementation
BlockInputStreamPtr input;
BlockOutputStreamPtr output;
{
- BlockIO io_select = InterpreterFactory::get(query_select_ast, *context_select)->execute();
- BlockIO io_insert = InterpreterFactory::get(query_insert_ast, *context_insert)->execute();
+ BlockIO io_select = InterpreterFactory::get(query_select_ast, context_select)->execute();
+ BlockIO io_insert = InterpreterFactory::get(query_insert_ast, context_insert)->execute();
input = io_select.getInputStream();
output = io_insert.out;
@@ -1581,7 +1581,7 @@ void ClusterCopier::dropAndCreateLocalTable(const ASTPtr & create_ast)
const auto & create = create_ast->as();
dropLocalTableIfExists({create.database, create.table});
- InterpreterCreateQuery interpreter(create_ast, context);
+ InterpreterCreateQuery interpreter(create_ast, getContext());
interpreter.execute();
}
@@ -1592,7 +1592,7 @@ void ClusterCopier::dropLocalTableIfExists(const DatabaseAndTableName & table_na
drop_ast->database = table_name.first;
drop_ast->table = table_name.second;
- InterpreterDropQuery interpreter(drop_ast, context);
+ InterpreterDropQuery interpreter(drop_ast, getContext());
interpreter.execute();
}
@@ -1654,8 +1654,8 @@ void ClusterCopier::dropParticularPartitionPieceFromAllHelpingTables(const TaskT
String ClusterCopier::getRemoteCreateTable(const DatabaseAndTableName & table, Connection & connection, const Settings & settings)
{
- Context remote_context(context);
- remote_context.setSettings(settings);
+ auto remote_context = Context::createCopy(context);
+ remote_context->setSettings(settings);
String query = "SHOW CREATE TABLE " + getQuotedTable(table);
Block block = getBlockWithAllStreamData(std::make_shared(
@@ -1674,7 +1674,7 @@ ASTPtr ClusterCopier::getCreateTableForPullShard(const ConnectionTimeouts & time
task_cluster->settings_pull);
ParserCreateQuery parser_create_query;
- const auto & settings = context.getSettingsRef();
+ const auto & settings = getContext()->getSettingsRef();
return parseQuery(parser_create_query, create_query_pull_str, settings.max_query_size, settings.max_parser_depth);
}
@@ -1703,7 +1703,7 @@ void ClusterCopier::createShardInternalTables(const ConnectionTimeouts & timeout
/// Create special cluster with single shard
String shard_read_cluster_name = read_shard_prefix + task_table.cluster_pull_name;
ClusterPtr cluster_pull_current_shard = task_table.cluster_pull->getClusterWithSingleShard(task_shard.indexInCluster());
- context.setCluster(shard_read_cluster_name, cluster_pull_current_shard);
+ getContext()->setCluster(shard_read_cluster_name, cluster_pull_current_shard);
auto storage_shard_ast = createASTStorageDistributed(shard_read_cluster_name, task_table.table_pull.first, task_table.table_pull.second);
@@ -1763,13 +1763,13 @@ std::set ClusterCopier::getShardPartitions(const ConnectionTimeouts & ti
}
ParserQuery parser_query(query.data() + query.size());
- const auto & settings = context.getSettingsRef();
+ const auto & settings = getContext()->getSettingsRef();
ASTPtr query_ast = parseQuery(parser_query, query, settings.max_query_size, settings.max_parser_depth);
LOG_DEBUG(log, "Computing destination partition set, executing query: {}", query);
- Context local_context = context;
- local_context.setSettings(task_cluster->settings_pull);
+ auto local_context = Context::createCopy(context);
+ local_context->setSettings(task_cluster->settings_pull);
Block block = getBlockWithAllStreamData(InterpreterFactory::get(query_ast, local_context)->execute().getInputStream());
if (block)
@@ -1809,11 +1809,11 @@ bool ClusterCopier::checkShardHasPartition(const ConnectionTimeouts & timeouts,
LOG_DEBUG(log, "Checking shard {} for partition {} existence, executing query: {}", task_shard.getDescription(), partition_quoted_name, query);
ParserQuery parser_query(query.data() + query.size());
-const auto & settings = context.getSettingsRef();
+ const auto & settings = getContext()->getSettingsRef();
ASTPtr query_ast = parseQuery(parser_query, query, settings.max_query_size, settings.max_parser_depth);
- Context local_context = context;
- local_context.setSettings(task_cluster->settings_pull);
+ auto local_context = Context::createCopy(context);
+ local_context->setSettings(task_cluster->settings_pull);
return InterpreterFactory::get(query_ast, local_context)->execute().getInputStream()->read().rows() != 0;
}
@@ -1848,11 +1848,11 @@ bool ClusterCopier::checkPresentPartitionPiecesOnCurrentShard(const ConnectionTi
LOG_DEBUG(log, "Checking shard {} for partition {} piece {} existence, executing query: {}", task_shard.getDescription(), partition_quoted_name, std::to_string(current_piece_number), query);
ParserQuery parser_query(query.data() + query.size());
- const auto & settings = context.getSettingsRef();
+ const auto & settings = getContext()->getSettingsRef();
ASTPtr query_ast = parseQuery(parser_query, query, settings.max_query_size, settings.max_parser_depth);
- Context local_context = context;
- local_context.setSettings(task_cluster->settings_pull);
+ auto local_context = Context::createCopy(context);
+ local_context->setSettings(task_cluster->settings_pull);
auto result = InterpreterFactory::get(query_ast, local_context)->execute().getInputStream()->read().rows();
if (result != 0)
LOG_DEBUG(log, "Partition {} piece number {} is PRESENT on shard {}", partition_quoted_name, std::to_string(current_piece_number), task_shard.getDescription());
@@ -1908,7 +1908,7 @@ UInt64 ClusterCopier::executeQueryOnCluster(
/// In that case we don't have local replicas, but do it just in case
for (UInt64 i = 0; i < num_local_replicas; ++i)
{
- auto interpreter = InterpreterFactory::get(query_ast, context);
+ auto interpreter = InterpreterFactory::get(query_ast, getContext());
interpreter->execute();
if (increment_and_check_exit())
@@ -1923,8 +1923,8 @@ UInt64 ClusterCopier::executeQueryOnCluster(
auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(shard_settings).getSaturated(shard_settings.max_execution_time);
auto connections = shard.pool->getMany(timeouts, &shard_settings, pool_mode);
- Context shard_context(context);
- shard_context.setSettings(shard_settings);
+ auto shard_context = Context::createCopy(context);
+ shard_context->setSettings(shard_settings);
for (auto & connection : connections)
{
diff --git a/programs/copier/ClusterCopier.h b/programs/copier/ClusterCopier.h
index 95bb54cf4e1..e875ca7df2e 100644
--- a/programs/copier/ClusterCopier.h
+++ b/programs/copier/ClusterCopier.h
@@ -12,18 +12,17 @@
namespace DB
{
-class ClusterCopier
+class ClusterCopier : WithContext
{
public:
ClusterCopier(const String & task_path_,
const String & host_id_,
const String & proxy_database_name_,
- Context & context_)
- :
+ ContextPtr context_)
+ : WithContext(context_),
task_zookeeper_path(task_path_),
host_id(host_id_),
working_database_name(proxy_database_name_),
- context(context_),
log(&Poco::Logger::get("ClusterCopier")) {}
void init();
@@ -36,7 +35,7 @@ public:
/// Compute set of partitions, assume set of partitions aren't changed during the processing
void discoverTablePartitions(const ConnectionTimeouts & timeouts, TaskTable & task_table, UInt64 num_threads = 0);
- void uploadTaskDescription(const std::string & task_path, const std::string & task_file, const bool force);
+ void uploadTaskDescription(const std::string & task_path, const std::string & task_file, bool force);
void reloadTaskDescription();
@@ -120,7 +119,7 @@ protected:
/// Removes MATERIALIZED and ALIAS columns from create table query
static ASTPtr removeAliasColumnsFromCreateQuery(const ASTPtr & query_ast);
- bool tryDropPartitionPiece(ShardPartition & task_partition, const size_t current_piece_number,
+ bool tryDropPartitionPiece(ShardPartition & task_partition, size_t current_piece_number,
const zkutil::ZooKeeperPtr & zookeeper, const CleanStateClock & clean_state_clock);
static constexpr UInt64 max_table_tries = 3;
@@ -141,7 +140,7 @@ protected:
TaskStatus processPartitionPieceTaskImpl(const ConnectionTimeouts & timeouts,
ShardPartition & task_partition,
- const size_t current_piece_number,
+ size_t current_piece_number,
bool is_unprioritized_task);
void dropAndCreateLocalTable(const ASTPtr & create_ast);
@@ -219,7 +218,6 @@ private:
bool experimental_use_sample_offset{false};
- Context & context;
Poco::Logger * log;
std::chrono::milliseconds default_sleep_time{1000};
diff --git a/programs/copier/ClusterCopierApp.cpp b/programs/copier/ClusterCopierApp.cpp
index e3169a49ecf..d3fff616b65 100644
--- a/programs/copier/ClusterCopierApp.cpp
+++ b/programs/copier/ClusterCopierApp.cpp
@@ -3,6 +3,7 @@
#include
#include
#include
+#include
#include
@@ -110,9 +111,9 @@ void ClusterCopierApp::mainImpl()
LOG_INFO(log, "Starting clickhouse-copier (id {}, host_id {}, path {}, revision {})", process_id, host_id, process_path, ClickHouseRevision::getVersionRevision());
SharedContextHolder shared_context = Context::createShared();
- auto context = std::make_unique(Context::createGlobal(shared_context.get()));
+ auto context = Context::createGlobal(shared_context.get());
context->makeGlobalContext();
- SCOPE_EXIT(context->shutdown());
+ SCOPE_EXIT_SAFE(context->shutdown());
context->setConfig(loaded_config.configuration);
context->setApplicationType(Context::ApplicationType::LOCAL);
@@ -127,13 +128,13 @@ void ClusterCopierApp::mainImpl()
registerFormats();
static const std::string default_database = "_local";
- DatabaseCatalog::instance().attachDatabase(default_database, std::make_shared(default_database, *context));
+ DatabaseCatalog::instance().attachDatabase(default_database, std::make_shared(default_database, context));
context->setCurrentDatabase(default_database);
/// Initialize query scope just in case.
- CurrentThread::QueryScope query_scope(*context);
+ CurrentThread::QueryScope query_scope(context);
- auto copier = std::make_unique(task_path, host_id, default_database, *context);
+ auto copier = std::make_unique(task_path, host_id, default_database, context);
copier->setSafeMode(is_safe_mode);
copier->setCopyFaultProbability(copy_fault_probability);
copier->setMoveFaultProbability(move_fault_probability);
diff --git a/programs/copier/Internals.cpp b/programs/copier/Internals.cpp
index ea2be469945..bec612a8226 100644
--- a/programs/copier/Internals.cpp
+++ b/programs/copier/Internals.cpp
@@ -222,8 +222,8 @@ Names extractPrimaryKeyColumnNames(const ASTPtr & storage_ast)
{
String pk_column = primary_key_expr_list->children[i]->getColumnName();
if (pk_column != sorting_key_column)
- throw Exception("Primary key must be a prefix of the sorting key, but in position "
- + toString(i) + " its column is " + pk_column + ", not " + sorting_key_column,
+ throw Exception("Primary key must be a prefix of the sorting key, but the column in the position "
+ + toString(i) + " is " + sorting_key_column +", not " + pk_column,
ErrorCodes::BAD_ARGUMENTS);
if (!primary_key_columns_set.emplace(pk_column).second)
diff --git a/programs/format/Format.cpp b/programs/format/Format.cpp
index ba3d6e8557b..5bf19191353 100644
--- a/programs/format/Format.cpp
+++ b/programs/format/Format.cpp
@@ -102,8 +102,8 @@ int mainEntryClickHouseFormat(int argc, char ** argv)
}
SharedContextHolder shared_context = Context::createShared();
- Context context = Context::createGlobal(shared_context.get());
- context.makeGlobalContext();
+ auto context = Context::createGlobal(shared_context.get());
+ context->makeGlobalContext();
registerFunctions();
registerAggregateFunctions();
diff --git a/programs/install/Install.cpp b/programs/install/Install.cpp
index ef72624e7ab..2b0f390f709 100644
--- a/programs/install/Install.cpp
+++ b/programs/install/Install.cpp
@@ -71,6 +71,9 @@ namespace ErrorCodes
}
+/// ANSI escape sequence for intense color in terminal.
+#define HILITE "\033[1m"
+#define END_HILITE "\033[0m"
using namespace DB;
namespace po = boost::program_options;
@@ -559,20 +562,32 @@ int mainEntryClickHouseInstall(int argc, char ** argv)
bool stdin_is_a_tty = isatty(STDIN_FILENO);
bool stdout_is_a_tty = isatty(STDOUT_FILENO);
- bool is_interactive = stdin_is_a_tty && stdout_is_a_tty;
+
+ /// dpkg or apt installers can ask for non-interactive work explicitly.
+
+ const char * debian_frontend_var = getenv("DEBIAN_FRONTEND");
+ bool noninteractive = debian_frontend_var && debian_frontend_var == std::string_view("noninteractive");
+
+ bool is_interactive = !noninteractive && stdin_is_a_tty && stdout_is_a_tty;
+
+ /// We can ask password even if stdin is closed/redirected but /dev/tty is available.
+ bool can_ask_password = !noninteractive && stdout_is_a_tty;
if (has_password_for_default_user)
{
- fmt::print("Password for default user is already specified. To remind or reset, see {} and {}.\n",
+ fmt::print(HILITE "Password for default user is already specified. To remind or reset, see {} and {}." END_HILITE "\n",
users_config_file.string(), users_d.string());
}
- else if (!is_interactive)
+ else if (!can_ask_password)
{
- fmt::print("Password for default user is empty string. See {} and {} to change it.\n",
+ fmt::print(HILITE "Password for default user is empty string. See {} and {} to change it." END_HILITE "\n",
users_config_file.string(), users_d.string());
}
else
{
+ /// NOTE: When installing debian package with dpkg -i, stdin is not a terminal but we are still being able to enter password.
+ /// More sophisticated method with /dev/tty is used inside the `readpassphrase` function.
+
char buf[1000] = {};
std::string password;
if (auto * result = readpassphrase("Enter password for default user: ", buf, sizeof(buf), 0))
@@ -600,7 +615,7 @@ int mainEntryClickHouseInstall(int argc, char ** argv)
"\n";
out.sync();
out.finalize();
- fmt::print("Password for default user is saved in file {}.\n", password_file);
+ fmt::print(HILITE "Password for default user is saved in file {}." END_HILITE "\n", password_file);
#else
out << "\n"
" \n"
@@ -611,12 +626,12 @@ int mainEntryClickHouseInstall(int argc, char ** argv)
"\n";
out.sync();
out.finalize();
- fmt::print("Password for default user is saved in plaintext in file {}.\n", password_file);
+ fmt::print(HILITE "Password for default user is saved in plaintext in file {}." END_HILITE "\n", password_file);
#endif
has_password_for_default_user = true;
}
else
- fmt::print("Password for default user is empty string. See {} and {} to change it.\n",
+ fmt::print(HILITE "Password for default user is empty string. See {} and {} to change it." END_HILITE "\n",
users_config_file.string(), users_d.string());
}
@@ -641,7 +656,6 @@ int mainEntryClickHouseInstall(int argc, char ** argv)
" This is optional. Taskstats accounting will be disabled."
" To enable taskstats accounting you may add the required capability later manually.\"",
"/tmp/test_setcap.sh", fs::canonical(main_bin_path).string());
- fmt::print(" {}\n", command);
executeScript(command);
#endif
diff --git a/programs/library-bridge/CMakeLists.txt b/programs/library-bridge/CMakeLists.txt
new file mode 100644
index 00000000000..0913c6e4a9a
--- /dev/null
+++ b/programs/library-bridge/CMakeLists.txt
@@ -0,0 +1,25 @@
+set (CLICKHOUSE_LIBRARY_BRIDGE_SOURCES
+ library-bridge.cpp
+ LibraryInterface.cpp
+ LibraryBridge.cpp
+ Handlers.cpp
+ HandlerFactory.cpp
+ SharedLibraryHandler.cpp
+ SharedLibraryHandlerFactory.cpp
+)
+
+if (OS_LINUX)
+ set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-export-dynamic")
+endif ()
+
+add_executable(clickhouse-library-bridge ${CLICKHOUSE_LIBRARY_BRIDGE_SOURCES})
+
+target_link_libraries(clickhouse-library-bridge PRIVATE
+ daemon
+ dbms
+ bridge
+)
+
+set_target_properties(clickhouse-library-bridge PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
+
+install(TARGETS clickhouse-library-bridge RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
diff --git a/programs/library-bridge/HandlerFactory.cpp b/programs/library-bridge/HandlerFactory.cpp
new file mode 100644
index 00000000000..9f53a24156f
--- /dev/null
+++ b/programs/library-bridge/HandlerFactory.cpp
@@ -0,0 +1,23 @@
+#include "HandlerFactory.h"
+
+#include
+#include
+#include "Handlers.h"
+
+
+namespace DB
+{
+ std::unique_ptr LibraryBridgeHandlerFactory::createRequestHandler(const HTTPServerRequest & request)
+ {
+ Poco::URI uri{request.getURI()};
+ LOG_DEBUG(log, "Request URI: {}", uri.toString());
+
+ if (uri == "/ping" && request.getMethod() == Poco::Net::HTTPRequest::HTTP_GET)
+ return std::make_unique(keep_alive_timeout);
+
+ if (request.getMethod() == Poco::Net::HTTPRequest::HTTP_POST)
+ return std::make_unique(keep_alive_timeout, getContext());
+
+ return nullptr;
+ }
+}
diff --git a/programs/library-bridge/HandlerFactory.h b/programs/library-bridge/HandlerFactory.h
new file mode 100644
index 00000000000..93f0721bf01
--- /dev/null
+++ b/programs/library-bridge/HandlerFactory.h
@@ -0,0 +1,37 @@
+#pragma once
+
+#include
+#include
+#include
+
+
+namespace DB
+{
+
+class SharedLibraryHandler;
+using SharedLibraryHandlerPtr = std::shared_ptr;
+
+/// Factory for '/ping', '/' handlers.
+class LibraryBridgeHandlerFactory : public HTTPRequestHandlerFactory, WithContext
+{
+public:
+ LibraryBridgeHandlerFactory(
+ const std::string & name_,
+ size_t keep_alive_timeout_,
+ ContextPtr context_)
+ : WithContext(context_)
+ , log(&Poco::Logger::get(name_))
+ , name(name_)
+ , keep_alive_timeout(keep_alive_timeout_)
+ {
+ }
+
+ std::unique_ptr createRequestHandler(const HTTPServerRequest & request) override;
+
+private:
+ Poco::Logger * log;
+ std::string name;
+ size_t keep_alive_timeout;
+};
+
+}
diff --git a/programs/library-bridge/Handlers.cpp b/programs/library-bridge/Handlers.cpp
new file mode 100644
index 00000000000..6a1bfbbccb7
--- /dev/null
+++ b/programs/library-bridge/Handlers.cpp
@@ -0,0 +1,288 @@
+#include "Handlers.h"
+#include "SharedLibraryHandlerFactory.h"
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+
+namespace DB
+{
+namespace
+{
+ std::shared_ptr parseColumns(std::string && column_string)
+ {
+ auto sample_block = std::make_shared();
+ auto names_and_types = NamesAndTypesList::parse(column_string);
+
+ for (const NameAndTypePair & column_data : names_and_types)
+ sample_block->insert({column_data.type, column_data.name});
+
+ return sample_block;
+ }
+
+ std::vector parseIdsFromBinary(const std::string & ids_string)
+ {
+ ReadBufferFromString buf(ids_string);
+ std::vector ids;
+ readVectorBinary(ids, buf);
+ return ids;
+ }
+
+ std::vector parseNamesFromBinary(const std::string & names_string)
+ {
+ ReadBufferFromString buf(names_string);
+ std::vector names;
+ readVectorBinary(names, buf);
+ return names;
+ }
+}
+
+
+void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response)
+{
+ LOG_TRACE(log, "Request URI: {}", request.getURI());
+ HTMLForm params(request);
+
+ if (!params.has("method"))
+ {
+ processError(response, "No 'method' in request URL");
+ return;
+ }
+
+ if (!params.has("dictionary_id"))
+ {
+ processError(response, "No 'dictionary_id in request URL");
+ return;
+ }
+
+ std::string method = params.get("method");
+ std::string dictionary_id = params.get("dictionary_id");
+ LOG_TRACE(log, "Library method: '{}', dictionary id: {}", method, dictionary_id);
+
+ WriteBufferFromHTTPServerResponse out(response, request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD, keep_alive_timeout);
+
+ try
+ {
+ if (method == "libNew")
+ {
+ auto & read_buf = request.getStream();
+ params.read(read_buf);
+
+ if (!params.has("library_path"))
+ {
+ processError(response, "No 'library_path' in request URL");
+ return;
+ }
+
+ if (!params.has("library_settings"))
+ {
+ processError(response, "No 'library_settings' in request URL");
+ return;
+ }
+
+ std::string library_path = params.get("library_path");
+ const auto & settings_string = params.get("library_settings");
+ std::vector library_settings = parseNamesFromBinary(settings_string);
+
+ /// Needed for library dictionary
+ if (!params.has("attributes_names"))
+ {
+ processError(response, "No 'attributes_names' in request URL");
+ return;
+ }
+
+ const auto & attributes_string = params.get("attributes_names");
+ std::vector attributes_names = parseNamesFromBinary(attributes_string);
+
+ /// Needed to parse block from binary string format
+ if (!params.has("sample_block"))
+ {
+ processError(response, "No 'sample_block' in request URL");
+ return;
+ }
+ std::string sample_block_string = params.get("sample_block");
+
+ std::shared_ptr sample_block;
+ try
+ {
+ sample_block = parseColumns(std::move(sample_block_string));
+ }
+ catch (const Exception & ex)
+ {
+ processError(response, "Invalid 'sample_block' parameter in request body '" + ex.message() + "'");
+ LOG_WARNING(log, ex.getStackTraceString());
+ return;
+ }
+
+ if (!params.has("null_values"))
+ {
+ processError(response, "No 'null_values' in request URL");
+ return;
+ }
+
+ ReadBufferFromString read_block_buf(params.get("null_values"));
+ auto format = FormatFactory::instance().getInput(FORMAT, read_block_buf, *sample_block, getContext(), DEFAULT_BLOCK_SIZE);
+ auto reader = std::make_shared(format);
+ auto sample_block_with_nulls = reader->read();
+
+ LOG_DEBUG(log, "Dictionary sample block with null values: {}", sample_block_with_nulls.dumpStructure());
+
+ SharedLibraryHandlerFactory::instance().create(dictionary_id, library_path, library_settings, sample_block_with_nulls, attributes_names);
+ writeStringBinary("1", out);
+ }
+ else if (method == "libClone")
+ {
+ if (!params.has("from_dictionary_id"))
+ {
+ processError(response, "No 'from_dictionary_id' in request URL");
+ return;
+ }
+
+ std::string from_dictionary_id = params.get("from_dictionary_id");
+ LOG_TRACE(log, "Calling libClone from {} to {}", from_dictionary_id, dictionary_id);
+ SharedLibraryHandlerFactory::instance().clone(from_dictionary_id, dictionary_id);
+ writeStringBinary("1", out);
+ }
+ else if (method == "libDelete")
+ {
+ SharedLibraryHandlerFactory::instance().remove(dictionary_id);
+ writeStringBinary("1", out);
+ }
+ else if (method == "isModified")
+ {
+ auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id);
+ bool res = library_handler->isModified();
+ writeStringBinary(std::to_string(res), out);
+ }
+ else if (method == "supportsSelectiveLoad")
+ {
+ auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id);
+ bool res = library_handler->supportsSelectiveLoad();
+ writeStringBinary(std::to_string(res), out);
+ }
+ else if (method == "loadAll")
+ {
+ auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id);
+ const auto & sample_block = library_handler->getSampleBlock();
+ auto input = library_handler->loadAll();
+
+ BlockOutputStreamPtr output = FormatFactory::instance().getOutputStream(FORMAT, out, sample_block, getContext());
+ copyData(*input, *output);
+ }
+ else if (method == "loadIds")
+ {
+ params.read(request.getStream());
+
+ if (!params.has("ids"))
+ {
+ processError(response, "No 'ids' in request URL");
+ return;
+ }
+
+ std::vector ids = parseIdsFromBinary(params.get("ids"));
+ auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id);
+ const auto & sample_block = library_handler->getSampleBlock();
+ auto input = library_handler->loadIds(ids);
+ BlockOutputStreamPtr output = FormatFactory::instance().getOutputStream(FORMAT, out, sample_block, getContext());
+ copyData(*input, *output);
+ }
+ else if (method == "loadKeys")
+ {
+ if (!params.has("requested_block_sample"))
+ {
+ processError(response, "No 'requested_block_sample' in request URL");
+ return;
+ }
+
+ std::string requested_block_string = params.get("requested_block_sample");
+
+ std::shared_ptr requested_sample_block;
+ try
+ {
+ requested_sample_block = parseColumns(std::move(requested_block_string));
+ }
+ catch (const Exception & ex)
+ {
+ processError(response, "Invalid 'requested_block' parameter in request body '" + ex.message() + "'");
+ LOG_WARNING(log, ex.getStackTraceString());
+ return;
+ }
+
+ auto & read_buf = request.getStream();
+ auto format = FormatFactory::instance().getInput(FORMAT, read_buf, *requested_sample_block, getContext(), DEFAULT_BLOCK_SIZE);
+ auto reader = std::make_shared(format);
+ auto block = reader->read();
+
+ auto library_handler = SharedLibraryHandlerFactory::instance().get(dictionary_id);
+ const auto & sample_block = library_handler->getSampleBlock();
+ auto input = library_handler->loadKeys(block.getColumns());
+ BlockOutputStreamPtr output = FormatFactory::instance().getOutputStream(FORMAT, out, sample_block, getContext());
+ copyData(*input, *output);
+ }
+ }
+ catch (...)
+ {
+ auto message = getCurrentExceptionMessage(true);
+ response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR, message); // can't call process_error, because of too soon response sending
+
+ try
+ {
+ writeStringBinary(message, out);
+ out.finalize();
+ }
+ catch (...)
+ {
+ tryLogCurrentException(log);
+ }
+
+ tryLogCurrentException(log);
+ }
+
+ try
+ {
+ out.finalize();
+ }
+ catch (...)
+ {
+ tryLogCurrentException(log);
+ }
+}
+
+
+void LibraryRequestHandler::processError(HTTPServerResponse & response, const std::string & message)
+{
+ response.setStatusAndReason(HTTPResponse::HTTP_INTERNAL_SERVER_ERROR);
+
+ if (!response.sent())
+ *response.send() << message << std::endl;
+
+ LOG_WARNING(log, message);
+}
+
+
+void PingHandler::handleRequest(HTTPServerRequest & /* request */, HTTPServerResponse & response)
+{
+ try
+ {
+ setResponseDefaultHeaders(response, keep_alive_timeout);
+ const char * data = "Ok.\n";
+ response.sendBuffer(data, strlen(data));
+ }
+ catch (...)
+ {
+ tryLogCurrentException("PingHandler");
+ }
+}
+
+
+}
diff --git a/programs/library-bridge/Handlers.h b/programs/library-bridge/Handlers.h
new file mode 100644
index 00000000000..dac61d3a735
--- /dev/null
+++ b/programs/library-bridge/Handlers.h
@@ -0,0 +1,59 @@
+#pragma once
+
+#include
+#include
+#include
+#include "SharedLibraryHandler.h"
+
+
+namespace DB
+{
+
+
+/// Handler for requests to Library Dictionary Source, returns response in RowBinary format.
+/// When a library dictionary source is created, it sends libNew request to library bridge (which is started on first
+/// request to it, if it was not yet started). On this request a new sharedLibrayHandler is added to a
+/// sharedLibraryHandlerFactory by a dictionary uuid. With libNew request come: library_path, library_settings,
+/// names of dictionary attributes, sample block to parse block of null values, block of null values. Everything is
+/// passed in binary format and is urlencoded. When dictionary is cloned, a new handler is created.
+/// Each handler is unique to dictionary.
+class LibraryRequestHandler : public HTTPRequestHandler, WithContext
+{
+public:
+
+ LibraryRequestHandler(
+ size_t keep_alive_timeout_,
+ ContextPtr context_)
+ : WithContext(context_)
+ , log(&Poco::Logger::get("LibraryRequestHandler"))
+ , keep_alive_timeout(keep_alive_timeout_)
+ {
+ }
+
+ void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) override;
+
+private:
+ static constexpr inline auto FORMAT = "RowBinary";
+
+ void processError(HTTPServerResponse & response, const std::string & message);
+
+ Poco::Logger * log;
+ size_t keep_alive_timeout;
+};
+
+
+class PingHandler : public HTTPRequestHandler
+{
+public:
+ explicit PingHandler(size_t keep_alive_timeout_)
+ : keep_alive_timeout(keep_alive_timeout_)
+ {
+ }
+
+ void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) override;
+
+private:
+ const size_t keep_alive_timeout;
+};
+
+}
diff --git a/programs/library-bridge/LibraryBridge.cpp b/programs/library-bridge/LibraryBridge.cpp
new file mode 100644
index 00000000000..2e5d6041151
--- /dev/null
+++ b/programs/library-bridge/LibraryBridge.cpp
@@ -0,0 +1,17 @@
+#include "LibraryBridge.h"
+
+#pragma GCC diagnostic ignored "-Wmissing-declarations"
+int mainEntryClickHouseLibraryBridge(int argc, char ** argv)
+{
+ DB::LibraryBridge app;
+ try
+ {
+ return app.run(argc, argv);
+ }
+ catch (...)
+ {
+ std::cerr << DB::getCurrentExceptionMessage(true) << "\n";
+ auto code = DB::getCurrentExceptionCode();
+ return code ? code : 1;
+ }
+}
diff --git a/programs/library-bridge/LibraryBridge.h b/programs/library-bridge/LibraryBridge.h
new file mode 100644
index 00000000000..9f2dafb89ab
--- /dev/null
+++ b/programs/library-bridge/LibraryBridge.h
@@ -0,0 +1,26 @@
+#pragma once
+
+#include
+#include
+#include "HandlerFactory.h"
+
+
+namespace DB
+{
+
+class LibraryBridge : public IBridge
+{
+
+protected:
+ std::string bridgeName() const override
+ {
+ return "LibraryBridge";
+ }
+
+ HandlerFactoryPtr getHandlerFactoryPtr(ContextPtr context) const override
+ {
+ return std::make_shared("LibraryRequestHandlerFactory-factory", keep_alive_timeout, context);
+ }
+};
+
+}
diff --git a/src/Dictionaries/LibraryDictionarySourceExternal.cpp b/programs/library-bridge/LibraryInterface.cpp
similarity index 97%
rename from src/Dictionaries/LibraryDictionarySourceExternal.cpp
rename to programs/library-bridge/LibraryInterface.cpp
index 259d0a2846a..3975368c17f 100644
--- a/src/Dictionaries/LibraryDictionarySourceExternal.cpp
+++ b/programs/library-bridge/LibraryInterface.cpp
@@ -1,4 +1,5 @@
-#include "LibraryDictionarySourceExternal.h"
+#include "LibraryInterface.h"
+
#include
namespace
diff --git a/src/Dictionaries/LibraryDictionarySourceExternal.h b/programs/library-bridge/LibraryInterface.h
similarity index 97%
rename from src/Dictionaries/LibraryDictionarySourceExternal.h
rename to programs/library-bridge/LibraryInterface.h
index 3b92707d091..d23de59bbb1 100644
--- a/src/Dictionaries/LibraryDictionarySourceExternal.h
+++ b/programs/library-bridge/LibraryInterface.h
@@ -101,7 +101,7 @@ using RequestedIds = const VectorUInt64 *;
using LibraryLoadIdsFunc = RawClickHouseLibraryTable (*)(LibraryData, LibrarySettings, RequestedColumnsNames, RequestedIds);
using RequestedKeys = Table *;
-/// There is no requested columns names for load keys func
+/// There are no requested column names for load keys func
using LibraryLoadKeysFunc = RawClickHouseLibraryTable (*)(LibraryData, LibrarySettings, RequestedKeys);
using LibraryIsModifiedFunc = bool (*)(LibraryContext, LibrarySettings);
diff --git a/programs/library-bridge/LibraryUtils.h b/programs/library-bridge/LibraryUtils.h
new file mode 100644
index 00000000000..8ced8df1c48
--- /dev/null
+++ b/programs/library-bridge/LibraryUtils.h
@@ -0,0 +1,44 @@
+#pragma once
+
+#include
+#include
+#include
+#include
+
+#include "LibraryInterface.h"
+
+
+namespace DB
+{
+
+class CStringsHolder
+{
+
+public:
+ using Container = std::vector;
+
+ explicit CStringsHolder(const Container & strings_pass)
+ {
+ strings_holder = strings_pass;
+ strings.size = strings_holder.size();
+
+ ptr_holder = std::make_unique(strings.size);
+ strings.data = ptr_holder.get();
+
+ size_t i = 0;
+ for (auto & str : strings_holder)
+ {
+ strings.data[i] = str.c_str();
+ ++i;
+ }
+ }
+
+ ClickHouseLibrary::CStrings strings; // will pass pointer to lib
+
+private:
+ std::unique_ptr ptr_holder = nullptr;
+ Container strings_holder;
+};
+
+
+}
diff --git a/programs/library-bridge/SharedLibraryHandler.cpp b/programs/library-bridge/SharedLibraryHandler.cpp
new file mode 100644
index 00000000000..ab8cf2417c2
--- /dev/null
+++ b/programs/library-bridge/SharedLibraryHandler.cpp
@@ -0,0 +1,219 @@
+#include "SharedLibraryHandler.h"
+
+#include
+#include
+#include
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+ extern const int EXTERNAL_LIBRARY_ERROR;
+ extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
+}
+
+
+SharedLibraryHandler::SharedLibraryHandler(
+ const std::string & library_path_,
+ const std::vector & library_settings,
+ const Block & sample_block_,
+ const std::vector & attributes_names_)
+ : library_path(library_path_)
+ , sample_block(sample_block_)
+ , attributes_names(attributes_names_)
+{
+ library = std::make_shared(library_path, RTLD_LAZY);
+ settings_holder = std::make_shared(CStringsHolder(library_settings));
+
+ auto lib_new = library->tryGet(ClickHouseLibrary::LIBRARY_CREATE_NEW_FUNC_NAME);
+
+ if (lib_new)
+ lib_data = lib_new(&settings_holder->strings, ClickHouseLibrary::log);
+ else
+ throw Exception("Method libNew failed", ErrorCodes::EXTERNAL_LIBRARY_ERROR);
+}
+
+
+SharedLibraryHandler::SharedLibraryHandler(const SharedLibraryHandler & other)
+ : library_path{other.library_path}
+ , sample_block{other.sample_block}
+ , attributes_names{other.attributes_names}
+ , library{other.library}
+ , settings_holder{other.settings_holder}
+{
+
+ auto lib_clone = library->tryGet(ClickHouseLibrary::LIBRARY_CLONE_FUNC_NAME);
+
+ if (lib_clone)
+ {
+ lib_data = lib_clone(other.lib_data);
+ }
+ else
+ {
+ auto lib_new = library->tryGet(ClickHouseLibrary::LIBRARY_CREATE_NEW_FUNC_NAME);
+
+ if (lib_new)
+ lib_data = lib_new(&settings_holder->strings, ClickHouseLibrary::log);
+ }
+}
+
+
+SharedLibraryHandler::~SharedLibraryHandler()
+{
+ auto lib_delete = library->tryGet(ClickHouseLibrary::LIBRARY_DELETE_FUNC_NAME);
+
+ if (lib_delete)
+ lib_delete(lib_data);
+}
+
+
+bool SharedLibraryHandler::isModified()
+{
+ auto func_is_modified = library->tryGet(ClickHouseLibrary::LIBRARY_IS_MODIFIED_FUNC_NAME);
+
+ if (func_is_modified)
+ return func_is_modified(lib_data, &settings_holder->strings);
+
+ return true;
+}
+
+
+bool SharedLibraryHandler::supportsSelectiveLoad()
+{
+ auto func_supports_selective_load = library->tryGet(ClickHouseLibrary::LIBRARY_SUPPORTS_SELECTIVE_LOAD_FUNC_NAME);
+
+ if (func_supports_selective_load)
+ return func_supports_selective_load(lib_data, &settings_holder->strings);
+
+ return true;
+}
+
+
+BlockInputStreamPtr SharedLibraryHandler::loadAll()
+{
+ auto columns_holder = std::make_unique(attributes_names.size());
+ ClickHouseLibrary::CStrings columns{static_cast(columns_holder.get()), attributes_names.size()};
+ for (size_t i = 0; i < attributes_names.size(); ++i)
+ columns.data[i] = attributes_names[i].c_str();
+
+ auto load_all_func = library->get(ClickHouseLibrary::LIBRARY_LOAD_ALL_FUNC_NAME);
+ auto data_new_func = library->get(ClickHouseLibrary::LIBRARY_DATA_NEW_FUNC_NAME);
+ auto data_delete_func = library->get(ClickHouseLibrary::LIBRARY_DATA_DELETE_FUNC_NAME);
+
+ ClickHouseLibrary::LibraryData data_ptr = data_new_func(lib_data);
+ SCOPE_EXIT(data_delete_func(lib_data, data_ptr));
+
+ ClickHouseLibrary::RawClickHouseLibraryTable data = load_all_func(data_ptr, &settings_holder->strings, &columns);
+ auto block = dataToBlock(data);
+
+ return std::make_shared(block);
+}
+
+
+BlockInputStreamPtr SharedLibraryHandler::loadIds(const std::vector & ids)
+{
+ const ClickHouseLibrary::VectorUInt64 ids_data{ext::bit_cast(ids.data()), ids.size()};
+
+ auto columns_holder = std::make_unique(attributes_names.size());
+ ClickHouseLibrary::CStrings columns_pass{static_cast(columns_holder.get()), attributes_names.size()};
+
+ auto load_ids_func = library->get(ClickHouseLibrary::LIBRARY_LOAD_IDS_FUNC_NAME);
+ auto data_new_func = library->get(ClickHouseLibrary::LIBRARY_DATA_NEW_FUNC_NAME);
+ auto data_delete_func = library->get(ClickHouseLibrary::LIBRARY_DATA_DELETE_FUNC_NAME);
+
+ ClickHouseLibrary::LibraryData data_ptr = data_new_func(lib_data);
+ SCOPE_EXIT(data_delete_func(lib_data, data_ptr));
+
+ ClickHouseLibrary::RawClickHouseLibraryTable data = load_ids_func(data_ptr, &settings_holder->strings, &columns_pass, &ids_data);
+ auto block = dataToBlock(data);
+
+ return std::make_shared(block);
+}
+
+
+BlockInputStreamPtr SharedLibraryHandler::loadKeys(const Columns & key_columns)
+{
+ auto holder = std::make_unique(key_columns.size());
+ std::vector> column_data_holders;
+
+ for (size_t i = 0; i < key_columns.size(); ++i)
+ {
+ auto cell_holder = std::make_unique(key_columns[i]->size());
+
+ for (size_t j = 0; j < key_columns[i]->size(); ++j)
+ {
+ auto data_ref = key_columns[i]->getDataAt(j);
+
+ cell_holder[j] = ClickHouseLibrary::Field{
+ .data = static_cast(data_ref.data),
+ .size = data_ref.size};
+ }
+
+ holder[i] = ClickHouseLibrary::Row{
+ .data = static_cast(cell_holder.get()),
+ .size = key_columns[i]->size()};
+
+ column_data_holders.push_back(std::move(cell_holder));
+ }
+
+ ClickHouseLibrary::Table request_cols{
+ .data = static_cast(holder.get()),
+ .size = key_columns.size()};
+
+ auto load_keys_func = library->get(ClickHouseLibrary::LIBRARY_LOAD_KEYS_FUNC_NAME);
+ auto data_new_func = library->get(ClickHouseLibrary::LIBRARY_DATA_NEW_FUNC_NAME);
+ auto data_delete_func = library->get(ClickHouseLibrary::LIBRARY_DATA_DELETE_FUNC_NAME);
+
+ ClickHouseLibrary::LibraryData data_ptr = data_new_func(lib_data);
+ SCOPE_EXIT(data_delete_func(lib_data, data_ptr));
+
+ ClickHouseLibrary::RawClickHouseLibraryTable data = load_keys_func(data_ptr, &settings_holder->strings, &request_cols);
+ auto block = dataToBlock(data);
+
+ return std::make_shared(block);
+}
+
+
+Block SharedLibraryHandler::dataToBlock(const ClickHouseLibrary::RawClickHouseLibraryTable data)
+{
+ if (!data)
+ throw Exception("LibraryDictionarySource: No data returned", ErrorCodes::EXTERNAL_LIBRARY_ERROR);
+
+ const auto * columns_received = static_cast(data);
+ if (columns_received->error_code)
+ throw Exception(
+ "LibraryDictionarySource: Returned error: " + std::to_string(columns_received->error_code) + " " + (columns_received->error_string ? columns_received->error_string : ""),
+ ErrorCodes::EXTERNAL_LIBRARY_ERROR);
+
+ MutableColumns columns = sample_block.cloneEmptyColumns();
+
+ for (size_t col_n = 0; col_n < columns_received->size; ++col_n)
+ {
+ if (columns.size() != columns_received->data[col_n].size)
+ throw Exception(
+ "LibraryDictionarySource: Returned unexpected number of columns: " + std::to_string(columns_received->data[col_n].size) + ", must be " + std::to_string(columns.size()),
+ ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH);
+
+ for (size_t row_n = 0; row_n < columns_received->data[col_n].size; ++row_n)
+ {
+ const auto & field = columns_received->data[col_n].data[row_n];
+ if (!field.data)
+ {
+ /// sample_block contains null_value (from config) inside corresponding column
+ const auto & col = sample_block.getByPosition(row_n);
+ columns[row_n]->insertFrom(*(col.column), 0);
+ }
+ else
+ {
+ const auto & size = field.size;
+ columns[row_n]->insertData(static_cast(field.data), size);
+ }
+ }
+ }
+
+ return sample_block.cloneWithColumns(std::move(columns));
+}
+
+}
diff --git a/programs/library-bridge/SharedLibraryHandler.h b/programs/library-bridge/SharedLibraryHandler.h
new file mode 100644
index 00000000000..5c0334ac89f
--- /dev/null
+++ b/programs/library-bridge/SharedLibraryHandler.h
@@ -0,0 +1,54 @@
+#pragma once
+
+#include
+#include
+#include
+#include "LibraryUtils.h"
+
+
+namespace DB
+{
+
+/// A class that manages all operations with library dictionary.
+/// Every library dictionary source has its own object of this class, accessed by UUID.
+class SharedLibraryHandler
+{
+
+public:
+ SharedLibraryHandler(
+ const std::string & library_path_,
+ const std::vector & library_settings,
+ const Block & sample_block_,
+ const std::vector