diff --git a/.clang-tidy b/.clang-tidy index b0971418e0e..ecb8ac6dcbf 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -203,3 +203,5 @@ CheckOptions: value: CamelCase - key: readability-identifier-naming.UsingCase value: CamelCase + - key: modernize-loop-convert.UseCxx20ReverseRanges + value: false diff --git a/.gitmodules b/.gitmodules index 37b22527eb4..74d1049ce01 100644 --- a/.gitmodules +++ b/.gitmodules @@ -246,3 +246,6 @@ [submodule "contrib/bzip2"] path = contrib/bzip2 url = https://github.com/ClickHouse-Extras/bzip2.git +[submodule "contrib/magic_enum"] + path = contrib/magic_enum + url = https://github.com/Neargye/magic_enum diff --git a/CHANGELOG.md b/CHANGELOG.md index 71cdac17825..0e92fc59509 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,214 @@ +### ClickHouse release v21.9, 2021-09-09 + +#### Backward Incompatible Change + +* Do not output trailing zeros in text representation of `Decimal` types. Example: `1.23` will be printed instead of `1.230000` for decimal with scale 6. This closes [#15794](https://github.com/ClickHouse/ClickHouse/issues/15794). It may introduce slight incompatibility if your applications somehow relied on the trailing zeros. Serialization in output formats can be controlled with the setting `output_format_decimal_trailing_zeros`. Implementation of `toString` and casting to String is changed unconditionally. [#27680](https://github.com/ClickHouse/ClickHouse/pull/27680) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Do not allow to apply parametric aggregate function with `-Merge` combinator to aggregate function state if state was produced by aggregate function with different parameters. For example, state of `fooState(42)(x)` cannot be finalized with `fooMerge(s)` or `fooMerge(123)(s)`, parameters must be specified explicitly like `fooMerge(42)(s)` and must be equal. It does not affect some special aggregate functions like `quantile` and `sequence*` that use parameters for finalization only. [#26847](https://github.com/ClickHouse/ClickHouse/pull/26847) ([tavplubix](https://github.com/tavplubix)). +* Under clickhouse-local, always treat local addresses with a port as remote. [#26736](https://github.com/ClickHouse/ClickHouse/pull/26736) ([Raúl Marín](https://github.com/Algunenano)). +* Fix the issue that in case of some sophisticated query with column aliases identical to the names of expressions, bad cast may happen. This fixes [#25447](https://github.com/ClickHouse/ClickHouse/issues/25447). This fixes [#26914](https://github.com/ClickHouse/ClickHouse/issues/26914). This fix may introduce backward incompatibility: if there are different expressions with identical names, exception will be thrown. It may break some rare cases when `enable_optimize_predicate_expression` is set. [#26639](https://github.com/ClickHouse/ClickHouse/pull/26639) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Now, scalar subquery always returns `Nullable` result if it's type can be `Nullable`. It is needed because in case of empty subquery it's result should be `Null`. Previously, it was possible to get error about incompatible types (type deduction does not execute scalar subquery, and it could use not-nullable type). Scalar subquery with empty result which can't be converted to `Nullable` (like `Array` or `Tuple`) now throws error. Fixes [#25411](https://github.com/ClickHouse/ClickHouse/issues/25411). [#26423](https://github.com/ClickHouse/ClickHouse/pull/26423) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). + +#### New Feature + +* Implementation of short circuit function evaluation, closes [#12587](https://github.com/ClickHouse/ClickHouse/issues/12587). Add settings `short_circuit_function_evaluation` to configure short circuit function evaluation. [#23367](https://github.com/ClickHouse/ClickHouse/pull/23367) ([Kruglov Pavel](https://github.com/Avogar)). +* Add support for INTERSECT, EXCEPT, ANY, ALL operators. [#24757](https://github.com/ClickHouse/ClickHouse/pull/24757) ([Kirill Ershov](https://github.com/zdikov)). ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add support for encryption at the virtual file system level (data encryption at rest) using AES-CTR algorithm. [#24206](https://github.com/ClickHouse/ClickHouse/pull/24206) ([Latysheva Alexandra](https://github.com/alexelex)). ([Vitaly Baranov](https://github.com/vitlibar)) [#26733](https://github.com/ClickHouse/ClickHouse/pull/26733) [#26377](https://github.com/ClickHouse/ClickHouse/pull/26377) [#26465](https://github.com/ClickHouse/ClickHouse/pull/26465). +* Added natural language processing (NLP) functions for tokenization, stemming, lemmatizing and search in synonyms extensions. [#24997](https://github.com/ClickHouse/ClickHouse/pull/24997) ([Nikolay Degterinsky](https://github.com/evillique)). +* Added integration with S2 geometry library. [#24980](https://github.com/ClickHouse/ClickHouse/pull/24980) ([Andr0901](https://github.com/Andr0901)). ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)). +* Add SQLite table engine, table function, database engine. [#24194](https://github.com/ClickHouse/ClickHouse/pull/24194) ([Arslan Gumerov](https://github.com/g-arslan)). ([Kseniia Sumarokova](https://github.com/kssenii)). +* Added support for custom query for `MySQL`, `PostgreSQL`, `ClickHouse`, `JDBC`, `Cassandra` dictionary source. Closes [#1270](https://github.com/ClickHouse/ClickHouse/issues/1270). [#26995](https://github.com/ClickHouse/ClickHouse/pull/26995) ([Maksim Kita](https://github.com/kitaisreal)). +* Introduce syntax for here documents. Example `SELECT $doc$ VALUE $doc$`. [#26671](https://github.com/ClickHouse/ClickHouse/pull/26671) ([Maksim Kita](https://github.com/kitaisreal)). +* Add shared (replicated) storage of user, roles, row policies, quotas and settings profiles through ZooKeeper. [#27426](https://github.com/ClickHouse/ClickHouse/pull/27426) ([Kevin Michel](https://github.com/kmichel-aiven)). +* Add compression for `INTO OUTFILE` that automatically choose compression algorithm. Closes [#3473](https://github.com/ClickHouse/ClickHouse/issues/3473). [#27134](https://github.com/ClickHouse/ClickHouse/pull/27134) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Add `INSERT ... FROM INFILE` similarly to `SELECT ... INTO OUTFILE`. [#27655](https://github.com/ClickHouse/ClickHouse/pull/27655) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Added `complex_key_range_hashed` dictionary. Closes [#22029](https://github.com/ClickHouse/ClickHouse/issues/22029). [#27629](https://github.com/ClickHouse/ClickHouse/pull/27629) ([Maksim Kita](https://github.com/kitaisreal)). +* Support expressions in JOIN ON section. Close [#21868](https://github.com/ClickHouse/ClickHouse/issues/21868). [#24420](https://github.com/ClickHouse/ClickHouse/pull/24420) ([Vladimir C](https://github.com/vdimir)). +* When client connects to server, it receives information about all warnings that are already were collected by server. (It can be disabled by using option `--no-warnings`). Add `system.warnings` table to collect warnings about server configuration. [#26246](https://github.com/ClickHouse/ClickHouse/pull/26246) ([Filatenkov Artur](https://github.com/FArthur-cmd)). [#26282](https://github.com/ClickHouse/ClickHouse/pull/26282) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Allow using constant expressions from with and select in aggregate function parameters. Close [#10945](https://github.com/ClickHouse/ClickHouse/issues/10945). [#27531](https://github.com/ClickHouse/ClickHouse/pull/27531) ([abel-cheng](https://github.com/abel-cheng)). +* Add `tupleToNameValuePairs`, a function that turns a named tuple into an array of pairs. [#27505](https://github.com/ClickHouse/ClickHouse/pull/27505) ([Braulio Valdivielso Martínez](https://github.com/BraulioVM)). +* Add support for `bzip2` compression method for import/export. Closes [#22428](https://github.com/ClickHouse/ClickHouse/issues/22428). [#27377](https://github.com/ClickHouse/ClickHouse/pull/27377) ([Nikolay Degterinsky](https://github.com/evillique)). +* Added `bitmapSubsetOffsetLimit(bitmap, offset, cardinality_limit)` function. It creates a subset of bitmap limit the results to `cardinality_limit` with offset of `offset`. [#27234](https://github.com/ClickHouse/ClickHouse/pull/27234) ([DHBin](https://github.com/DHBin)). +* Add column `default_database` to `system.users`. [#27054](https://github.com/ClickHouse/ClickHouse/pull/27054) ([kevin wan](https://github.com/MaxWk)). +* Supported `cluster` macros inside table functions 'cluster' and 'clusterAllReplicas'. [#26913](https://github.com/ClickHouse/ClickHouse/pull/26913) ([polyprogrammist](https://github.com/PolyProgrammist)). +* Add new functions `currentRoles()`, `enabledRoles()`, `defaultRoles()`. [#26780](https://github.com/ClickHouse/ClickHouse/pull/26780) ([Vitaly Baranov](https://github.com/vitlibar)). +* New functions `currentProfiles()`, `enabledProfiles()`, `defaultProfiles()`. [#26714](https://github.com/ClickHouse/ClickHouse/pull/26714) ([Vitaly Baranov](https://github.com/vitlibar)). +* Add functions that return (initial_)query_id of the current query. This closes [#23682](https://github.com/ClickHouse/ClickHouse/issues/23682). [#26410](https://github.com/ClickHouse/ClickHouse/pull/26410) ([Alexey Boykov](https://github.com/mathalex)). +* Add `REPLACE GRANT` feature. [#26384](https://github.com/ClickHouse/ClickHouse/pull/26384) ([Caspian](https://github.com/Cas-pian)). +* Implement window function `nth_value(expr, N)` that returns the value of the Nth row of the window frame. [#26334](https://github.com/ClickHouse/ClickHouse/pull/26334) ([Zuo, RuoYu](https://github.com/ryzuo)). +* `EXPLAIN` query now has `EXPLAIN ESTIMATE ...` mode that will show information about read rows, marks and parts from MergeTree tables. Closes [#23941](https://github.com/ClickHouse/ClickHouse/issues/23941). [#26131](https://github.com/ClickHouse/ClickHouse/pull/26131) ([fastio](https://github.com/fastio)). +* Added `system.zookeeper_log` table. All actions of ZooKeeper client are logged into this table. Implements [#25449](https://github.com/ClickHouse/ClickHouse/issues/25449). [#26129](https://github.com/ClickHouse/ClickHouse/pull/26129) ([tavplubix](https://github.com/tavplubix)). +* Zero-copy replication for `ReplicatedMergeTree` over `HDFS` storage. [#25918](https://github.com/ClickHouse/ClickHouse/pull/25918) ([Zhichang Yu](https://github.com/yuzhichang)). +* Allow to insert Nested type as array of structs in `Arrow`, `ORC` and `Parquet` input format. [#25902](https://github.com/ClickHouse/ClickHouse/pull/25902) ([Kruglov Pavel](https://github.com/Avogar)). +* Add a new datatype `Date32` (store data as Int32), support date range same with `DateTime64` support load parquet date32 to ClickHouse `Date32` Add new function `toDate32` like `toDate`. [#25774](https://github.com/ClickHouse/ClickHouse/pull/25774) ([LiuNeng](https://github.com/liuneng1994)). +* Allow setting default database for users. [#25268](https://github.com/ClickHouse/ClickHouse/issues/25268). [#25687](https://github.com/ClickHouse/ClickHouse/pull/25687) ([kevin wan](https://github.com/MaxWk)). +* Add an optional parameter to `MongoDB` engine to accept connection string options and support SSL connection. Closes [#21189](https://github.com/ClickHouse/ClickHouse/issues/21189). Closes [#21041](https://github.com/ClickHouse/ClickHouse/issues/21041). [#22045](https://github.com/ClickHouse/ClickHouse/pull/22045) ([Omar Bazaraa](https://github.com/OmarBazaraa)). + +#### Experimental Feature + +* Added a compression codec `AES_128_GCM_SIV` which encrypts columns instead of compressing them. [#19896](https://github.com/ClickHouse/ClickHouse/pull/19896) ([PHO](https://github.com/depressed-pho)). Will be rewritten, do not use. +* Rename `MaterializeMySQL` to `MaterializedMySQL`. [#26822](https://github.com/ClickHouse/ClickHouse/pull/26822) ([tavplubix](https://github.com/tavplubix)). + +#### Performance Improvement + +* Improve the performance of fast queries when `max_execution_time = 0` by reducing the number of `clock_gettime` system calls. [#27325](https://github.com/ClickHouse/ClickHouse/pull/27325) ([filimonov](https://github.com/filimonov)). +* Specialize date time related comparison to achieve better performance. This fixes [#27083](https://github.com/ClickHouse/ClickHouse/issues/27083) . [#27122](https://github.com/ClickHouse/ClickHouse/pull/27122) ([Amos Bird](https://github.com/amosbird)). +* Share file descriptors in concurrent reads of the same files. There is no noticeable performance difference on Linux. But the number of opened files will be significantly (10..100 times) lower on typical servers and it makes operations easier. See [#26214](https://github.com/ClickHouse/ClickHouse/issues/26214). [#26768](https://github.com/ClickHouse/ClickHouse/pull/26768) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Improve latency of short queries, that require reading from tables with large number of columns. [#26371](https://github.com/ClickHouse/ClickHouse/pull/26371) ([Anton Popov](https://github.com/CurtizJ)). +* Don't build sets for indices when analyzing a query. [#26365](https://github.com/ClickHouse/ClickHouse/pull/26365) ([Raúl Marín](https://github.com/Algunenano)). +* Vectorize the SUM of Nullable integer types with native representation ([David Manzanares](https://github.com/davidmanzanares), [Raúl Marín](https://github.com/Algunenano)). [#26248](https://github.com/ClickHouse/ClickHouse/pull/26248) ([Raúl Marín](https://github.com/Algunenano)). +* Compile expressions involving columns with `Enum` types. [#26237](https://github.com/ClickHouse/ClickHouse/pull/26237) ([Maksim Kita](https://github.com/kitaisreal)). +* Compile aggregate functions `groupBitOr`, `groupBitAnd`, `groupBitXor`. [#26161](https://github.com/ClickHouse/ClickHouse/pull/26161) ([Maksim Kita](https://github.com/kitaisreal)). +* Improved memory usage with better block size prediction when reading empty DEFAULT columns. Closes [#17317](https://github.com/ClickHouse/ClickHouse/issues/17317). [#25917](https://github.com/ClickHouse/ClickHouse/pull/25917) ([Vladimir Chebotarev](https://github.com/excitoon)). +* Reduce memory usage and number of read rows in queries with `ORDER BY primary_key`. [#25721](https://github.com/ClickHouse/ClickHouse/pull/25721) ([Anton Popov](https://github.com/CurtizJ)). +* Enable `distributed_push_down_limit` by default. [#27104](https://github.com/ClickHouse/ClickHouse/pull/27104) ([Azat Khuzhin](https://github.com/azat)). +* Make `toTimeZone` monotonicity when timeZone is a constant value to support partition puring when use sql like:. [#26261](https://github.com/ClickHouse/ClickHouse/pull/26261) ([huangzhaowei](https://github.com/SaintBacchus)). + +#### Improvement + +* Mark window functions as ready for general use. Remove the `allow_experimental_window_functions` setting. [#27184](https://github.com/ClickHouse/ClickHouse/pull/27184) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Improve compatibility with non-whole-minute timezone offsets. [#27080](https://github.com/ClickHouse/ClickHouse/pull/27080) ([Raúl Marín](https://github.com/Algunenano)). +* If file descriptor in `File` table is regular file - allow to read multiple times from it. It allows `clickhouse-local` to read multiple times from stdin (with multiple SELECT queries or subqueries) if stdin is a regular file like `clickhouse-local --query "SELECT * FROM table UNION ALL SELECT * FROM table" ... < file`. This closes [#11124](https://github.com/ClickHouse/ClickHouse/issues/11124). Co-authored with ([alexey-milovidov](https://github.com/alexey-milovidov)). [#25960](https://github.com/ClickHouse/ClickHouse/pull/25960) ([BoloniniD](https://github.com/BoloniniD)). +* Remove duplicate index analysis and avoid possible invalid limit checks during projection analysis. [#27742](https://github.com/ClickHouse/ClickHouse/pull/27742) ([Amos Bird](https://github.com/amosbird)). +* Enable query parameters to be passed in the body of HTTP requests. [#27706](https://github.com/ClickHouse/ClickHouse/pull/27706) ([Hermano Lustosa](https://github.com/hllustosa)). +* Disallow `arrayJoin` on partition expressions. [#27648](https://github.com/ClickHouse/ClickHouse/pull/27648) ([Raúl Marín](https://github.com/Algunenano)). +* Log client IP address if authentication fails. [#27514](https://github.com/ClickHouse/ClickHouse/pull/27514) ([Misko Lee](https://github.com/imiskolee)). +* Use bytes instead of strings for binary data in the GRPC protocol. [#27431](https://github.com/ClickHouse/ClickHouse/pull/27431) ([Vitaly Baranov](https://github.com/vitlibar)). +* Send response with error message if HTTP port is not set and user tries to send HTTP request to TCP port. [#27385](https://github.com/ClickHouse/ClickHouse/pull/27385) ([Braulio Valdivielso Martínez](https://github.com/BraulioVM)). +* Add `_CAST` function for internal usage, which will not preserve type nullability, but non-internal cast will preserve according to setting `cast_keep_nullable`. Closes [#12636](https://github.com/ClickHouse/ClickHouse/issues/12636). [#27382](https://github.com/ClickHouse/ClickHouse/pull/27382) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add setting `log_formatted_queries` to log additional formatted query into `system.query_log`. It's useful for normalized query analysis because functions like `normalizeQuery` and `normalizeQueryKeepNames` don't parse/format queries in order to achieve better performance. [#27380](https://github.com/ClickHouse/ClickHouse/pull/27380) ([Amos Bird](https://github.com/amosbird)). +* Add two settings `max_hyperscan_regexp_length` and `max_hyperscan_regexp_total_length` to prevent huge regexp being used in hyperscan related functions, such as `multiMatchAny`. [#27378](https://github.com/ClickHouse/ClickHouse/pull/27378) ([Amos Bird](https://github.com/amosbird)). +* Memory consumed by bitmap aggregate functions now is taken into account for memory limits. This closes [#26555](https://github.com/ClickHouse/ClickHouse/issues/26555). [#27252](https://github.com/ClickHouse/ClickHouse/pull/27252) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Add new index data skipping minmax index format for proper Nullable support. [#27250](https://github.com/ClickHouse/ClickHouse/pull/27250) ([Azat Khuzhin](https://github.com/azat)). +* Add 10 seconds cache for S3 proxy resolver. [#27216](https://github.com/ClickHouse/ClickHouse/pull/27216) ([ianton-ru](https://github.com/ianton-ru)). +* Split global mutex into individual regexp construction. This helps avoid huge regexp construction blocking other related threads. [#27211](https://github.com/ClickHouse/ClickHouse/pull/27211) ([Amos Bird](https://github.com/amosbird)). +* Support schema for PostgreSQL database engine. Closes [#27166](https://github.com/ClickHouse/ClickHouse/issues/27166). [#27198](https://github.com/ClickHouse/ClickHouse/pull/27198) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Track memory usage in clickhouse-client. [#27191](https://github.com/ClickHouse/ClickHouse/pull/27191) ([Filatenkov Artur](https://github.com/FArthur-cmd)). +* Try recording `query_kind` in `system.query_log` even when query fails to start. [#27182](https://github.com/ClickHouse/ClickHouse/pull/27182) ([Amos Bird](https://github.com/amosbird)). +* Added columns `replica_is_active` that maps replica name to is replica active status to table `system.replicas`. Closes [#27138](https://github.com/ClickHouse/ClickHouse/issues/27138). [#27180](https://github.com/ClickHouse/ClickHouse/pull/27180) ([Maksim Kita](https://github.com/kitaisreal)). +* Allow to pass query settings via server URI in Web UI. [#27177](https://github.com/ClickHouse/ClickHouse/pull/27177) ([kolsys](https://github.com/kolsys)). +* Add a new metric called `MaxPushedDDLEntryID` which is the maximum ddl entry id that current node push to zookeeper. [#27174](https://github.com/ClickHouse/ClickHouse/pull/27174) ([Fuwang Hu](https://github.com/fuwhu)). +* Improved the existence condition judgment and empty string node judgment when `clickhouse-keeper` creates znode. [#27125](https://github.com/ClickHouse/ClickHouse/pull/27125) ([小路](https://github.com/nicelulu)). +* Merge JOIN correctly handles empty set in the right. [#27078](https://github.com/ClickHouse/ClickHouse/pull/27078) ([Vladimir C](https://github.com/vdimir)). +* Now functions can be shard-level constants, which means if it's executed in the context of some distributed table, it generates a normal column, otherwise it produces a constant value. Notable functions are: `hostName()`, `tcpPort()`, `version()`, `buildId()`, `uptime()`, etc. [#27020](https://github.com/ClickHouse/ClickHouse/pull/27020) ([Amos Bird](https://github.com/amosbird)). +* Updated `extractAllGroupsHorizontal` - upper limit on the number of matches per row can be set via optional third argument. [#26961](https://github.com/ClickHouse/ClickHouse/pull/26961) ([Vasily Nemkov](https://github.com/Enmk)). +* Expose `RocksDB` statistics via system.rocksdb table. Read rocksdb options from ClickHouse config (`rocksdb...` keys). NOTE: ClickHouse does not rely on RocksDB, it is just one of the additional integration storage engines. [#26821](https://github.com/ClickHouse/ClickHouse/pull/26821) ([Azat Khuzhin](https://github.com/azat)). +* Less verbose internal RocksDB logs. NOTE: ClickHouse does not rely on RocksDB, it is just one of the additional integration storage engines. This closes [#26252](https://github.com/ClickHouse/ClickHouse/issues/26252). [#26789](https://github.com/ClickHouse/ClickHouse/pull/26789) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Changing default roles affects new sessions only. [#26759](https://github.com/ClickHouse/ClickHouse/pull/26759) ([Vitaly Baranov](https://github.com/vitlibar)). +* Watchdog is disabled in docker by default. Fix for not handling ctrl+c. [#26757](https://github.com/ClickHouse/ClickHouse/pull/26757) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* `SET PROFILE` now applies constraints too if they're set for a passed profile. [#26730](https://github.com/ClickHouse/ClickHouse/pull/26730) ([Vitaly Baranov](https://github.com/vitlibar)). +* Improve handling of `KILL QUERY` requests. [#26675](https://github.com/ClickHouse/ClickHouse/pull/26675) ([Raúl Marín](https://github.com/Algunenano)). +* `mapPopulatesSeries` function supports `Map` type. [#26663](https://github.com/ClickHouse/ClickHouse/pull/26663) ([Ildus Kurbangaliev](https://github.com/ildus)). +* Fix excessive (x2) connect attempts with `skip_unavailable_shards`. [#26658](https://github.com/ClickHouse/ClickHouse/pull/26658) ([Azat Khuzhin](https://github.com/azat)). +* Avoid hanging `clickhouse-benchmark` if connection fails (i.e. on EMFILE). [#26656](https://github.com/ClickHouse/ClickHouse/pull/26656) ([Azat Khuzhin](https://github.com/azat)). +* Allow more threads to be used by the Kafka engine. [#26642](https://github.com/ClickHouse/ClickHouse/pull/26642) ([feihengye](https://github.com/feihengye)). +* Add round-robin support for `clickhouse-benchmark` (it does not differ from the regular multi host/port run except for statistics report). [#26607](https://github.com/ClickHouse/ClickHouse/pull/26607) ([Azat Khuzhin](https://github.com/azat)). +* Executable dictionaries (`executable`, `executable_pool`) enable creation with DDL query using `clickhouse-local`. Closes [#22355](https://github.com/ClickHouse/ClickHouse/issues/22355). [#26510](https://github.com/ClickHouse/ClickHouse/pull/26510) ([Maksim Kita](https://github.com/kitaisreal)). +* Set client query kind for `mysql` and `postgresql` compatibility protocol handlers. [#26498](https://github.com/ClickHouse/ClickHouse/pull/26498) ([anneji-dev](https://github.com/anneji-dev)). +* Apply `LIMIT` on the shards for queries like `SELECT * FROM dist ORDER BY key LIMIT 10` w/ `distributed_push_down_limit=1`. Avoid running `Distinct`/`LIMIT BY` steps for queries like `SELECT DISTINCT shading_key FROM dist ORDER BY key`. Now `distributed_push_down_limit` is respected by `optimize_distributed_group_by_sharding_key` optimization. [#26466](https://github.com/ClickHouse/ClickHouse/pull/26466) ([Azat Khuzhin](https://github.com/azat)). +* Updated protobuf to 3.17.3. Changelogs are available on https://github.com/protocolbuffers/protobuf/releases. [#26424](https://github.com/ClickHouse/ClickHouse/pull/26424) ([Ilya Yatsishin](https://github.com/qoega)). +* Enable `use_hedged_requests` setting that allows to mitigate tail latencies on large clusters. [#26380](https://github.com/ClickHouse/ClickHouse/pull/26380) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Improve behaviour with non-existing host in user allowed host list. [#26368](https://github.com/ClickHouse/ClickHouse/pull/26368) ([ianton-ru](https://github.com/ianton-ru)). +* Add ability to set `Distributed` directory monitor settings via CREATE TABLE (i.e. `CREATE TABLE dist (key Int) Engine=Distributed(cluster, db, table) SETTINGS monitor_batch_inserts=1` and similar). [#26336](https://github.com/ClickHouse/ClickHouse/pull/26336) ([Azat Khuzhin](https://github.com/azat)). +* Save server address in history URLs in web UI if it differs from the origin of web UI. This closes [#26044](https://github.com/ClickHouse/ClickHouse/issues/26044). [#26322](https://github.com/ClickHouse/ClickHouse/pull/26322) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Add events to profile calls to `sleep` / `sleepEachRow`. [#26320](https://github.com/ClickHouse/ClickHouse/pull/26320) ([Raúl Marín](https://github.com/Algunenano)). +* Allow to reuse connections of shards among different clusters. It also avoids creating new connections when using `cluster` table function. [#26318](https://github.com/ClickHouse/ClickHouse/pull/26318) ([Amos Bird](https://github.com/amosbird)). +* Control the execution period of clear old temporary directories by parameter with default value. [#26212](https://github.com/ClickHouse/ClickHouse/issues/26212). [#26313](https://github.com/ClickHouse/ClickHouse/pull/26313) ([fastio](https://github.com/fastio)). +* Add a setting `function_range_max_elements_in_block` to tune the safety threshold for data volume generated by function `range`. This closes [#26303](https://github.com/ClickHouse/ClickHouse/issues/26303). [#26305](https://github.com/ClickHouse/ClickHouse/pull/26305) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Check hash function at table creation, not at sampling. Add settings for MergeTree, if someone create a table with incorrect sampling column but sampling never be used, disable this settings for starting the server without exception. [#26256](https://github.com/ClickHouse/ClickHouse/pull/26256) ([zhaoyu](https://github.com/zxc111)). +* Added `output_format_avro_string_column_pattern` setting to put specified String columns to Avro as string instead of default bytes. Implements [#22414](https://github.com/ClickHouse/ClickHouse/issues/22414). [#26245](https://github.com/ClickHouse/ClickHouse/pull/26245) ([Ilya Golshtein](https://github.com/ilejn)). +* Add information about column sizes in `system.columns` table for `Log` and `TinyLog` tables. This closes [#9001](https://github.com/ClickHouse/ClickHouse/issues/9001). [#26241](https://github.com/ClickHouse/ClickHouse/pull/26241) ([Nikolay Degterinsky](https://github.com/evillique)). +* Don't throw exception when querying `system.detached_parts` table if there is custom disk configuration and `detached` directory does not exist on some disks. This closes [#26078](https://github.com/ClickHouse/ClickHouse/issues/26078). [#26236](https://github.com/ClickHouse/ClickHouse/pull/26236) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Check for non-deterministic functions in keys, including constant expressions like `now()`, `today()`. This closes [#25875](https://github.com/ClickHouse/ClickHouse/issues/25875). This closes [#11333](https://github.com/ClickHouse/ClickHouse/issues/11333). [#26235](https://github.com/ClickHouse/ClickHouse/pull/26235) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* convert timestamp and timestamptz data types to `DateTime64` in PostgreSQL table engine. [#26234](https://github.com/ClickHouse/ClickHouse/pull/26234) ([jasine](https://github.com/jasine)). +* Apply aggressive IN index analysis for projections so that better projection candidate can be selected. [#26218](https://github.com/ClickHouse/ClickHouse/pull/26218) ([Amos Bird](https://github.com/amosbird)). +* Remove GLOBAL keyword for IN when scalar function is passed. In previous versions, if user specified `GLOBAL IN f(x)` exception was thrown. [#26217](https://github.com/ClickHouse/ClickHouse/pull/26217) ([Amos Bird](https://github.com/amosbird)). +* Add error id (like `BAD_ARGUMENTS`) to exception messages. This closes [#25862](https://github.com/ClickHouse/ClickHouse/issues/25862). [#26172](https://github.com/ClickHouse/ClickHouse/pull/26172) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix incorrect output with --progress option for clickhouse-local. Progress bar will be cleared once it gets to 100% - same as it is done for clickhouse-client. Closes [#17484](https://github.com/ClickHouse/ClickHouse/issues/17484). [#26128](https://github.com/ClickHouse/ClickHouse/pull/26128) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Add `merge_selecting_sleep_ms` setting. [#26120](https://github.com/ClickHouse/ClickHouse/pull/26120) ([lthaooo](https://github.com/lthaooo)). +* Remove complicated usage of Linux AIO with one block readahead and replace it with plain simple synchronous IO with O_DIRECT. In previous versions, the setting `min_bytes_to_use_direct_io` may not work correctly if `max_threads` is greater than one. Reading with direct IO (that is disabled by default for queries and enabled by default for large merges) will work in less efficient way. This closes [#25997](https://github.com/ClickHouse/ClickHouse/issues/25997). [#26003](https://github.com/ClickHouse/ClickHouse/pull/26003) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Flush `Distributed` table on `REPLACE TABLE` query. Resolves [#24566](https://github.com/ClickHouse/ClickHouse/issues/24566) - Do not replace (or create) table on `[CREATE OR] REPLACE TABLE ... AS SELECT` query if insertion into new table fails. Resolves [#23175](https://github.com/ClickHouse/ClickHouse/issues/23175). [#25895](https://github.com/ClickHouse/ClickHouse/pull/25895) ([tavplubix](https://github.com/tavplubix)). +* Add `views` column to system.query_log containing the names of the (materialized or live) views executed by the query. Adds a new log table (`system.query_views_log`) that contains information about each view executed during a query. Modifies view execution: When an exception is thrown while executing a view, any view that has already startedwill continue running until it finishes. This used to be the behaviour under parallel_view_processing=true and now it's always the same behaviour. - Dependent views now report reading progress to the context. [#25714](https://github.com/ClickHouse/ClickHouse/pull/25714) ([Raúl Marín](https://github.com/Algunenano)). +* Do connection draining asynchonously upon finishing executing distributed queries. A new server setting is added `max_threads_for_connection_collector` which specifies the number of workers to recycle connections in background. If the pool is full, connection will be drained synchronously but a bit different than before: It's drained after we send EOS to client, query will succeed immediately after receiving enough data, and any exception will be logged instead of throwing to the client. Added setting `drain_timeout` (3 seconds by default). Connection draining will disconnect upon timeout. [#25674](https://github.com/ClickHouse/ClickHouse/pull/25674) ([Amos Bird](https://github.com/amosbird)). +* Support for multiple includes in configuration. It is possible to include users configuration, remote servers configuration from multiple sources. Simply place `` element with `from_zk`, `from_env` or `incl` attribute and it will be replaced with the substitution. [#24404](https://github.com/ClickHouse/ClickHouse/pull/24404) ([nvartolomei](https://github.com/nvartolomei)). +* Fix multiple block insertion into distributed table with `insert_distributed_one_random_shard = 1`. This is a marginal feature. Mark as improvement. [#23140](https://github.com/ClickHouse/ClickHouse/pull/23140) ([Amos Bird](https://github.com/amosbird)). +* Support `LowCardinality` and `FixedString` keys/values for `Map` type. [#21543](https://github.com/ClickHouse/ClickHouse/pull/21543) ([hexiaoting](https://github.com/hexiaoting)). +* Enable reloading of local disk config. [#19526](https://github.com/ClickHouse/ClickHouse/pull/19526) ([taiyang-li](https://github.com/taiyang-li)). +* Now KeyConditions can correctly skip nullable keys, including `isNull` and `isNotNull`. https://github.com/ClickHouse/ClickHouse/pull/12433. [#12455](https://github.com/ClickHouse/ClickHouse/pull/12455) ([Amos Bird](https://github.com/amosbird)). + +#### Bug Fix + +* Fix a couple of bugs that may cause replicas to diverge. [#27808](https://github.com/ClickHouse/ClickHouse/pull/27808) ([tavplubix](https://github.com/tavplubix)). +* Fix a rare bug in `DROP PART` which can lead to the error `Unexpected merged part intersects drop range`. [#27807](https://github.com/ClickHouse/ClickHouse/pull/27807) ([alesapin](https://github.com/alesapin)). +* Prevent crashes for some formats when NULL (tombstone) message was coming from Kafka. Closes [#19255](https://github.com/ClickHouse/ClickHouse/issues/19255). [#27794](https://github.com/ClickHouse/ClickHouse/pull/27794) ([filimonov](https://github.com/filimonov)). +* Fix column filtering with union distinct in subquery. Closes [#27578](https://github.com/ClickHouse/ClickHouse/issues/27578). [#27689](https://github.com/ClickHouse/ClickHouse/pull/27689) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix bad type cast when functions like `arrayHas` are applied to arrays of LowCardinality of Nullable of different non-numeric types like `DateTime` and `DateTime64`. In previous versions bad cast occurs. In new version it will lead to exception. This closes [#26330](https://github.com/ClickHouse/ClickHouse/issues/26330). [#27682](https://github.com/ClickHouse/ClickHouse/pull/27682) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix postgresql table function resulting in non-closing connections. Closes [#26088](https://github.com/ClickHouse/ClickHouse/issues/26088). [#27662](https://github.com/ClickHouse/ClickHouse/pull/27662) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fixed another case of `Unexpected merged part ... intersecting drop range ...` error. [#27656](https://github.com/ClickHouse/ClickHouse/pull/27656) ([tavplubix](https://github.com/tavplubix)). +* Fix an error with aliased column in `Distributed` table. [#27652](https://github.com/ClickHouse/ClickHouse/pull/27652) ([Vladimir C](https://github.com/vdimir)). +* After setting `max_memory_usage*` to non-zero value it was not possible to reset it back to 0 (unlimited). It's fixed. [#27638](https://github.com/ClickHouse/ClickHouse/pull/27638) ([tavplubix](https://github.com/tavplubix)). +* Fixed underflow of the time value when constructing it from components. Closes [#27193](https://github.com/ClickHouse/ClickHouse/issues/27193). [#27605](https://github.com/ClickHouse/ClickHouse/pull/27605) ([Vasily Nemkov](https://github.com/Enmk)). +* Fix crash during projection materialization when some parts contain missing columns. This fixes [#27512](https://github.com/ClickHouse/ClickHouse/issues/27512). [#27528](https://github.com/ClickHouse/ClickHouse/pull/27528) ([Amos Bird](https://github.com/amosbird)). +* fix metric `BackgroundMessageBrokerSchedulePoolTask`, maybe mistyped. [#27452](https://github.com/ClickHouse/ClickHouse/pull/27452) ([Ben](https://github.com/benbiti)). +* Fix distributed queries with zero shards and aggregation. [#27427](https://github.com/ClickHouse/ClickHouse/pull/27427) ([Azat Khuzhin](https://github.com/azat)). +* Compatibility when `/proc/meminfo` does not contain KB suffix. [#27361](https://github.com/ClickHouse/ClickHouse/pull/27361) ([Mike Kot](https://github.com/myrrc)). +* Fix incorrect result for query with row-level security, PREWHERE and LowCardinality filter. Fixes [#27179](https://github.com/ClickHouse/ClickHouse/issues/27179). [#27329](https://github.com/ClickHouse/ClickHouse/pull/27329) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fixed incorrect validation of partition id for MergeTree tables that created with old syntax. [#27328](https://github.com/ClickHouse/ClickHouse/pull/27328) ([tavplubix](https://github.com/tavplubix)). +* Fix MySQL protocol when using parallel formats (CSV / TSV). [#27326](https://github.com/ClickHouse/ClickHouse/pull/27326) ([Raúl Marín](https://github.com/Algunenano)). +* Fix `Cannot find column` error for queries with sampling. Was introduced in [#24574](https://github.com/ClickHouse/ClickHouse/issues/24574). Fixes [#26522](https://github.com/ClickHouse/ClickHouse/issues/26522). [#27301](https://github.com/ClickHouse/ClickHouse/pull/27301) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix errors like `Expected ColumnLowCardinality, gotUInt8` or `Bad cast from type DB::ColumnVector to DB::ColumnLowCardinality` for some queries with `LowCardinality` in `PREWHERE`. And more importantly, fix the lack of whitespace in the error message. Fixes [#23515](https://github.com/ClickHouse/ClickHouse/issues/23515). [#27298](https://github.com/ClickHouse/ClickHouse/pull/27298) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix `distributed_group_by_no_merge = 2` with `distributed_push_down_limit = 1` or `optimize_distributed_group_by_sharding_key = 1` with `LIMIT BY` and `LIMIT OFFSET`. [#27249](https://github.com/ClickHouse/ClickHouse/pull/27249) ([Azat Khuzhin](https://github.com/azat)). These are obscure combination of settings that no one is using. +* Fix mutation stuck on invalid partitions in non-replicated MergeTree. [#27248](https://github.com/ClickHouse/ClickHouse/pull/27248) ([Azat Khuzhin](https://github.com/azat)). +* In case of ambiguity, lambda functions prefer its arguments to other aliases or identifiers. [#27235](https://github.com/ClickHouse/ClickHouse/pull/27235) ([Raúl Marín](https://github.com/Algunenano)). +* Fix column structure in merge join, close [#27091](https://github.com/ClickHouse/ClickHouse/issues/27091). [#27217](https://github.com/ClickHouse/ClickHouse/pull/27217) ([Vladimir C](https://github.com/vdimir)). +* In rare cases `system.detached_parts` table might contain incorrect information for some parts, it's fixed. Fixes [#27114](https://github.com/ClickHouse/ClickHouse/issues/27114). [#27183](https://github.com/ClickHouse/ClickHouse/pull/27183) ([tavplubix](https://github.com/tavplubix)). +* Fix uninitialized memory in functions `multiSearch*` with empty array, close [#27169](https://github.com/ClickHouse/ClickHouse/issues/27169). [#27181](https://github.com/ClickHouse/ClickHouse/pull/27181) ([Vladimir C](https://github.com/vdimir)). +* Fix synchronization in GRPCServer. This PR fixes [#27024](https://github.com/ClickHouse/ClickHouse/issues/27024). [#27064](https://github.com/ClickHouse/ClickHouse/pull/27064) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fixed `cache`, `complex_key_cache`, `ssd_cache`, `complex_key_ssd_cache` configuration parsing. Options `allow_read_expired_keys`, `max_update_queue_size`, `update_queue_push_timeout_milliseconds`, `query_wait_timeout_milliseconds` were not parsed for dictionaries with non `cache` type. [#27032](https://github.com/ClickHouse/ClickHouse/pull/27032) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix possible mutation stack due to race with DROP_RANGE. [#27002](https://github.com/ClickHouse/ClickHouse/pull/27002) ([Azat Khuzhin](https://github.com/azat)). +* Now partition ID in queries like `ALTER TABLE ... PARTITION ID xxx` validates for correctness. Fixes [#25718](https://github.com/ClickHouse/ClickHouse/issues/25718). [#26963](https://github.com/ClickHouse/ClickHouse/pull/26963) ([alesapin](https://github.com/alesapin)). +* Fix "Unknown column name" error with multiple JOINs in some cases, close [#26899](https://github.com/ClickHouse/ClickHouse/issues/26899). [#26957](https://github.com/ClickHouse/ClickHouse/pull/26957) ([Vladimir C](https://github.com/vdimir)). +* Fix reading of custom TLDs (stops processing with lower buffer or bigger file). [#26948](https://github.com/ClickHouse/ClickHouse/pull/26948) ([Azat Khuzhin](https://github.com/azat)). +* Fix error `Missing columns: 'xxx'` when `DEFAULT` column references other non materialized column without `DEFAULT` expression. Fixes [#26591](https://github.com/ClickHouse/ClickHouse/issues/26591). [#26900](https://github.com/ClickHouse/ClickHouse/pull/26900) ([alesapin](https://github.com/alesapin)). +* Fix loading of dictionary keys in `library-bridge` for `library` dictionary source. [#26834](https://github.com/ClickHouse/ClickHouse/pull/26834) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Aggregate function parameters might be lost when applying some combinators causing exceptions like `Conversion from AggregateFunction(topKArray, Array(String)) to AggregateFunction(topKArray(10), Array(String)) is not supported`. It's fixed. Fixes [#26196](https://github.com/ClickHouse/ClickHouse/issues/26196) and [#26433](https://github.com/ClickHouse/ClickHouse/issues/26433). [#26814](https://github.com/ClickHouse/ClickHouse/pull/26814) ([tavplubix](https://github.com/tavplubix)). +* Add `event_time_microseconds` value for `REMOVE_PART` in `system.part_log`. In previous versions is was not set. [#26720](https://github.com/ClickHouse/ClickHouse/pull/26720) ([Azat Khuzhin](https://github.com/azat)). +* Do not remove data on ReplicatedMergeTree table shutdown to avoid creating data to metadata inconsistency. [#26716](https://github.com/ClickHouse/ClickHouse/pull/26716) ([nvartolomei](https://github.com/nvartolomei)). +* Sometimes `SET ROLE` could work incorrectly, this PR fixes that. [#26707](https://github.com/ClickHouse/ClickHouse/pull/26707) ([Vitaly Baranov](https://github.com/vitlibar)). +* Some fixes for parallel formatting (https://github.com/ClickHouse/ClickHouse/issues/26694). [#26703](https://github.com/ClickHouse/ClickHouse/pull/26703) ([Raúl Marín](https://github.com/Algunenano)). +* Fix potential nullptr dereference in window functions. This fixes [#25276](https://github.com/ClickHouse/ClickHouse/issues/25276). [#26668](https://github.com/ClickHouse/ClickHouse/pull/26668) ([Alexander Kuzmenkov](https://github.com/akuzm)). +* Fix clickhouse-client history file conversion (when upgrading from the format of 3 years old version of clickhouse-client) if file is empty. [#26589](https://github.com/ClickHouse/ClickHouse/pull/26589) ([Azat Khuzhin](https://github.com/azat)). +* Fix incorrect function names of groupBitmapAnd/Or/Xor (can be displayed in some occasions). This fixes. [#26557](https://github.com/ClickHouse/ClickHouse/pull/26557) ([Amos Bird](https://github.com/amosbird)). +* Update `chown` cmd check in clickhouse-server docker entrypoint. It fixes the bug that cluster pod restart failed (or timeout) on kubernetes. [#26545](https://github.com/ClickHouse/ClickHouse/pull/26545) ([Ky Li](https://github.com/Kylinrix)). +* Fix crash in `RabbitMQ` shutdown in case `RabbitMQ` setup was not started. Closes [#26504](https://github.com/ClickHouse/ClickHouse/issues/26504). [#26529](https://github.com/ClickHouse/ClickHouse/pull/26529) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix issues with `CREATE DICTIONARY` query if dictionary name or database name was quoted. Closes [#26491](https://github.com/ClickHouse/ClickHouse/issues/26491). [#26508](https://github.com/ClickHouse/ClickHouse/pull/26508) ([Maksim Kita](https://github.com/kitaisreal)). +* Fix broken column name resolution after rewriting column aliases. This fixes [#26432](https://github.com/ClickHouse/ClickHouse/issues/26432). [#26475](https://github.com/ClickHouse/ClickHouse/pull/26475) ([Amos Bird](https://github.com/amosbird)). +* Fix some fuzzed msan crash. Fixes [#22517](https://github.com/ClickHouse/ClickHouse/issues/22517). [#26428](https://github.com/ClickHouse/ClickHouse/pull/26428) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix infinite non joined block stream in `partial_merge_join` close [#26325](https://github.com/ClickHouse/ClickHouse/issues/26325). [#26374](https://github.com/ClickHouse/ClickHouse/pull/26374) ([Vladimir C](https://github.com/vdimir)). +* Fix possible crash when login as dropped user. This PR fixes [#26073](https://github.com/ClickHouse/ClickHouse/issues/26073). [#26363](https://github.com/ClickHouse/ClickHouse/pull/26363) ([Vitaly Baranov](https://github.com/vitlibar)). +* Fix `optimize_distributed_group_by_sharding_key` for multiple columns (leads to incorrect result w/ `optimize_skip_unused_shards=1`/`allow_nondeterministic_optimize_skip_unused_shards=1` and multiple columns in sharding key expression). [#26353](https://github.com/ClickHouse/ClickHouse/pull/26353) ([Azat Khuzhin](https://github.com/azat)). +* Fixed rare bug in lost replica recovery that may cause replicas to diverge. [#26321](https://github.com/ClickHouse/ClickHouse/pull/26321) ([tavplubix](https://github.com/tavplubix)). +* Fix zstd decompression (for import/export in zstd framing format that is unrelated to tables data) in case there are escape sequences at the end of internal buffer. Closes [#26013](https://github.com/ClickHouse/ClickHouse/issues/26013). [#26314](https://github.com/ClickHouse/ClickHouse/pull/26314) ([Kseniia Sumarokova](https://github.com/kssenii)). +* Fix logical error on join with totals, close [#26017](https://github.com/ClickHouse/ClickHouse/issues/26017). [#26250](https://github.com/ClickHouse/ClickHouse/pull/26250) ([Vladimir C](https://github.com/vdimir)). +* Remove excessive newline in `thread_name` column in `system.stack_trace` table. This fixes [#24124](https://github.com/ClickHouse/ClickHouse/issues/24124). [#26210](https://github.com/ClickHouse/ClickHouse/pull/26210) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix potential crash if more than one `untuple` expression is used. [#26179](https://github.com/ClickHouse/ClickHouse/pull/26179) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Don't throw exception in `toString` for Nullable Enum if Enum does not have a value for zero, close [#25806](https://github.com/ClickHouse/ClickHouse/issues/25806). [#26123](https://github.com/ClickHouse/ClickHouse/pull/26123) ([Vladimir C](https://github.com/vdimir)). +* Fixed incorrect `sequence_id` in MySQL protocol packets that ClickHouse sends on exception during query execution. It might cause MySQL client to reset connection to ClickHouse server. Fixes [#21184](https://github.com/ClickHouse/ClickHouse/issues/21184). [#26051](https://github.com/ClickHouse/ClickHouse/pull/26051) ([tavplubix](https://github.com/tavplubix)). +* Fix for the case that `cutToFirstSignificantSubdomainCustom()`/`cutToFirstSignificantSubdomainCustomWithWWW()`/`firstSignificantSubdomainCustom()` returns incorrect type for consts, and hence `optimize_skip_unused_shards` does not work:. [#26041](https://github.com/ClickHouse/ClickHouse/pull/26041) ([Azat Khuzhin](https://github.com/azat)). +* Fix possible mismatched header when using normal projection with prewhere. This fixes [#26020](https://github.com/ClickHouse/ClickHouse/issues/26020). [#26038](https://github.com/ClickHouse/ClickHouse/pull/26038) ([Amos Bird](https://github.com/amosbird)). +* Fix sharding_key from column w/o function for remote() (before `select * from remote('127.1', system.one, dummy)` leads to `Unknown column: dummy, there are only columns .` error). [#25824](https://github.com/ClickHouse/ClickHouse/pull/25824) ([Azat Khuzhin](https://github.com/azat)). +* Fixed `Not found column ...` and `Missing column ...` errors when selecting from `MaterializeMySQL`. Fixes [#23708](https://github.com/ClickHouse/ClickHouse/issues/23708), [#24830](https://github.com/ClickHouse/ClickHouse/issues/24830), [#25794](https://github.com/ClickHouse/ClickHouse/issues/25794). [#25822](https://github.com/ClickHouse/ClickHouse/pull/25822) ([tavplubix](https://github.com/tavplubix)). +* Fix `optimize_skip_unused_shards_rewrite_in` for non-UInt64 types (may select incorrect shards eventually or throw `Cannot infer type of an empty tuple` or `Function tuple requires at least one argument`). [#25798](https://github.com/ClickHouse/ClickHouse/pull/25798) ([Azat Khuzhin](https://github.com/azat)). + +#### Build/Testing/Packaging Improvement + +* Now we ran stateful and stateless tests in random timezones. Fixes [#12439](https://github.com/ClickHouse/ClickHouse/issues/12439). Reading String as DateTime and writing DateTime as String in Protobuf format now respect timezone. Reading UInt16 as DateTime in Arrow and Parquet formats now treat it as Date and then converts to DateTime with respect to DateTime's timezone, because Date is serialized in Arrow and Parquet as UInt16. GraphiteMergeTree now respect time zone for rounding of times. Fixes [#5098](https://github.com/ClickHouse/ClickHouse/issues/5098). Author: @alexey-milovidov. [#15408](https://github.com/ClickHouse/ClickHouse/pull/15408) ([alesapin](https://github.com/alesapin)). +* `clickhouse-test` supports SQL tests with [Jinja2](https://jinja.palletsprojects.com/en/3.0.x/templates/#synopsis) templates. [#26579](https://github.com/ClickHouse/ClickHouse/pull/26579) ([Vladimir C](https://github.com/vdimir)). +* Add support for build with `clang-13`. This closes [#27705](https://github.com/ClickHouse/ClickHouse/issues/27705). [#27714](https://github.com/ClickHouse/ClickHouse/pull/27714) ([alexey-milovidov](https://github.com/alexey-milovidov)). [#27777](https://github.com/ClickHouse/ClickHouse/pull/27777) ([Sergei Semin](https://github.com/syominsergey)) +* Add CMake options to build with or without specific CPU instruction set. This is for [#17469](https://github.com/ClickHouse/ClickHouse/issues/17469) and [#27509](https://github.com/ClickHouse/ClickHouse/issues/27509). [#27508](https://github.com/ClickHouse/ClickHouse/pull/27508) ([alexey-milovidov](https://github.com/alexey-milovidov)). +* Fix linking of auxiliar programs when using dynamic libraries. [#26958](https://github.com/ClickHouse/ClickHouse/pull/26958) ([Raúl Marín](https://github.com/Algunenano)). +* Update RocksDB to `2021-07-16` master. [#26411](https://github.com/ClickHouse/ClickHouse/pull/26411) ([alexey-milovidov](https://github.com/alexey-milovidov)). + + ### ClickHouse release v21.8, 2021-08-12 #### Upgrade Notes diff --git a/CMakeLists.txt b/CMakeLists.txt index de517b1b589..897111e66bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,16 +80,16 @@ include (cmake/find/ccache.cmake) # ccache ignore it. option(ENABLE_CHECK_HEAVY_BUILDS "Don't allow C++ translation units to compile too long or to take too much memory while compiling." OFF) if (ENABLE_CHECK_HEAVY_BUILDS) - # set DATA (since RSS does not work since 2.6.x+) to 2G + # set DATA (since RSS does not work since 2.6.x+) to 5G set (RLIMIT_DATA 5000000000) # set VIRT (RLIMIT_AS) to 10G (DATA*10) set (RLIMIT_AS 10000000000) - # set CPU time limit to 600 seconds - set (RLIMIT_CPU 600) + # set CPU time limit to 1000 seconds + set (RLIMIT_CPU 1000) # gcc10/gcc10/clang -fsanitize=memory is too heavy if (SANITIZE STREQUAL "memory" OR COMPILER_GCC) - set (RLIMIT_DATA 10000000000) + set (RLIMIT_DATA 10000000000) # 10G endif() set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=${RLIMIT_CPU} ${CMAKE_CXX_COMPILER_LAUNCHER}) @@ -165,6 +165,13 @@ if (COMPILER_CLANG) if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE") set(COMPILER_FLAGS "${COMPILER_FLAGS} -gdwarf-aranges") endif () + + if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 12.0.0) + if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -fuse-ctor-homing") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing") + endif() + endif() endif () # If turned `ON`, assumes the user has either the system GTest library or the bundled one. @@ -185,7 +192,7 @@ endif () # Make sure the final executable has symbols exported set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic") -find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-12" "llvm-objcopy-11" "llvm-objcopy-10" "llvm-objcopy-9" "llvm-objcopy-8" "objcopy") +find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-13" "llvm-objcopy-12" "llvm-objcopy-11" "llvm-objcopy-10" "llvm-objcopy-9" "llvm-objcopy-8" "objcopy") if (NOT OBJCOPY_PATH AND OS_DARWIN) find_program (BREW_PATH NAMES "brew") @@ -372,7 +379,7 @@ if (COMPILER_CLANG) endif () # Always prefer llvm tools when using clang. For instance, we cannot use GNU ar when llvm LTO is enabled - find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8") + find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-13" "llvm-ar-12" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8") if (LLVM_AR_PATH) message(STATUS "Using llvm-ar: ${LLVM_AR_PATH}.") @@ -381,7 +388,7 @@ if (COMPILER_CLANG) message(WARNING "Cannot find llvm-ar. System ar will be used instead. It does not work with ThinLTO.") endif () - find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9" "llvm-ranlib-8") + find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-13" "llvm-ranlib-12" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9" "llvm-ranlib-8") if (LLVM_RANLIB_PATH) message(STATUS "Using llvm-ranlib: ${LLVM_RANLIB_PATH}.") diff --git a/base/common/CMakeLists.txt b/base/common/CMakeLists.txt index e5e18669ebe..202a2dff56f 100644 --- a/base/common/CMakeLists.txt +++ b/base/common/CMakeLists.txt @@ -85,6 +85,7 @@ target_link_libraries (common replxx cctz fmt + magic_enum ) if (ENABLE_TESTS) diff --git a/base/common/Decimal.h b/base/common/Decimal.h new file mode 100644 index 00000000000..e856abf23c7 --- /dev/null +++ b/base/common/Decimal.h @@ -0,0 +1,157 @@ +#pragma once +#include "common/extended_types.h" + +#if !defined(NO_SANITIZE_UNDEFINED) +#if defined(__clang__) + #define NO_SANITIZE_UNDEFINED __attribute__((__no_sanitize__("undefined"))) +#else + #define NO_SANITIZE_UNDEFINED +#endif +#endif + +namespace DB +{ +template struct Decimal; +class DateTime64; + +using Decimal32 = Decimal; +using Decimal64 = Decimal; +using Decimal128 = Decimal; +using Decimal256 = Decimal; + +template +concept is_decimal = + std::is_same_v + || std::is_same_v + || std::is_same_v + || std::is_same_v + || std::is_same_v; + +template +concept is_over_big_int = + std::is_same_v + || std::is_same_v + || std::is_same_v + || std::is_same_v + || std::is_same_v + || std::is_same_v; + +template struct NativeTypeT { using Type = T; }; +template struct NativeTypeT { using Type = typename T::NativeType; }; +template using NativeType = typename NativeTypeT::Type; + +/// Own FieldType for Decimal. +/// It is only a "storage" for decimal. +/// To perform operations, you also have to provide a scale (number of digits after point). +template +struct Decimal +{ + using NativeType = T; + + constexpr Decimal() = default; + constexpr Decimal(Decimal &&) = default; + constexpr Decimal(const Decimal &) = default; + + constexpr Decimal(const T & value_): value(value_) {} + + template + constexpr Decimal(const Decimal & x): value(x.value) {} + + constexpr Decimal & operator = (Decimal &&) = default; + constexpr Decimal & operator = (const Decimal &) = default; + + constexpr operator T () const { return value; } + + template + constexpr U convertTo() const + { + if constexpr (is_decimal) + return convertTo(); + else + return static_cast(value); + } + + const Decimal & operator += (const T & x) { value += x; return *this; } + const Decimal & operator -= (const T & x) { value -= x; return *this; } + const Decimal & operator *= (const T & x) { value *= x; return *this; } + const Decimal & operator /= (const T & x) { value /= x; return *this; } + const Decimal & operator %= (const T & x) { value %= x; return *this; } + + template const Decimal & operator += (const Decimal & x) { value += x.value; return *this; } + template const Decimal & operator -= (const Decimal & x) { value -= x.value; return *this; } + template const Decimal & operator *= (const Decimal & x) { value *= x.value; return *this; } + template const Decimal & operator /= (const Decimal & x) { value /= x.value; return *this; } + template const Decimal & operator %= (const Decimal & x) { value %= x.value; return *this; } + + /// This is to avoid UB for sumWithOverflow() + void NO_SANITIZE_UNDEFINED addOverflow(const T & x) { value += x; } + + T value; +}; + +template inline bool operator< (const Decimal & x, const Decimal & y) { return x.value < y.value; } +template inline bool operator> (const Decimal & x, const Decimal & y) { return x.value > y.value; } +template inline bool operator<= (const Decimal & x, const Decimal & y) { return x.value <= y.value; } +template inline bool operator>= (const Decimal & x, const Decimal & y) { return x.value >= y.value; } +template inline bool operator== (const Decimal & x, const Decimal & y) { return x.value == y.value; } +template inline bool operator!= (const Decimal & x, const Decimal & y) { return x.value != y.value; } + +template inline Decimal operator+ (const Decimal & x, const Decimal & y) { return x.value + y.value; } +template inline Decimal operator- (const Decimal & x, const Decimal & y) { return x.value - y.value; } +template inline Decimal operator* (const Decimal & x, const Decimal & y) { return x.value * y.value; } +template inline Decimal operator/ (const Decimal & x, const Decimal & y) { return x.value / y.value; } +template inline Decimal operator- (const Decimal & x) { return -x.value; } + +/// Distinguishable type to allow function resolution/deduction based on value type, +/// but also relatively easy to convert to/from Decimal64. +class DateTime64 : public Decimal64 +{ +public: + using Base = Decimal64; + using Base::Base; + using NativeType = Base::NativeType; + + constexpr DateTime64(const Base & v): Base(v) {} +}; +} + +constexpr DB::UInt64 max_uint_mask = std::numeric_limits::max(); + +namespace std +{ + template + struct hash> + { + size_t operator()(const DB::Decimal & x) const { return hash()(x.value); } + }; + + template <> + struct hash + { + size_t operator()(const DB::Decimal128 & x) const + { + return std::hash()(x.value >> 64) + ^ std::hash()(x.value & max_uint_mask); + } + }; + + template <> + struct hash + { + size_t operator()(const DB::DateTime64 & x) const + { + return std::hash()(x); + } + }; + + template <> + struct hash + { + size_t operator()(const DB::Decimal256 & x) const + { + // FIXME temp solution + return std::hash()(static_cast(x.value >> 64 & max_uint_mask)) + ^ std::hash()(static_cast(x.value & max_uint_mask)); + } + }; +} diff --git a/base/common/EnumReflection.h b/base/common/EnumReflection.h new file mode 100644 index 00000000000..0d1f8ae0a40 --- /dev/null +++ b/base/common/EnumReflection.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include + +template concept is_enum = std::is_enum_v; + +namespace detail +{ +template +constexpr void static_for(F && f, std::index_sequence) +{ + (std::forward(f)(std::integral_constant(I)>()) , ...); +} +} + +/** + * Iterate over enum values in compile-time (compile-time switch/case, loop unrolling). + * + * @example static_for([](auto enum_value) { return template_func(); } + * ^ enum_value can be used as a template parameter + */ +template +constexpr void static_for(F && f) +{ + constexpr size_t count = magic_enum::enum_count(); + detail::static_for(std::forward(f), std::make_index_sequence()); +} + +/// Enable printing enum values as strings via fmt + magic_enum +template +struct fmt::formatter : fmt::formatter +{ + constexpr auto format(T value, auto& format_context) + { + return formatter::format(magic_enum::enum_name(value), format_context); + } +}; diff --git a/base/common/LineReader.cpp b/base/common/LineReader.cpp index a32906dd5a5..7dc37322bb0 100644 --- a/base/common/LineReader.cpp +++ b/base/common/LineReader.cpp @@ -16,6 +16,10 @@ extern "C" } #endif +#if defined(__clang__) && __clang_major__ >= 13 +#pragma clang diagnostic ignored "-Wreserved-identifier" +#endif + namespace { diff --git a/base/common/extended_types.h b/base/common/extended_types.h index 79209568ef5..cbd5688c491 100644 --- a/base/common/extended_types.h +++ b/base/common/extended_types.h @@ -41,22 +41,14 @@ template <> struct is_unsigned { static constexpr bool value = true; }; template inline constexpr bool is_unsigned_v = is_unsigned::value; +template concept is_integer = + std::is_integral_v + || std::is_same_v + || std::is_same_v + || std::is_same_v + || std::is_same_v; -/// TODO: is_integral includes char, char8_t and wchar_t. -template -struct is_integer -{ - static constexpr bool value = std::is_integral_v; -}; - -template <> struct is_integer { static constexpr bool value = true; }; -template <> struct is_integer { static constexpr bool value = true; }; -template <> struct is_integer { static constexpr bool value = true; }; -template <> struct is_integer { static constexpr bool value = true; }; - -template -inline constexpr bool is_integer_v = is_integer::value; - +template concept is_floating_point = std::is_floating_point_v; template struct is_arithmetic diff --git a/base/common/find_symbols.h b/base/common/find_symbols.h index a5921b813a1..b28749afda6 100644 --- a/base/common/find_symbols.h +++ b/base/common/find_symbols.h @@ -36,18 +36,7 @@ namespace detail { - -template -inline bool is_in(char x) -{ - return x == s0; -} - -template -inline bool is_in(char x) -{ - return x == s0 || is_in(x); -} +template constexpr bool is_in(char x) { return ((x == chars) || ...); } #if defined(__SSE2__) template @@ -67,16 +56,10 @@ inline __m128i mm_is_in(__m128i bytes) #endif template -bool maybe_negate(bool x) -{ - if constexpr (positive) - return x; - else - return !x; -} +constexpr bool maybe_negate(bool x) { return x == positive; } template -uint16_t maybe_negate(uint16_t x) +constexpr uint16_t maybe_negate(uint16_t x) { if constexpr (positive) return x; @@ -149,12 +132,13 @@ template -inline const char * find_first_symbols_sse42_impl(const char * const begin, const char * const end) +inline const char * find_first_symbols_sse42(const char * const begin, const char * const end) { const char * pos = begin; #if defined(__SSE4_2__) -#define MODE (_SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT) + constexpr int mode = _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT; + __m128i set = _mm_setr_epi8(c01, c02, c03, c04, c05, c06, c07, c08, c09, c10, c11, c12, c13, c14, c15, c16); for (; pos + 15 < end; pos += 16) @@ -163,16 +147,15 @@ inline const char * find_first_symbols_sse42_impl(const char * const begin, cons if constexpr (positive) { - if (_mm_cmpestrc(set, num_chars, bytes, 16, MODE)) - return pos + _mm_cmpestri(set, num_chars, bytes, 16, MODE); + if (_mm_cmpestrc(set, num_chars, bytes, 16, mode)) + return pos + _mm_cmpestri(set, num_chars, bytes, 16, mode); } else { - if (_mm_cmpestrc(set, num_chars, bytes, 16, MODE | _SIDD_NEGATIVE_POLARITY)) - return pos + _mm_cmpestri(set, num_chars, bytes, 16, MODE | _SIDD_NEGATIVE_POLARITY); + if (_mm_cmpestrc(set, num_chars, bytes, 16, mode | _SIDD_NEGATIVE_POLARITY)) + return pos + _mm_cmpestri(set, num_chars, bytes, 16, mode | _SIDD_NEGATIVE_POLARITY); } } -#undef MODE #endif for (; pos < end; ++pos) @@ -197,20 +180,15 @@ inline const char * find_first_symbols_sse42_impl(const char * const begin, cons } -template -inline const char * find_first_symbols_sse42(const char * begin, const char * end) -{ - return find_first_symbols_sse42_impl(begin, end); -} - /// NOTE No SSE 4.2 implementation for find_last_symbols_or_null. Not worth to do. template inline const char * find_first_symbols_dispatch(const char * begin, const char * end) + requires(0 <= sizeof...(symbols) && sizeof...(symbols) <= 16) { #if defined(__SSE4_2__) if (sizeof...(symbols) >= 5) - return find_first_symbols_sse42(begin, end); + return find_first_symbols_sse42(begin, end); else #endif return find_first_symbols_sse2(begin, end); diff --git a/base/common/phdr_cache.cpp b/base/common/phdr_cache.cpp index 49d566dac19..8ca9137cd29 100644 --- a/base/common/phdr_cache.cpp +++ b/base/common/phdr_cache.cpp @@ -1,3 +1,7 @@ +#if defined(__clang__) && __clang_major__ >= 13 +#pragma clang diagnostic ignored "-Wreserved-identifier" +#endif + /// This code was based on the code by Fedor Korotkiy (prime@yandex-team.ru) for YT product in Yandex. #include diff --git a/base/common/strong_typedef.h b/base/common/strong_typedef.h index a1e2b253aa7..0c2e9ca7e8e 100644 --- a/base/common/strong_typedef.h +++ b/base/common/strong_typedef.h @@ -15,15 +15,15 @@ private: public: using UnderlyingType = T; template ::type> - explicit StrongTypedef(const T & t_) : t(t_) {} + constexpr explicit StrongTypedef(const T & t_) : t(t_) {} template ::type> - explicit StrongTypedef(T && t_) : t(std::move(t_)) {} + constexpr explicit StrongTypedef(T && t_) : t(std::move(t_)) {} template ::type> - StrongTypedef(): t() {} + constexpr StrongTypedef(): t() {} - StrongTypedef(const Self &) = default; - StrongTypedef(Self &&) = default; + constexpr StrongTypedef(const Self &) = default; + constexpr StrongTypedef(Self &&) = default; Self & operator=(const Self &) = default; Self & operator=(Self &&) = default; diff --git a/base/common/unit.h b/base/common/unit.h index d5c8d5c9027..5bf5e00c049 100644 --- a/base/common/unit.h +++ b/base/common/unit.h @@ -1,6 +1,10 @@ #pragma once #include +#if defined(__clang__) && __clang_major__ >= 13 +#pragma clang diagnostic ignored "-Wreserved-identifier" +#endif + constexpr size_t KiB = 1024; constexpr size_t MiB = 1024 * KiB; constexpr size_t GiB = 1024 * MiB; diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 745e020c8bb..141b7b7bbc6 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -1,3 +1,7 @@ +#if defined(__clang__) && __clang_major__ >= 13 +#pragma clang diagnostic ignored "-Wreserved-identifier" +#endif + #include #include diff --git a/base/mysqlxx/CMakeLists.txt b/base/mysqlxx/CMakeLists.txt index c5230c2b49f..947f61f9051 100644 --- a/base/mysqlxx/CMakeLists.txt +++ b/base/mysqlxx/CMakeLists.txt @@ -49,6 +49,8 @@ if (NOT USE_INTERNAL_MYSQL_LIBRARY AND OPENSSL_INCLUDE_DIR) target_include_directories (mysqlxx SYSTEM PRIVATE ${OPENSSL_INCLUDE_DIR}) endif () +target_no_warning(mysqlxx reserved-macro-identifier) + if (NOT USE_INTERNAL_MYSQL_LIBRARY AND USE_STATIC_LIBRARIES) message(WARNING "Statically linking with system mysql/mariadb only works " "if mysql client libraries are built with same openssl version as " diff --git a/base/mysqlxx/Pool.cpp b/base/mysqlxx/Pool.cpp index 2f47aa67356..cee386311d4 100644 --- a/base/mysqlxx/Pool.cpp +++ b/base/mysqlxx/Pool.cpp @@ -7,10 +7,22 @@ #endif #include - #include - #include +#include + + +namespace +{ + +inline uint64_t clock_gettime_ns(clockid_t clock_type = CLOCK_MONOTONIC) +{ + struct timespec ts; + clock_gettime(clock_type, &ts); + return uint64_t(ts.tv_sec * 1000000000LL + ts.tv_nsec); +} + +} namespace mysqlxx @@ -124,10 +136,15 @@ Pool::~Pool() } -Pool::Entry Pool::get() +Pool::Entry Pool::get(uint64_t wait_timeout) { std::unique_lock lock(mutex); + uint64_t deadline = 0; + /// UINT64_MAX -- wait indefinitely + if (wait_timeout && wait_timeout != UINT64_MAX) + deadline = clock_gettime_ns() + wait_timeout * 1'000'000'000; + initialize(); for (;;) { @@ -153,6 +170,12 @@ Pool::Entry Pool::get() logger.trace("(%s): Unable to create a new connection: Max number of connections has been reached.", getDescription()); } + if (!wait_timeout) + throw Poco::Exception("mysqlxx::Pool is full (wait is disabled, see connection_wait_timeout setting)"); + + if (deadline && clock_gettime_ns() >= deadline) + throw Poco::Exception("mysqlxx::Pool is full (connection_wait_timeout is exceeded)"); + lock.unlock(); logger.trace("(%s): Sleeping for %d seconds.", getDescription(), MYSQLXX_POOL_SLEEP_ON_CONNECT_FAIL); sleepForSeconds(MYSQLXX_POOL_SLEEP_ON_CONNECT_FAIL); diff --git a/base/mysqlxx/Pool.h b/base/mysqlxx/Pool.h index 530e2c78cf2..f542c3d3b76 100644 --- a/base/mysqlxx/Pool.h +++ b/base/mysqlxx/Pool.h @@ -189,7 +189,7 @@ public: ~Pool(); /// Allocates connection. - Entry get(); + Entry get(uint64_t wait_timeout = UINT64_MAX); /// Allocates connection. /// If database is not accessible, returns empty Entry object. diff --git a/base/mysqlxx/PoolFactory.cpp b/base/mysqlxx/PoolFactory.cpp index f0a5543d723..1d2f9fc74c7 100644 --- a/base/mysqlxx/PoolFactory.cpp +++ b/base/mysqlxx/PoolFactory.cpp @@ -79,7 +79,7 @@ PoolWithFailover PoolFactory::get(const Poco::Util::AbstractConfiguration & conf std::lock_guard lock(impl->mutex); if (auto entry = impl->pools.find(config_name); entry != impl->pools.end()) { - return *(entry->second.get()); + return *(entry->second); } else { @@ -100,7 +100,7 @@ PoolWithFailover PoolFactory::get(const Poco::Util::AbstractConfiguration & conf impl->pools.insert_or_assign(config_name, pool); impl->pools_by_ids.insert_or_assign(entry_name, config_name); } - return *(pool.get()); + return *pool; } } diff --git a/base/mysqlxx/PoolWithFailover.cpp b/base/mysqlxx/PoolWithFailover.cpp index e317ab7f228..14c0db9ecd5 100644 --- a/base/mysqlxx/PoolWithFailover.cpp +++ b/base/mysqlxx/PoolWithFailover.cpp @@ -21,8 +21,9 @@ PoolWithFailover::PoolWithFailover( const unsigned max_connections_, const size_t max_tries_) : max_tries(max_tries_) + , shareable(config_.getBool(config_name_ + ".share_connection", false)) + , wait_timeout(UINT64_MAX) { - shareable = config_.getBool(config_name_ + ".share_connection", false); if (config_.has(config_name_ + ".replica")) { Poco::Util::AbstractConfiguration::Keys replica_keys; @@ -80,9 +81,11 @@ PoolWithFailover::PoolWithFailover( const std::string & password, unsigned default_connections_, unsigned max_connections_, - size_t max_tries_) + size_t max_tries_, + uint64_t wait_timeout_) : max_tries(max_tries_) , shareable(false) + , wait_timeout(wait_timeout_) { /// Replicas have the same priority, but traversed replicas are moved to the end of the queue. for (const auto & [host, port] : addresses) @@ -101,6 +104,7 @@ PoolWithFailover::PoolWithFailover( PoolWithFailover::PoolWithFailover(const PoolWithFailover & other) : max_tries{other.max_tries} , shareable{other.shareable} + , wait_timeout(other.wait_timeout) { if (shareable) { @@ -140,7 +144,7 @@ PoolWithFailover::Entry PoolWithFailover::get() try { - Entry entry = shareable ? pool->get() : pool->tryGet(); + Entry entry = shareable ? pool->get(wait_timeout) : pool->tryGet(); if (!entry.isNull()) { @@ -172,7 +176,7 @@ PoolWithFailover::Entry PoolWithFailover::get() if (full_pool) { app.logger().error("All connections failed, trying to wait on a full pool " + (*full_pool)->getDescription()); - return (*full_pool)->get(); + return (*full_pool)->get(wait_timeout); } std::stringstream message; diff --git a/base/mysqlxx/PoolWithFailover.h b/base/mysqlxx/PoolWithFailover.h index 1c7a63e76c0..2bd5ec9f30a 100644 --- a/base/mysqlxx/PoolWithFailover.h +++ b/base/mysqlxx/PoolWithFailover.h @@ -80,6 +80,8 @@ namespace mysqlxx std::mutex mutex; /// Can the Pool be shared bool shareable; + /// Timeout for waiting free connection. + uint64_t wait_timeout = 0; public: using Entry = Pool::Entry; @@ -96,6 +98,7 @@ namespace mysqlxx * default_connections Number of connection in pool to each replica at start. * max_connections Maximum number of connections in pool to each replica. * max_tries_ Max number of connection tries. + * wait_timeout_ Timeout for waiting free connection. */ PoolWithFailover( const std::string & config_name_, @@ -117,7 +120,8 @@ namespace mysqlxx const std::string & password, unsigned default_connections_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_START_CONNECTIONS, unsigned max_connections_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_CONNECTIONS, - size_t max_tries_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES); + size_t max_tries_ = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES, + uint64_t wait_timeout_ = UINT64_MAX); PoolWithFailover(const PoolWithFailover & other); diff --git a/base/readpassphrase/readpassphrase.c b/base/readpassphrase/readpassphrase.c index 8a7d3153915..3bcad34e481 100644 --- a/base/readpassphrase/readpassphrase.c +++ b/base/readpassphrase/readpassphrase.c @@ -27,6 +27,10 @@ #define _PATH_TTY "/dev/tty" #endif +#if defined(__clang__) && __clang_major__ >= 13 +#pragma clang diagnostic ignored "-Wreserved-identifier" +#endif + #include #include #include diff --git a/cmake/analysis.cmake b/cmake/analysis.cmake index 267bb34248b..d1b9c86f15f 100644 --- a/cmake/analysis.cmake +++ b/cmake/analysis.cmake @@ -6,7 +6,7 @@ if (ENABLE_CLANG_TIDY) message(FATAL_ERROR "clang-tidy requires CMake version at least 3.6.") endif() - find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-11" "clang-tidy-10" "clang-tidy-9" "clang-tidy-8") + find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-13" "clang-tidy-12" "clang-tidy-11" "clang-tidy-10" "clang-tidy-9" "clang-tidy-8") if (CLANG_TIDY_PATH) message(STATUS diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 03247b4b3ea..dd6ead7d97f 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54455) +SET(VERSION_REVISION 54456) SET(VERSION_MAJOR 21) -SET(VERSION_MINOR 10) +SET(VERSION_MINOR 11) SET(VERSION_PATCH 1) -SET(VERSION_GITHASH 09df5018f95edcd0f759d4689ac5d029dd400c2a) -SET(VERSION_DESCRIBE v21.10.1.1-testing) -SET(VERSION_STRING 21.10.1.1) +SET(VERSION_GITHASH 7a4a0b0edef0ad6e0aa662cd3b90c3f4acf796e7) +SET(VERSION_DESCRIBE v21.11.1.1-prestable) +SET(VERSION_STRING 21.11.1.1) # end of autochange diff --git a/cmake/find/llvm.cmake b/cmake/find/llvm.cmake index 816164bef10..84ac29991ab 100644 --- a/cmake/find/llvm.cmake +++ b/cmake/find/llvm.cmake @@ -1,8 +1,10 @@ -if (APPLE OR SPLIT_SHARED_LIBRARIES OR NOT ARCH_AMD64 OR SANITIZE STREQUAL "undefined") - set (ENABLE_EMBEDDED_COMPILER OFF CACHE INTERNAL "") +if (APPLE OR NOT ARCH_AMD64 OR SANITIZE STREQUAL "undefined") + set (ENABLE_EMBEDDED_COMPILER_DEFAULT OFF) +else() + set (ENABLE_EMBEDDED_COMPILER_DEFAULT ON) endif() -option (ENABLE_EMBEDDED_COMPILER "Enable support for 'compile_expressions' option for query execution" ON) +option (ENABLE_EMBEDDED_COMPILER "Enable support for 'compile_expressions' option for query execution" ${ENABLE_EMBEDDED_COMPILER_DEFAULT}) if (NOT ENABLE_EMBEDDED_COMPILER) set (USE_EMBEDDED_COMPILER 0) diff --git a/cmake/freebsd/toolchain-x86_64.cmake b/cmake/freebsd/toolchain-x86_64.cmake index d9839ec74ee..f9e45686db7 100644 --- a/cmake/freebsd/toolchain-x86_64.cmake +++ b/cmake/freebsd/toolchain-x86_64.cmake @@ -10,7 +10,7 @@ set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it set (CMAKE_AR "/usr/bin/ar" CACHE FILEPATH "" FORCE) set (CMAKE_RANLIB "/usr/bin/ranlib" CACHE FILEPATH "" FORCE) -set (LINKER_NAME "lld" CACHE STRING "" FORCE) +set (LINKER_NAME "ld.lld" CACHE STRING "" FORCE) set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld") set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld") diff --git a/cmake/linux/toolchain-aarch64.cmake b/cmake/linux/toolchain-aarch64.cmake index e3924fdc537..b4dc6e45cbb 100644 --- a/cmake/linux/toolchain-aarch64.cmake +++ b/cmake/linux/toolchain-aarch64.cmake @@ -13,7 +13,7 @@ set (CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS} --gcc-toolchain=${CMAKE_CURRENT_LIST_D set (CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS} --gcc-toolchain=${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64") set (CMAKE_ASM_FLAGS_INIT "${CMAKE_ASM_FLAGS} --gcc-toolchain=${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64") -set (LINKER_NAME "lld" CACHE STRING "" FORCE) +set (LINKER_NAME "ld.lld" CACHE STRING "" FORCE) set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld") set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld") diff --git a/cmake/tools.cmake b/cmake/tools.cmake index 8ff94ab867b..f94f4b289a3 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -79,8 +79,9 @@ endif () if (LINKER_NAME) if (COMPILER_CLANG AND (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 12.0.0 OR CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 12.0.0)) - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --ld-path=${LINKER_NAME}") - set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --ld-path=${LINKER_NAME}") + find_program (LLD_PATH NAMES ${LINKER_NAME}) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --ld-path=${LLD_PATH}") + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --ld-path=${LLD_PATH}") else () set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") diff --git a/cmake/warnings.cmake b/cmake/warnings.cmake index a85fe8963c7..0b6bbe36cf9 100644 --- a/cmake/warnings.cmake +++ b/cmake/warnings.cmake @@ -192,4 +192,29 @@ elseif (COMPILER_GCC) # For some reason (bug in gcc?) macro 'GCC diagnostic ignored "-Wstringop-overflow"' doesn't help. add_cxx_compile_options(-Wno-stringop-overflow) endif() + + if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 11) + # reinterpretAs.cpp:182:31: error: ‘void* memcpy(void*, const void*, size_t)’ copying an object of non-trivial type + # ‘using ToFieldType = using FieldType = using UUID = struct StrongTypedef, DB::UUIDTag>’ + # {aka ‘struct StrongTypedef, DB::UUIDTag>’} from an array of ‘const char8_t’ + add_cxx_compile_options(-Wno-error=class-memaccess) + + # Maybe false positive... + # In file included from /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/memory:673, + # In function ‘void std::__1::__libcpp_operator_delete(_Args ...) [with _Args = {void*, long unsigned int}]’, + # inlined from ‘void std::__1::__do_deallocate_handle_size(void*, size_t, _Args ...) [with _Args = {}]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/new:271:34, + # inlined from ‘void std::__1::__libcpp_deallocate(void*, size_t, size_t)’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/new:285:41, + # inlined from ‘constexpr void std::__1::allocator<_Tp>::deallocate(_Tp*, size_t) [with _Tp = char]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/memory:849:39, + # inlined from ‘static constexpr void std::__1::allocator_traits<_Alloc>::deallocate(std::__1::allocator_traits<_Alloc>::allocator_type&, std::__1::allocator_traits<_Alloc>::pointer, std::__1::allocator_traits<_Alloc>::size_type) [with _Alloc = std::__1::allocator]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/__memory/allocator_traits.h:476:24, + # inlined from ‘std::__1::basic_string<_CharT, _Traits, _Allocator>::~basic_string() [with _CharT = char; _Traits = std::__1::char_traits; _Allocator = std::__1::allocator]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/string:2219:35, + # inlined from ‘std::__1::basic_string<_CharT, _Traits, _Allocator>::~basic_string() [with _CharT = char; _Traits = std::__1::char_traits; _Allocator = std::__1::allocator]’ at /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/string:2213:1, + # inlined from ‘DB::JSONBuilder::JSONMap::Pair::~Pair()’ at /home/jakalletti/ClickHouse/ClickHouse/src/Common/JSONBuilder.h:90:12, + # inlined from ‘void DB::JSONBuilder::JSONMap::add(std::__1::string, DB::JSONBuilder::ItemPtr)’ at /home/jakalletti/ClickHouse/ClickHouse/src/Common/JSONBuilder.h:97:68, + # inlined from ‘virtual void DB::ExpressionStep::describeActions(DB::JSONBuilder::JSONMap&) const’ at /home/jakalletti/ClickHouse/ClickHouse/src/Processors/QueryPlan/ExpressionStep.cpp:102:12: + # /home/jakalletti/ClickHouse/ClickHouse/contrib/libcxx/include/new:247:20: error: ‘void operator delete(void*, size_t)’ called on a pointer to an unallocated object ‘7598543875853023301’ [-Werror=free-nonheap-object] + add_cxx_compile_options(-Wno-error=free-nonheap-object) + + # AggregateFunctionAvg.h:203:100: error: ‘this’ pointer is null [-Werror=nonnull] + add_cxx_compile_options(-Wno-error=nonnull) + endif() endif () diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index e6e098a05b3..140cc0846ec 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -33,6 +33,7 @@ endif() set_property(DIRECTORY PROPERTY EXCLUDE_FROM_ALL 1) add_subdirectory (abseil-cpp-cmake) +add_subdirectory (magic-enum-cmake) add_subdirectory (boost-cmake) add_subdirectory (cctz-cmake) add_subdirectory (consistent-hashing) @@ -206,12 +207,14 @@ elseif(GTEST_SRC_DIR) target_compile_definitions(gtest INTERFACE GTEST_HAS_POSIX_RE=0) endif() -if (USE_EMBEDDED_COMPILER) +function(add_llvm) # ld: unknown option: --color-diagnostics if (APPLE) set (LINKER_SUPPORTS_COLOR_DIAGNOSTICS 0 CACHE INTERNAL "") endif () + # Do not adjust RPATH in llvm, since then it will not be able to find libcxx/libcxxabi/libunwind + set (CMAKE_INSTALL_RPATH "ON") set (LLVM_ENABLE_EH 1 CACHE INTERNAL "") set (LLVM_ENABLE_RTTI 1 CACHE INTERNAL "") set (LLVM_ENABLE_PIC 0 CACHE INTERNAL "") @@ -219,13 +222,12 @@ if (USE_EMBEDDED_COMPILER) # Need to use C++17 since the compilation is not possible with C++20 currently, due to ambiguous operator != etc. # LLVM project will set its default value for the -std=... but our global setting from CMake will override it. - set (CMAKE_CXX_STANDARD_bak ${CMAKE_CXX_STANDARD}) set (CMAKE_CXX_STANDARD 17) add_subdirectory (llvm/llvm) - - set (CMAKE_CXX_STANDARD ${CMAKE_CXX_STANDARD_bak}) - unset (CMAKE_CXX_STANDARD_bak) +endfunction() +if (USE_EMBEDDED_COMPILER) + add_llvm() endif () if (USE_INTERNAL_LIBGSASL_LIBRARY) diff --git a/contrib/abseil-cpp b/contrib/abseil-cpp index 4f3b686f86c..b004a8a0241 160000 --- a/contrib/abseil-cpp +++ b/contrib/abseil-cpp @@ -1 +1 @@ -Subproject commit 4f3b686f86c3ebaba7e4e926e62a79cb1c659a54 +Subproject commit b004a8a02418b83de8b686caa0b0f6e39ac2191f diff --git a/contrib/boost b/contrib/boost index 9cf09dbfd55..66d17f060c4 160000 --- a/contrib/boost +++ b/contrib/boost @@ -1 +1 @@ -Subproject commit 9cf09dbfd55a5c6202dedbdf40781a51b02c2675 +Subproject commit 66d17f060c4867aeea99fa2a20cfdae89ae2a2ec diff --git a/contrib/boost-cmake/CMakeLists.txt b/contrib/boost-cmake/CMakeLists.txt index 675931d319f..27072910135 100644 --- a/contrib/boost-cmake/CMakeLists.txt +++ b/contrib/boost-cmake/CMakeLists.txt @@ -16,7 +16,7 @@ if (NOT USE_INTERNAL_BOOST_LIBRARY) graph ) - if(Boost_INCLUDE_DIR AND Boost_FILESYSTEM_LIBRARY AND Boost_FILESYSTEM_LIBRARY AND + if(Boost_INCLUDE_DIR AND Boost_FILESYSTEM_LIBRARY AND Boost_PROGRAM_OPTIONS_LIBRARY AND Boost_REGEX_LIBRARY AND Boost_SYSTEM_LIBRARY AND Boost_CONTEXT_LIBRARY AND Boost_COROUTINE_LIBRARY AND Boost_GRAPH_LIBRARY) @@ -238,4 +238,14 @@ if (NOT EXTERNAL_BOOST_FOUND) target_include_directories (_boost_graph PRIVATE ${LIBRARY_DIR}) target_link_libraries(_boost_graph PRIVATE _boost_regex) + # circular buffer + add_library(_boost_circular_buffer INTERFACE) + add_library(boost::circular_buffer ALIAS _boost_circular_buffer) + target_include_directories(_boost_circular_buffer SYSTEM BEFORE INTERFACE ${LIBRARY_DIR}) + + # heap + add_library(_boost_heap INTERFACE) + add_library(boost::heap ALIAS _boost_heap) + target_include_directories(_boost_heap SYSTEM BEFORE INTERFACE ${LIBRARY_DIR}) + endif () diff --git a/contrib/fastops b/contrib/fastops index 88752a5e03c..012b777df9e 160000 --- a/contrib/fastops +++ b/contrib/fastops @@ -1 +1 @@ -Subproject commit 88752a5e03cf34639a4a37a4b41d8b463fffd2b5 +Subproject commit 012b777df9e2d145a24800a6c8c3d4a0249bb09e diff --git a/contrib/llvm b/contrib/llvm index e5751459412..f30bbecef78 160000 --- a/contrib/llvm +++ b/contrib/llvm @@ -1 +1 @@ -Subproject commit e5751459412bce1391fb7a2e9bbc01e131bf72f1 +Subproject commit f30bbecef78b75b527e257c1304d0be2f2f95975 diff --git a/contrib/magic-enum-cmake/CMakeLists.txt b/contrib/magic-enum-cmake/CMakeLists.txt new file mode 100644 index 00000000000..142f9c7c755 --- /dev/null +++ b/contrib/magic-enum-cmake/CMakeLists.txt @@ -0,0 +1,3 @@ +set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/magic_enum") +add_library (magic_enum INTERFACE) +target_include_directories(magic_enum INTERFACE ${LIBRARY_DIR}/include) diff --git a/contrib/magic_enum b/contrib/magic_enum new file mode 160000 index 00000000000..38f86e4d093 --- /dev/null +++ b/contrib/magic_enum @@ -0,0 +1 @@ +Subproject commit 38f86e4d093cfc9034a140d37de2168e3951bef3 diff --git a/contrib/rocksdb b/contrib/rocksdb index b6480c69bf3..5ea892c8673 160000 --- a/contrib/rocksdb +++ b/contrib/rocksdb @@ -1 +1 @@ -Subproject commit b6480c69bf3ab6e298e0d019a07fd4f69029b26a +Subproject commit 5ea892c8673e6c5a052887653673b967d44cc59b diff --git a/debian/changelog b/debian/changelog index f3e740d20cf..460424bdb36 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (21.10.1.1) unstable; urgency=low +clickhouse (21.11.1.1) unstable; urgency=low * Modified source code - -- clickhouse-release Sat, 17 Jul 2021 08:45:03 +0300 + -- clickhouse-release Thu, 09 Sep 2021 12:03:26 +0300 diff --git a/debian/clickhouse-server.init b/debian/clickhouse-server.init index 4e70d382b36..1dd87fe80ae 100755 --- a/debian/clickhouse-server.init +++ b/debian/clickhouse-server.init @@ -3,10 +3,17 @@ # Provides: clickhouse-server # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 -# Required-Start: $network -# Required-Stop: $network +# Should-Start: $time $network +# Should-Stop: $network # Short-Description: Yandex clickhouse-server daemon ### END INIT INFO +# +# NOTES: +# - Should-* -- script can start if the listed facilities are missing, unlike Required-* +# +# For the documentation [1]: +# +# [1]: https://wiki.debian.org/LSBInitScripts CLICKHOUSE_USER=clickhouse CLICKHOUSE_GROUP=${CLICKHOUSE_USER} diff --git a/debian/clickhouse-server.service b/debian/clickhouse-server.service index b9681f9279e..bc19235cb3a 100644 --- a/debian/clickhouse-server.service +++ b/debian/clickhouse-server.service @@ -1,7 +1,12 @@ [Unit] Description=ClickHouse Server (analytic DBMS for big data) Requires=network-online.target -After=network-online.target +# NOTE: that After/Wants=time-sync.target is not enough, you need to ensure +# that the time was adjusted already, if you use systemd-timesyncd you are +# safe, but if you use ntp or some other daemon, you should configure it +# additionaly. +After=time-sync.target network-online.target +Wants=time-sync.target [Service] Type=simple @@ -16,4 +21,5 @@ LimitNOFILE=500000 CapabilityBoundingSet=CAP_NET_ADMIN CAP_IPC_LOCK CAP_SYS_NICE [Install] +# ClickHouse should not start from the rescue shell (rescue.target). WantedBy=multi-user.target diff --git a/debian/rules b/debian/rules index 73d1f3d3b34..4562d24bec4 100755 --- a/debian/rules +++ b/debian/rules @@ -36,8 +36,8 @@ endif CMAKE_FLAGS += -DENABLE_UTILS=0 -DEB_CC ?= $(shell which gcc-10 gcc-9 gcc | head -n1) -DEB_CXX ?= $(shell which g++-10 g++-9 g++ | head -n1) +DEB_CC ?= $(shell which gcc-11 gcc-10 gcc-9 gcc | head -n1) +DEB_CXX ?= $(shell which g++-11 g++-10 g++-9 g++ | head -n1) ifdef DEB_CXX DEB_BUILD_GNU_TYPE := $(shell dpkg-architecture -qDEB_BUILD_GNU_TYPE) diff --git a/docker/builder/Dockerfile b/docker/builder/Dockerfile index abe102e9c80..5a998ffbe3e 100644 --- a/docker/builder/Dockerfile +++ b/docker/builder/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:20.04 -ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=11 +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13 RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list diff --git a/docker/builder/build.sh b/docker/builder/build.sh index d4cf662e91b..1025af3f96e 100755 --- a/docker/builder/build.sh +++ b/docker/builder/build.sh @@ -4,7 +4,7 @@ set -e #ccache -s # uncomment to display CCache statistics mkdir -p /server/build_docker cd /server/build_docker -cmake -G Ninja /server "-DCMAKE_C_COMPILER=$(command -v clang-11)" "-DCMAKE_CXX_COMPILER=$(command -v clang++-11)" +cmake -G Ninja /server "-DCMAKE_C_COMPILER=$(command -v clang-13)" "-DCMAKE_CXX_COMPILER=$(command -v clang++-13)" # Set the number of build jobs to the half of number of virtual CPU cores (rounded up). # By default, ninja use all virtual CPU cores, that leads to very high memory consumption without much improvement in build time. diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index 1b398b43c32..962459ccead 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=21.10.1.* +ARG version=21.11.1.* RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list diff --git a/docker/images.json b/docker/images.json index e2e22468596..96424bb9432 100644 --- a/docker/images.json +++ b/docker/images.json @@ -1,12 +1,12 @@ { "docker/packager/deb": { - "name": "yandex/clickhouse-deb-builder", + "name": "clickhouse/deb-builder", "dependent": [ "docker/packager/unbundled" ] }, "docker/packager/binary": { - "name": "yandex/clickhouse-binary-builder", + "name": "clickhouse/binary-builder", "dependent": [ "docker/test/split_build_smoke_test", "docker/test/pvs", @@ -14,155 +14,150 @@ ] }, "docker/packager/unbundled": { - "name": "yandex/clickhouse-unbundled-builder", + "name": "clickhouse/unbundled-builder", "dependent": [] }, "docker/test/compatibility/centos": { - "name": "yandex/clickhouse-test-old-centos", + "name": "clickhouse/test-old-centos", "dependent": [] }, "docker/test/compatibility/ubuntu": { - "name": "yandex/clickhouse-test-old-ubuntu", + "name": "clickhouse/test-old-ubuntu", "dependent": [] }, "docker/test/integration/base": { - "name": "yandex/clickhouse-integration-test", + "name": "clickhouse/integration-test", "dependent": [] }, "docker/test/fuzzer": { - "name": "yandex/clickhouse-fuzzer", + "name": "clickhouse/fuzzer", "dependent": [] }, "docker/test/performance-comparison": { - "name": "yandex/clickhouse-performance-comparison", + "name": "clickhouse/performance-comparison", "dependent": [] }, "docker/test/pvs": { - "name": "yandex/clickhouse-pvs-test", + "name": "clickhouse/pvs-test", "dependent": [] }, "docker/test/stateless": { - "name": "yandex/clickhouse-stateless-test", + "name": "clickhouse/stateless-test", "dependent": [ "docker/test/stateful", "docker/test/coverage", "docker/test/unit" ] }, - "docker/test/stateless_pytest": { - "name": "yandex/clickhouse-stateless-pytest", - "dependent": [] - }, "docker/test/stateful": { - "name": "yandex/clickhouse-stateful-test", + "name": "clickhouse/stateful-test", "dependent": [ "docker/test/stress" ] }, "docker/test/coverage": { - "name": "yandex/clickhouse-test-coverage", + "name": "clickhouse/test-coverage", "dependent": [] }, "docker/test/unit": { - "name": "yandex/clickhouse-unit-test", + "name": "clickhouse/unit-test", "dependent": [] }, "docker/test/stress": { - "name": "yandex/clickhouse-stress-test", + "name": "clickhouse/stress-test", "dependent": [] }, "docker/test/split_build_smoke_test": { - "name": "yandex/clickhouse-split-build-smoke-test", + "name": "clickhouse/split-build-smoke-test", "dependent": [] }, "docker/test/codebrowser": { - "name": "yandex/clickhouse-codebrowser", + "name": "clickhouse/codebrowser", "dependent": [] }, "docker/test/integration/runner": { - "name": "yandex/clickhouse-integration-tests-runner", + "name": "clickhouse/integration-tests-runner", "dependent": [] }, "docker/test/testflows/runner": { - "name": "yandex/clickhouse-testflows-runner", + "name": "clickhouse/testflows-runner", "dependent": [] }, "docker/test/fasttest": { - "name": "yandex/clickhouse-fasttest", + "name": "clickhouse/fasttest", "dependent": [] }, "docker/test/style": { - "name": "yandex/clickhouse-style-test", + "name": "clickhouse/style-test", "dependent": [] }, "docker/test/integration/s3_proxy": { - "name": "yandex/clickhouse-s3-proxy", + "name": "clickhouse/s3-proxy", "dependent": [] }, "docker/test/integration/resolver": { - "name": "yandex/clickhouse-python-bottle", + "name": "clickhouse/python-bottle", "dependent": [] }, "docker/test/integration/helper_container": { - "name": "yandex/clickhouse-integration-helper", + "name": "clickhouse/integration-helper", "dependent": [] }, "docker/test/integration/mysql_golang_client": { - "name": "yandex/clickhouse-mysql-golang-client", + "name": "clickhouse/mysql-golang-client", "dependent": [] }, "docker/test/integration/mysql_java_client": { - "name": "yandex/clickhouse-mysql-java-client", + "name": "clickhouse/mysql-java-client", "dependent": [] }, "docker/test/integration/mysql_js_client": { - "name": "yandex/clickhouse-mysql-js-client", + "name": "clickhouse/mysql-js-client", "dependent": [] }, "docker/test/integration/mysql_php_client": { - "name": "yandex/clickhouse-mysql-php-client", + "name": "clickhouse/mysql-php-client", "dependent": [] }, "docker/test/integration/postgresql_java_client": { - "name": "yandex/clickhouse-postgresql-java-client", + "name": "clickhouse/postgresql-java-client", "dependent": [] }, "docker/test/integration/kerberos_kdc": { - "name": "yandex/clickhouse-kerberos-kdc", + "name": "clickhouse/kerberos-kdc", "dependent": [] }, "docker/test/base": { - "name": "yandex/clickhouse-test-base", + "name": "clickhouse/test-base", "dependent": [ "docker/test/stateless", "docker/test/stateless_unbundled", - "docker/test/stateless_pytest", "docker/test/integration/base", "docker/test/fuzzer", "docker/test/keeper-jepsen" ] }, "docker/packager/unbundled": { - "name": "yandex/clickhouse-unbundled-builder", + "name": "clickhouse/unbundled-builder", "dependent": [ "docker/test/stateless_unbundled" ] }, "docker/test/stateless_unbundled": { - "name": "yandex/clickhouse-stateless-unbundled-test", + "name": "clickhouse/stateless-unbundled-test", "dependent": [ ] }, "docker/test/integration/kerberized_hadoop": { - "name": "yandex/clickhouse-kerberized-hadoop", + "name": "clickhouse/kerberized-hadoop", "dependent": [] }, "docker/test/sqlancer": { - "name": "yandex/clickhouse-sqlancer-test", + "name": "clickhouse/sqlancer-test", "dependent": [] }, "docker/test/keeper-jepsen": { - "name": "yandex/clickhouse-keeper-jepsen-test", + "name": "clickhouse/keeper-jepsen-test", "dependent": [] } } diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 0393669df48..23012a38f9d 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -1,7 +1,7 @@ -# docker build -t yandex/clickhouse-binary-builder . +# docker build -t clickhouse/binary-builder . FROM ubuntu:20.04 -ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=11 +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13 RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list @@ -39,20 +39,18 @@ RUN apt-get update \ bash \ build-essential \ ccache \ - clang-11 \ - clang-tidy-11 \ cmake \ curl \ - g++-10 \ - gcc-10 \ gdb \ git \ gperf \ + clang-${LLVM_VERSION} \ + clang-tidy-${LLVM_VERSION} \ + lld-${LLVM_VERSION} \ + llvm-${LLVM_VERSION} \ + llvm-${LLVM_VERSION}-dev \ libicu-dev \ libreadline-dev \ - lld-11 \ - llvm-11 \ - llvm-11-dev \ moreutils \ ninja-build \ pigz \ @@ -102,15 +100,10 @@ RUN wget -nv "https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.0 # Download toolchain for FreeBSD 11.3 RUN wget -nv https://clickhouse-datasets.s3.yandex.net/toolchains/toolchains/freebsd-11.3-toolchain.tar.xz -# NOTE: For some reason we have outdated version of gcc-10 in ubuntu 20.04 stable. -# Current workaround is to use latest version proposed repo. Remove as soon as -# gcc-10.2 appear in stable repo. -RUN echo 'deb http://archive.ubuntu.com/ubuntu/ focal-proposed restricted main multiverse universe' > /etc/apt/sources.list.d/proposed-repositories.list - -RUN apt-get update \ - && apt-get install gcc-10 g++-10 --yes - -RUN rm /etc/apt/sources.list.d/proposed-repositories.list && apt-get update +# NOTE: Seems like gcc-11 is too new for ubuntu20 repository +RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \ + && apt-get update \ + && apt-get install gcc-11 g++-11 --yes COPY build.sh / diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index b9900e34bf1..71402a2fd66 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -4,7 +4,6 @@ set -x -e mkdir -p build/cmake/toolchain/darwin-x86_64 tar xJf MacOSX11.0.sdk.tar.xz -C build/cmake/toolchain/darwin-x86_64 --strip-components=1 - ln -sf darwin-x86_64 build/cmake/toolchain/darwin-aarch64 mkdir -p build/cmake/toolchain/linux-aarch64 @@ -23,6 +22,7 @@ cd build/build_docker rm -f CMakeCache.txt # Read cmake arguments into array (possibly empty) read -ra CMAKE_FLAGS <<< "${CMAKE_FLAGS:-}" +env cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" .. ccache --show-config ||: diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index 294c8645455..318b960e0b4 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -1,7 +1,7 @@ -# docker build -t yandex/clickhouse-deb-builder . +# docker build -t clickhouse/deb-builder . FROM ubuntu:20.04 -ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=11 +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13 RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list @@ -37,17 +37,17 @@ RUN curl -O https://clickhouse-datasets.s3.yandex.net/utils/1/dpkg-deb \ RUN apt-get update \ && apt-get install \ alien \ - clang-11 \ - clang-tidy-11 \ + clang-${LLVM_VERSION} \ + clang-tidy-${LLVM_VERSION} \ cmake \ debhelper \ devscripts \ gdb \ git \ gperf \ - lld-11 \ - llvm-11 \ - llvm-11-dev \ + lld-${LLVM_VERSION} \ + llvm-${LLVM_VERSION} \ + llvm-${LLVM_VERSION}-dev \ moreutils \ ninja-build \ perl \ @@ -57,15 +57,11 @@ RUN apt-get update \ tzdata \ --yes --no-install-recommends -# NOTE: For some reason we have outdated version of gcc-10 in ubuntu 20.04 stable. -# Current workaround is to use latest version proposed repo. Remove as soon as -# gcc-10.2 appear in stable repo. -RUN echo 'deb http://archive.ubuntu.com/ubuntu/ focal-proposed restricted main multiverse universe' > /etc/apt/sources.list.d/proposed-repositories.list +# NOTE: Seems like gcc-11 is too new for ubuntu20 repository +RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \ + && apt-get update \ + && apt-get install gcc-11 g++-11 --yes -RUN apt-get update \ - && apt-get install gcc-10 g++-10 --yes --no-install-recommends - -RUN rm /etc/apt/sources.list.d/proposed-repositories.list && apt-get update # This symlink required by gcc to find lld compiler RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld diff --git a/docker/packager/packager b/docker/packager/packager index 673878bce43..f8e5fd717cf 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -9,9 +9,9 @@ import sys SCRIPT_PATH = os.path.realpath(__file__) IMAGE_MAP = { - "deb": "yandex/clickhouse-deb-builder", - "binary": "yandex/clickhouse-binary-builder", - "unbundled": "yandex/clickhouse-unbundled-builder" + "deb": "clickhouse/deb-builder", + "binary": "clickhouse/binary-builder", + "unbundled": "clickhouse/unbundled-builder" } def check_image_exists_locally(image_name): @@ -75,7 +75,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ # Explicitly use LLD with Clang by default. # Don't force linker for cross-compilation. if is_clang and not is_cross_compile: - cmake_flags.append("-DLINKER_NAME=lld") + cmake_flags.append("-DLINKER_NAME=ld.lld") if is_cross_darwin: cc = compiler[:-len(DARWIN_SUFFIX)] @@ -204,7 +204,9 @@ if __name__ == "__main__": parser.add_argument("--output-dir", required=True) parser.add_argument("--build-type", choices=("debug", ""), default="") parser.add_argument("--compiler", choices=("clang-11", "clang-11-darwin", "clang-11-darwin-aarch64", "clang-11-aarch64", - "clang-11-freebsd", "gcc-10"), default="clang-11") + "clang-12", "clang-12-darwin", "clang-12-darwin-aarch64", "clang-12-aarch64", + "clang-13", "clang-13-darwin", "clang-13-darwin-aarch64", "clang-13-aarch64", + "clang-11-freebsd", "clang-12-freebsd", "clang-13-freebsd", "gcc-11"), default="clang-13") parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="") parser.add_argument("--unbundled", action="store_true") parser.add_argument("--split-binary", action="store_true") diff --git a/docker/packager/unbundled/Dockerfile b/docker/packager/unbundled/Dockerfile index b2d9f555f19..4fe093a0751 100644 --- a/docker/packager/unbundled/Dockerfile +++ b/docker/packager/unbundled/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-unbundled-builder . -FROM yandex/clickhouse-deb-builder +# docker build -t clickhouse/unbundled-builder . +FROM clickhouse/deb-builder RUN export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \ && wget -nv -O /tmp/arrow-keyring.deb "https://apache.jfrog.io/artifactory/arrow/ubuntu/apache-arrow-apt-source-latest-${CODENAME}.deb" \ @@ -17,6 +17,7 @@ RUN apt-get update \ devscripts \ libc++-dev \ libc++abi-dev \ + libboost-all-dev \ libboost-program-options-dev \ libboost-system-dev \ libboost-filesystem-dev \ diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 6307e74c633..7309574b16c 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:20.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=21.10.1.* +ARG version=21.11.1.* ARG gosu_ver=1.10 # set non-empty deb_location_url url to create a docker image diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index 62cfcf9e896..b3a23bf9662 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=21.10.1.* +ARG version=21.11.1.* RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile index 611ef6b7702..2d49c207ac3 100644 --- a/docker/test/base/Dockerfile +++ b/docker/test/base/Dockerfile @@ -1,7 +1,7 @@ -# docker build -t yandex/clickhouse-test-base . +# docker build -t clickhouse/test-base . FROM ubuntu:20.04 -ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=11 +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13 RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list diff --git a/docker/test/codebrowser/Dockerfile b/docker/test/codebrowser/Dockerfile index 33173ab90f9..94aa321252b 100644 --- a/docker/test/codebrowser/Dockerfile +++ b/docker/test/codebrowser/Dockerfile @@ -1,6 +1,6 @@ -# docker build --network=host -t yandex/clickhouse-codebrowser . -# docker run --volume=path_to_repo:/repo_folder --volume=path_to_result:/test_output yandex/clickhouse-codebrowser -FROM yandex/clickhouse-binary-builder +# docker build --network=host -t clickhouse/codebrowser . +# docker run --volume=path_to_repo:/repo_folder --volume=path_to_result:/test_output clickhouse/codebrowser +FROM clickhouse/binary-builder RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list @@ -11,7 +11,7 @@ RUN apt-get update && apt-get --yes --allow-unauthenticated install clang-9 libl # https://github.com/ClickHouse-Extras/woboq_codebrowser/commit/37e15eaf377b920acb0b48dbe82471be9203f76b RUN git clone https://github.com/ClickHouse-Extras/woboq_codebrowser -RUN cd woboq_codebrowser && cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-9 -DCMAKE_C_COMPILER=clang-9 && make -j +RUN cd woboq_codebrowser && cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-13 -DCMAKE_C_COMPILER=clang-13 && make -j ENV CODEGEN=/woboq_codebrowser/generator/codebrowser_generator ENV CODEINDEX=/woboq_codebrowser/indexgenerator/codebrowser_indexgenerator @@ -24,7 +24,7 @@ ENV SHA=nosha ENV DATA="data" CMD mkdir -p $BUILD_DIRECTORY && cd $BUILD_DIRECTORY && \ - cmake $SOURCE_DIRECTORY -DCMAKE_CXX_COMPILER=/usr/bin/clang\+\+-11 -DCMAKE_C_COMPILER=/usr/bin/clang-11 -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_S3=0 && \ + cmake $SOURCE_DIRECTORY -DCMAKE_CXX_COMPILER=/usr/bin/clang\+\+-13 -DCMAKE_C_COMPILER=/usr/bin/clang-13 -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_S3=0 && \ mkdir -p $HTML_RESULT_DIRECTORY && \ $CODEGEN -b $BUILD_DIRECTORY -a -o $HTML_RESULT_DIRECTORY -p ClickHouse:$SOURCE_DIRECTORY:$SHA -d $DATA | ts '%Y-%m-%d %H:%M:%S' && \ cp -r $STATIC_DATA $HTML_RESULT_DIRECTORY/ &&\ diff --git a/docker/test/compatibility/centos/Dockerfile b/docker/test/compatibility/centos/Dockerfile index 0ef119d1bb1..628609e374f 100644 --- a/docker/test/compatibility/centos/Dockerfile +++ b/docker/test/compatibility/centos/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-test-old-centos . +# docker build -t clickhouse/test-old-centos . FROM centos:5 CMD /bin/sh -c "/clickhouse server --config /config/config.xml > /var/log/clickhouse-server/stderr.log 2>&1 & \ diff --git a/docker/test/compatibility/ubuntu/Dockerfile b/docker/test/compatibility/ubuntu/Dockerfile index 28f89e47b95..ddd0a76bd44 100644 --- a/docker/test/compatibility/ubuntu/Dockerfile +++ b/docker/test/compatibility/ubuntu/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-test-old-ubuntu . +# docker build -t clickhouse/test-old-ubuntu . FROM ubuntu:12.04 CMD /bin/sh -c "/clickhouse server --config /config/config.xml > /var/log/clickhouse-server/stderr.log 2>&1 & \ diff --git a/docker/test/coverage/Dockerfile b/docker/test/coverage/Dockerfile index 681f65e0f6f..ccf0bbc7c83 100644 --- a/docker/test/coverage/Dockerfile +++ b/docker/test/coverage/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-test-coverage . -FROM yandex/clickhouse-stateless-test +# docker build -t clickhouse/test-coverage . +FROM clickhouse/stateless-test RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/coverage/run.sh b/docker/test/coverage/run.sh index 11b6ce13ea1..807efdf1e47 100755 --- a/docker/test/coverage/run.sh +++ b/docker/test/coverage/run.sh @@ -80,7 +80,7 @@ LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "RENAM LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits" LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "SHOW TABLES FROM test" -LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-test -j 8 --testname --shard --zookeeper --print-time --use-skip-list 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee /test_result.txt +LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-test -j 8 --testname --shard --zookeeper --print-time 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee /test_result.txt readarray -t FAILED_TESTS < <(awk '/FAIL|TIMEOUT|ERROR/ { print substr($3, 1, length($3)-1) }' "/test_result.txt") @@ -97,7 +97,7 @@ then echo "Going to run again: ${FAILED_TESTS[*]}" - LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-test --order=random --testname --shard --zookeeper --use-skip-list "${FAILED_TESTS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a /test_result.txt + LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-test --order=random --testname --shard --zookeeper "${FAILED_TESTS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a /test_result.txt else echo "No failed tests" fi diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index 2e0bbcd350f..3119b2f1bb3 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -1,7 +1,7 @@ -# docker build -t yandex/clickhouse-fasttest . +# docker build -t clickhouse/fasttest . FROM ubuntu:20.04 -ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=11 +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13 RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index b7c8240abba..0cda2f8f17f 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -9,7 +9,7 @@ trap 'kill $(jobs -pr) ||:' EXIT stage=${stage:-} # Compiler version, normally set by Dockerfile -export LLVM_VERSION=${LLVM_VERSION:-11} +export LLVM_VERSION=${LLVM_VERSION:-13} # A variable to pass additional flags to CMake. # Here we explicitly default it to nothing so that bash doesn't complain about @@ -159,6 +159,7 @@ function clone_submodules cd "$FASTTEST_SOURCE" SUBMODULES_TO_UPDATE=( + contrib/magic_enum contrib/abseil-cpp contrib/boost contrib/zlib-ng @@ -261,150 +262,8 @@ function run_tests start_server - TESTS_TO_SKIP=( - 00105_shard_collations - 00109_shard_totals_after_having - 00110_external_sort - 00302_http_compression - 00417_kill_query - 00436_convert_charset - 00490_special_line_separators_and_characters_outside_of_bmp - 00652_replicated_mutations_zookeeper - 00682_empty_parts_merge - 00701_rollup - 00834_cancel_http_readonly_queries_on_client_close - 00911_tautological_compare - - # Hyperscan - 00926_multimatch - 00929_multi_match_edit_distance - 01681_hyperscan_debug_assertion - 02004_max_hyperscan_regex_length - - 01176_mysql_client_interactive # requires mysql client - 01031_mutations_interpreter_and_context - 01053_ssd_dictionary # this test mistakenly requires acces to /var/lib/clickhouse -- can't run this locally, disabled - 01083_expressions_in_engine_arguments - 01092_memory_profiler - 01098_msgpack_format - 01098_temporary_and_external_tables - 01103_check_cpu_instructions_at_startup # avoid dependency on qemu -- invonvenient when running locally - 01193_metadata_loading - 01238_http_memory_tracking # max_memory_usage_for_user can interfere another queries running concurrently - 01251_dict_is_in_infinite_loop - 01259_dictionary_custom_settings_ddl - 01268_dictionary_direct_layout - 01280_ssd_complex_key_dictionary - 01281_group_by_limit_memory_tracking # max_memory_usage_for_user can interfere another queries running concurrently - 01318_encrypt # Depends on OpenSSL - 01318_decrypt # Depends on OpenSSL - 01663_aes_msan # Depends on OpenSSL - 01667_aes_args_check # Depends on OpenSSL - 01683_codec_encrypted # Depends on OpenSSL - 01776_decrypt_aead_size_check # Depends on OpenSSL - 01811_filter_by_null # Depends on OpenSSL - 02012_sha512_fixedstring # Depends on OpenSSL - 01281_unsucceeded_insert_select_queries_counter - 01292_create_user - 01294_lazy_database_concurrent - 01305_replica_create_drop_zookeeper - 01354_order_by_tuple_collate_const - 01355_ilike - 01411_bayesian_ab_testing - 01798_uniq_theta_sketch - 01799_long_uniq_theta_sketch - 01890_stem # depends on libstemmer_c - 02003_compress_bz2 # depends on bzip2 - 01059_storage_file_compression # depends on brotli and bzip2 - collate - collation - _orc_ - arrow - avro - base64 - brotli - capnproto - client - ddl_dictionaries - h3 - hashing - hdfs - java_hash - json - limit_memory - live_view - memory_leak - memory_limit - mysql - odbc - parallel_alter - parquet - protobuf - secure - sha256 - xz - - # Not sure why these two fail even in sequential mode. Disabled for now - # to make some progress. - 00646_url_engine - 00974_query_profiler - - # In fasttest, ENABLE_LIBRARIES=0, so rocksdb engine is not enabled by default - 01504_rocksdb - 01686_rocksdb - - # Look at DistributedFilesToInsert, so cannot run in parallel. - 01460_DistributedFilesToInsert - - 01541_max_memory_usage_for_user_long - - # Require python libraries like scipy, pandas and numpy - 01322_ttest_scipy - 01561_mann_whitney_scipy - - 01545_system_errors - # Checks system.errors - 01563_distributed_query_finish - - # nc - command not found - 01601_proxy_protocol - 01622_defaults_for_url_engine - - # JSON functions - 01666_blns - - # Requires postgresql-client - 01802_test_postgresql_protocol_with_row_policy - - # Depends on AWS - 01801_s3_cluster - 02012_settings_clause_for_s3 - - # needs psql - 01889_postgresql_protocol_null_fields - - # needs pv - 01923_network_receive_time_metric_insert - - 01889_sqlite_read_write - - # needs s2 - 01849_geoToS2 - 01851_s2_to_geo - 01852_s2_get_neighbours - 01853_s2_cells_intersect - 01854_s2_cap_contains - 01854_s2_cap_union - - # needs s3 - 01944_insert_partition_by - - # depends on Go - 02013_zlib_read_after_eof - ) - - time clickhouse-test --hung-check -j 8 --order=random --use-skip-list \ - --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" \ + time clickhouse-test --hung-check -j 8 --order=random \ + --fast-tests-only --no-long --testname --shard --zookeeper \ -- "$FASTTEST_FOCUS" 2>&1 \ | ts '%Y-%m-%d %H:%M:%S' \ | tee "$FASTTEST_OUTPUT/test_log.txt" diff --git a/docker/test/fuzzer/Dockerfile b/docker/test/fuzzer/Dockerfile index 9a96ac1dfa7..6444e745c47 100644 --- a/docker/test/fuzzer/Dockerfile +++ b/docker/test/fuzzer/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-fuzzer . -FROM yandex/clickhouse-test-base +# docker build -t clickhouse/fuzzer . +FROM clickhouse/test-base ENV LANG=C.UTF-8 ENV TZ=Europe/Moscow @@ -36,5 +36,5 @@ CMD set -o pipefail \ && cd /workspace \ && /run-fuzzer.sh 2>&1 | ts "$(printf '%%Y-%%m-%%d %%H:%%M:%%S\t')" | tee main.log -# docker run --network=host --volume :/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> yandex/clickhouse-fuzzer +# docker run --network=host --volume :/workspace -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/fuzzer diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 603c35ede54..0b26476c20e 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -12,7 +12,7 @@ stage=${stage:-} script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" echo "$script_dir" repo_dir=ch -BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-11_debug_none_bundled_unsplitted_disable_False_binary"} +BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-13_debug_none_bundled_unsplitted_disable_False_binary"} function clone { diff --git a/docker/test/integration/base/Dockerfile b/docker/test/integration/base/Dockerfile index 344c1b9a698..519c64297e5 100644 --- a/docker/test/integration/base/Dockerfile +++ b/docker/test/integration/base/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-integration-test . -FROM yandex/clickhouse-test-base +# docker build -t clickhouse/integration-test . +FROM clickhouse/test-base SHELL ["/bin/bash", "-c"] diff --git a/docker/test/integration/helper_container/Dockerfile b/docker/test/integration/helper_container/Dockerfile index 922eb2c6f22..6a093081bf2 100644 --- a/docker/test/integration/helper_container/Dockerfile +++ b/docker/test/integration/helper_container/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-integration-helper . +# docker build -t clickhouse/integration-helper . # Helper docker container to run iptables without sudo FROM alpine diff --git a/docker/test/integration/kerberized_hadoop/Dockerfile b/docker/test/integration/kerberized_hadoop/Dockerfile index 6a2fd96e7a7..11da590f901 100644 --- a/docker/test/integration/kerberized_hadoop/Dockerfile +++ b/docker/test/integration/kerberized_hadoop/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-kerberized-hadoop . +# docker build -t clickhouse/kerberized-hadoop . FROM sequenceiq/hadoop-docker:2.7.0 RUN sed -i -e 's/^\#baseurl/baseurl/' /etc/yum.repos.d/CentOS-Base.repo diff --git a/docker/test/integration/kerberos_kdc/Dockerfile b/docker/test/integration/kerberos_kdc/Dockerfile index ea231b1191d..a203c33a331 100644 --- a/docker/test/integration/kerberos_kdc/Dockerfile +++ b/docker/test/integration/kerberos_kdc/Dockerfile @@ -1,9 +1,9 @@ -# docker build -t yandex/clickhouse-kerberos-kdc . +# docker build -t clickhouse/kerberos-kdc . +FROM centos:6 -FROM centos:6.6 -# old OS to make is faster and smaller +RUN sed -i '/^mirrorlist/s/^/#/;/^#baseurl/{s/#//;s/mirror.centos.org\/centos\/$releasever/vault.centos.org\/6.10/}' /etc/yum.repos.d/*B* -RUN yum install -y krb5-server krb5-libs krb5-auth-dialog krb5-workstation +RUN yum install -y ca-certificates krb5-server krb5-libs krb5-auth-dialog krb5-workstation EXPOSE 88 749 diff --git a/docker/test/integration/mysql_golang_client/Dockerfile b/docker/test/integration/mysql_golang_client/Dockerfile index 4380383d1fb..68b0aaab42c 100644 --- a/docker/test/integration/mysql_golang_client/Dockerfile +++ b/docker/test/integration/mysql_golang_client/Dockerfile @@ -1,7 +1,7 @@ -# docker build -t yandex/clickhouse-mysql-golang-client . +# docker build -t clickhouse/mysql-golang-client . # MySQL golang client docker container -FROM golang:1.12.2 +FROM golang:1.13 RUN go get "github.com/go-sql-driver/mysql" diff --git a/docker/test/integration/mysql_java_client/Dockerfile b/docker/test/integration/mysql_java_client/Dockerfile index fcb6a39f33b..0abf50cd493 100644 --- a/docker/test/integration/mysql_java_client/Dockerfile +++ b/docker/test/integration/mysql_java_client/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-mysql-java-client . +# docker build -t clickhouse/mysql-java-client . # MySQL Java client docker container FROM ubuntu:18.04 diff --git a/docker/test/integration/mysql_js_client/Dockerfile b/docker/test/integration/mysql_js_client/Dockerfile index 4f12de004ac..b1397b40d38 100644 --- a/docker/test/integration/mysql_js_client/Dockerfile +++ b/docker/test/integration/mysql_js_client/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-mysql-js-client . +# docker build -t clickhouse/mysql-js-client . # MySQL JavaScript client docker container FROM node:8 diff --git a/docker/test/integration/mysql_php_client/Dockerfile b/docker/test/integration/mysql_php_client/Dockerfile index e2ceb62f44f..0fb77bf8ffb 100644 --- a/docker/test/integration/mysql_php_client/Dockerfile +++ b/docker/test/integration/mysql_php_client/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-mysql-php-client . +# docker build -t clickhouse/mysql-php-client . # MySQL PHP client docker container FROM php:7.3-cli diff --git a/docker/test/integration/postgresql_java_client/Dockerfile b/docker/test/integration/postgresql_java_client/Dockerfile index eab236c9590..f5484028ec9 100644 --- a/docker/test/integration/postgresql_java_client/Dockerfile +++ b/docker/test/integration/postgresql_java_client/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-postgresql-java-client . +# docker build -t clickhouse/postgresql-java-client . # PostgreSQL Java client docker container FROM ubuntu:18.04 diff --git a/docker/test/integration/resolver/Dockerfile b/docker/test/integration/resolver/Dockerfile index b0efb4b46d5..01b9b777614 100644 --- a/docker/test/integration/resolver/Dockerfile +++ b/docker/test/integration/resolver/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-python-bottle . +# docker build -t clickhouse/python-bottle . # Helper docker container to run python bottle apps FROM python:3 diff --git a/docker/test/integration/runner/Dockerfile b/docker/test/integration/runner/Dockerfile index ef333e461c5..06e1f64ced2 100644 --- a/docker/test/integration/runner/Dockerfile +++ b/docker/test/integration/runner/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-integration-tests-runner . +# docker build -t clickhouse/integration-tests-runner . FROM ubuntu:20.04 RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list diff --git a/docker/test/integration/runner/compose/docker_compose_jdbc_bridge.yml b/docker/test/integration/runner/compose/docker_compose_jdbc_bridge.yml index a65ef629df6..b3686adc21c 100644 --- a/docker/test/integration/runner/compose/docker_compose_jdbc_bridge.yml +++ b/docker/test/integration/runner/compose/docker_compose_jdbc_bridge.yml @@ -1,7 +1,7 @@ version: '2.3' services: bridge1: - image: yandex/clickhouse-jdbc-bridge + image: clickhouse/jdbc-bridge command: | /bin/bash -c 'cat << EOF > config/datasources/self.json { diff --git a/docker/test/integration/runner/compose/docker_compose_keeper.yml b/docker/test/integration/runner/compose/docker_compose_keeper.yml index e11a13e6eab..134ffbff1f7 100644 --- a/docker/test/integration/runner/compose/docker_compose_keeper.yml +++ b/docker/test/integration/runner/compose/docker_compose_keeper.yml @@ -1,7 +1,7 @@ version: '2.3' services: zoo1: - image: ${image:-yandex/clickhouse-integration-test} + image: ${image:-clickhouse/integration-test} restart: always user: ${user:-} volumes: @@ -31,7 +31,7 @@ services: - inet6 - rotate zoo2: - image: ${image:-yandex/clickhouse-integration-test} + image: ${image:-clickhouse/integration-test} restart: always user: ${user:-} volumes: @@ -61,7 +61,7 @@ services: - inet6 - rotate zoo3: - image: ${image:-yandex/clickhouse-integration-test} + image: ${image:-clickhouse/integration-test} restart: always user: ${user:-} volumes: diff --git a/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml b/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml index b09e75a8515..88be3e45085 100644 --- a/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml +++ b/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml @@ -4,7 +4,7 @@ services: kerberizedhdfs1: cap_add: - DAC_READ_SEARCH - image: yandex/clickhouse-kerberized-hadoop:16621 + image: clickhouse/kerberized-hadoop hostname: kerberizedhdfs1 restart: always volumes: @@ -22,7 +22,7 @@ services: entrypoint: /etc/bootstrap.sh -d hdfskerberos: - image: yandex/clickhouse-kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest} + image: clickhouse/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest} hostname: hdfskerberos volumes: - ${KERBERIZED_HDFS_DIR}/secrets:/tmp/keytab diff --git a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml index 081b90c4f27..d57e4e4d5be 100644 --- a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml +++ b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml @@ -50,7 +50,7 @@ services: - label:disable kafka_kerberos: - image: yandex/clickhouse-kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest} + image: clickhouse/kerberos-kdc:${DOCKER_KERBEROS_KDC_TAG:-latest} hostname: kafka_kerberos volumes: - ${KERBERIZED_KAFKA_DIR}/secrets:/tmp/keytab diff --git a/docker/test/integration/runner/compose/docker_compose_minio.yml b/docker/test/integration/runner/compose/docker_compose_minio.yml index 96a5f8bdc31..4384c0868a5 100644 --- a/docker/test/integration/runner/compose/docker_compose_minio.yml +++ b/docker/test/integration/runner/compose/docker_compose_minio.yml @@ -19,14 +19,14 @@ services: # HTTP proxies for Minio. proxy1: - image: yandex/clickhouse-s3-proxy + image: clickhouse/s3-proxy expose: - "8080" # Redirect proxy port - "80" # Reverse proxy port - "443" # Reverse proxy port (secure) proxy2: - image: yandex/clickhouse-s3-proxy + image: clickhouse/s3-proxy expose: - "8080" - "80" @@ -34,7 +34,7 @@ services: # Empty container to run proxy resolver. resolver: - image: yandex/clickhouse-python-bottle + image: clickhouse/python-bottle expose: - "8080" tty: true diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_golang_client.yml b/docker/test/integration/runner/compose/docker_compose_mysql_golang_client.yml index a6a338eb6a8..56cc0410574 100644 --- a/docker/test/integration/runner/compose/docker_compose_mysql_golang_client.yml +++ b/docker/test/integration/runner/compose/docker_compose_mysql_golang_client.yml @@ -1,6 +1,6 @@ version: '2.3' services: golang1: - image: yandex/clickhouse-mysql-golang-client:${DOCKER_MYSQL_GOLANG_CLIENT_TAG:-latest} + image: clickhouse/mysql-golang-client:${DOCKER_MYSQL_GOLANG_CLIENT_TAG:-latest} # to keep container running command: sleep infinity diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_java_client.yml b/docker/test/integration/runner/compose/docker_compose_mysql_java_client.yml index 21d927df82c..eb5ffb01baa 100644 --- a/docker/test/integration/runner/compose/docker_compose_mysql_java_client.yml +++ b/docker/test/integration/runner/compose/docker_compose_mysql_java_client.yml @@ -1,6 +1,6 @@ version: '2.3' services: java1: - image: yandex/clickhouse-mysql-java-client:${DOCKER_MYSQL_JAVA_CLIENT_TAG:-latest} + image: clickhouse/mysql-java-client:${DOCKER_MYSQL_JAVA_CLIENT_TAG:-latest} # to keep container running command: sleep infinity diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_js_client.yml b/docker/test/integration/runner/compose/docker_compose_mysql_js_client.yml index dbd85cf2382..90939449c5f 100644 --- a/docker/test/integration/runner/compose/docker_compose_mysql_js_client.yml +++ b/docker/test/integration/runner/compose/docker_compose_mysql_js_client.yml @@ -1,6 +1,6 @@ version: '2.3' services: mysqljs1: - image: yandex/clickhouse-mysql-js-client:${DOCKER_MYSQL_JS_CLIENT_TAG:-latest} + image: clickhouse/mysql-js-client:${DOCKER_MYSQL_JS_CLIENT_TAG:-latest} # to keep container running command: sleep infinity diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_php_client.yml b/docker/test/integration/runner/compose/docker_compose_mysql_php_client.yml index f24f5337a7e..408b8ff089a 100644 --- a/docker/test/integration/runner/compose/docker_compose_mysql_php_client.yml +++ b/docker/test/integration/runner/compose/docker_compose_mysql_php_client.yml @@ -1,6 +1,6 @@ version: '2.3' services: php1: - image: yandex/clickhouse-mysql-php-client:${DOCKER_MYSQL_PHP_CLIENT_TAG:-latest} + image: clickhouse/mysql-php-client:${DOCKER_MYSQL_PHP_CLIENT_TAG:-latest} # to keep container running command: sleep infinity diff --git a/docker/test/integration/runner/compose/docker_compose_postgresql_java_client.yml b/docker/test/integration/runner/compose/docker_compose_postgresql_java_client.yml index 38191f1bdd6..904bfffdfd5 100644 --- a/docker/test/integration/runner/compose/docker_compose_postgresql_java_client.yml +++ b/docker/test/integration/runner/compose/docker_compose_postgresql_java_client.yml @@ -1,6 +1,6 @@ version: '2.2' services: java: - image: yandex/clickhouse-postgresql-java-client:${DOCKER_POSTGRESQL_JAVA_CLIENT_TAG:-latest} + image: clickhouse/postgresql-java-client:${DOCKER_POSTGRESQL_JAVA_CLIENT_TAG:-latest} # to keep container running command: sleep infinity diff --git a/docker/test/integration/s3_proxy/Dockerfile b/docker/test/integration/s3_proxy/Dockerfile index d8b1754fa71..5858218e4e4 100644 --- a/docker/test/integration/s3_proxy/Dockerfile +++ b/docker/test/integration/s3_proxy/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-s3-proxy . +# docker build -t clickhouse/s3-proxy . FROM nginx:alpine COPY run.sh /run.sh diff --git a/docker/test/keeper-jepsen/Dockerfile b/docker/test/keeper-jepsen/Dockerfile index 1a62d5e793f..5bb7f9433c2 100644 --- a/docker/test/keeper-jepsen/Dockerfile +++ b/docker/test/keeper-jepsen/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-keeper-jepsen-test . -FROM yandex/clickhouse-test-base +# docker build -t clickhouse/keeper-jepsen-test . +FROM clickhouse/test-base ENV DEBIAN_FRONTEND=noninteractive ENV CLOJURE_VERSION=1.10.3.814 diff --git a/docker/test/keeper-jepsen/run.sh b/docker/test/keeper-jepsen/run.sh index 352585e16e3..d7534270e2c 100644 --- a/docker/test/keeper-jepsen/run.sh +++ b/docker/test/keeper-jepsen/run.sh @@ -2,7 +2,7 @@ set -euo pipefail -CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-11_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse"} +CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.yandex.net/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-13_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse"} CLICKHOUSE_REPO_PATH=${CLICKHOUSE_REPO_PATH:=""} diff --git a/docker/test/performance-comparison/Dockerfile b/docker/test/performance-comparison/Dockerfile index 1a61c4b274a..88b66d42ecb 100644 --- a/docker/test/performance-comparison/Dockerfile +++ b/docker/test/performance-comparison/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-performance-comparison . +# docker build -t clickhouse/performance-comparison . FROM ubuntu:18.04 ENV LANG=C.UTF-8 @@ -54,4 +54,4 @@ COPY * / # it gives '/bin/sh: 1: [bash,: not found' otherwise. CMD ["bash", "-c", "node=$((RANDOM % $(numactl --hardware | sed -n 's/^.*available:\\(.*\\)nodes.*$/\\1/p'))); echo Will bind to NUMA node $node; numactl --cpunodebind=$node --membind=$node /entrypoint.sh"] -# docker run --network=host --volume :/workspace --volume=:/output -e PR_TO_TEST=<> -e SHA_TO_TEST=<> yandex/clickhouse-performance-comparison +# docker run --network=host --volume :/workspace --volume=:/output -e PR_TO_TEST=<> -e SHA_TO_TEST=<> clickhouse/performance-comparison diff --git a/docker/test/performance-comparison/README.md b/docker/test/performance-comparison/README.md index 782644a81dd..75213fad077 100644 --- a/docker/test/performance-comparison/README.md +++ b/docker/test/performance-comparison/README.md @@ -116,7 +116,7 @@ pull requests (0 for master) manually. docker run --network=host --volume=$(pwd)/workspace:/workspace --volume=$(pwd)/output:/output [-e REF_PR={} -e REF_SHA={}] -e PR_TO_TEST={} -e SHA_TO_TEST={} - yandex/clickhouse-performance-comparison + clickhouse/performance-comparison ``` Then see the `report.html` in the `output` directory. diff --git a/docker/test/pvs/Dockerfile b/docker/test/pvs/Dockerfile index 7bd45ba4018..f4675d35819 100644 --- a/docker/test/pvs/Dockerfile +++ b/docker/test/pvs/Dockerfile @@ -1,6 +1,6 @@ -# docker build -t yandex/clickhouse-pvs-test . +# docker build -t clickhouse/pvs-test . -FROM yandex/clickhouse-binary-builder +FROM clickhouse/binary-builder RUN apt-get update --yes \ && apt-get install \ @@ -28,7 +28,7 @@ RUN apt-get update --yes \ ENV PKG_VERSION="pvs-studio-latest" RUN set -x \ - && export PUBKEY_HASHSUM="686e5eb8b3c543a5c54442c39ec876b6c2d912fe8a729099e600017ae53c877dda3368fe38ed7a66024fe26df6b5892a" \ + && export PUBKEY_HASHSUM="ad369a2e9d8b8c30f5a9f2eb131121739b79c78e03fef0f016ea51871a5f78cd4e6257b270dca0ac3be3d1f19d885516" \ && wget -nv https://files.viva64.com/etc/pubkey.txt -O /tmp/pubkey.txt \ && echo "${PUBKEY_HASHSUM} /tmp/pubkey.txt" | sha384sum -c \ && apt-key add /tmp/pubkey.txt \ @@ -38,7 +38,7 @@ RUN set -x \ && dpkg -i "${PKG_VERSION}.deb" CMD echo "Running PVS version $PKG_VERSION" && cd /repo_folder && pvs-studio-analyzer credentials $LICENCE_NAME $LICENCE_KEY -o ./licence.lic \ - && cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF -D"USE_INTERNAL_PROTOBUF_LIBRARY"=OFF -D"USE_INTERNAL_GRPC_LIBRARY"=OFF \ + && cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF -D"USE_INTERNAL_PROTOBUF_LIBRARY"=OFF -D"USE_INTERNAL_GRPC_LIBRARY"=OFF -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang\+\+-13 \ && ninja re2_st clickhouse_grpc_protos \ && pvs-studio-analyzer analyze -o pvs-studio.log -e contrib -j 4 -l ./licence.lic; \ cp /repo_folder/pvs-studio.log /test_output; \ diff --git a/docker/test/split_build_smoke_test/Dockerfile b/docker/test/split_build_smoke_test/Dockerfile index 54a9eb17868..3cc2f26a507 100644 --- a/docker/test/split_build_smoke_test/Dockerfile +++ b/docker/test/split_build_smoke_test/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-split-build-smoke-test . -FROM yandex/clickhouse-binary-builder +# docker build -t clickhouse/split-build-smoke-test . +FROM clickhouse/binary-builder COPY run.sh /run.sh COPY process_split_build_smoke_test_result.py / diff --git a/docker/test/sqlancer/Dockerfile b/docker/test/sqlancer/Dockerfile index 67236402352..e73fd03fb6d 100644 --- a/docker/test/sqlancer/Dockerfile +++ b/docker/test/sqlancer/Dockerfile @@ -1,9 +1,9 @@ -# docker build -t yandex/clickhouse-sqlancer-test . +# docker build -t clickhouse/sqlancer-test . FROM ubuntu:20.04 RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list -RUN apt-get update --yes && env DEBIAN_FRONTEND=noninteractive apt-get install wget unzip git openjdk-14-jdk maven python3 --yes --no-install-recommends +RUN apt-get update --yes && env DEBIAN_FRONTEND=noninteractive apt-get install wget unzip git default-jdk maven python3 --yes --no-install-recommends RUN wget https://github.com/sqlancer/sqlancer/archive/master.zip -O /sqlancer.zip RUN mkdir /sqlancer && \ cd /sqlancer && \ diff --git a/docker/test/stateful/Dockerfile b/docker/test/stateful/Dockerfile index 07aad75a2ea..c237a712f52 100644 --- a/docker/test/stateful/Dockerfile +++ b/docker/test/stateful/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-stateful-test . -FROM yandex/clickhouse-stateless-test +# docker build -t clickhouse/stateful-test . +FROM clickhouse/stateless-test RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh index de058469192..dd5984fd7b5 100755 --- a/docker/test/stateful/run.sh +++ b/docker/test/stateful/run.sh @@ -108,7 +108,7 @@ function run_tests() ADDITIONAL_OPTIONS+=('--replicated-database') fi - clickhouse-test --testname --shard --zookeeper --no-stateless --hung-check --use-skip-list --print-time "${ADDITIONAL_OPTIONS[@]}" \ + clickhouse-test --testname --shard --zookeeper --no-stateless --hung-check --print-time "${ADDITIONAL_OPTIONS[@]}" \ "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt } diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index b66fa055e7b..3b5edb2c869 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-stateless-test . -FROM yandex/clickhouse-test-base +# docker build -t clickhouse/stateless-test . +FROM clickhouse/test-base ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz" diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index 154126a3880..ed721690281 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -97,7 +97,7 @@ function run_tests() fi clickhouse-test --testname --shard --zookeeper --hung-check --print-time \ - --use-skip-list --test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \ + --test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \ | ts '%Y-%m-%d %H:%M:%S' \ | tee -a test_output/test_result.txt } diff --git a/docker/test/stateless_pytest/Dockerfile b/docker/test/stateless_pytest/Dockerfile index 947a70426d6..c1e47523f6d 100644 --- a/docker/test/stateless_pytest/Dockerfile +++ b/docker/test/stateless_pytest/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-stateless-pytest . -FROM yandex/clickhouse-test-base +# docker build -t clickhouse/stateless-pytest . +FROM clickhouse/test-base RUN apt-get update -y && \ apt-get install -y --no-install-recommends \ diff --git a/docker/test/stateless_unbundled/Dockerfile b/docker/test/stateless_unbundled/Dockerfile index 53857a90ac7..dfe441e08a6 100644 --- a/docker/test/stateless_unbundled/Dockerfile +++ b/docker/test/stateless_unbundled/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-stateless-unbundled-test . -FROM yandex/clickhouse-test-base +# docker build -t clickhouse/stateless-unbundled-test . +FROM clickhouse/test-base ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz" diff --git a/docker/test/stateless_unbundled/run.sh b/docker/test/stateless_unbundled/run.sh index 8b8612d4211..f8396706ed2 100755 --- a/docker/test/stateless_unbundled/run.sh +++ b/docker/test/stateless_unbundled/run.sh @@ -13,8 +13,4 @@ dpkg -i package_folder/clickhouse-test_*.deb service clickhouse-server start && sleep 5 -if grep -q -- "--use-skip-list" /usr/bin/clickhouse-test; then - SKIP_LIST_OPT="--use-skip-list" -fi - -clickhouse-test --testname --shard --zookeeper "$SKIP_LIST_OPT" "$ADDITIONAL_OPTIONS" "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt +clickhouse-test --testname --shard --zookeeper "$ADDITIONAL_OPTIONS" "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt diff --git a/docker/test/stress/Dockerfile b/docker/test/stress/Dockerfile index e1df32ec3d7..3fe1b790d5a 100644 --- a/docker/test/stress/Dockerfile +++ b/docker/test/stress/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-stress-test . -FROM yandex/clickhouse-stateful-test +# docker build -t clickhouse/stress-test . +FROM clickhouse/stateful-test RUN apt-get update -y \ && env DEBIAN_FRONTEND=noninteractive \ diff --git a/docker/test/stress/README.md b/docker/test/stress/README.md index f747996fa2d..b1519e7968d 100644 --- a/docker/test/stress/README.md +++ b/docker/test/stress/README.md @@ -6,7 +6,7 @@ Usage: ``` $ ls $HOME/someclickhouse clickhouse-client_18.14.9_all.deb clickhouse-common-static_18.14.9_amd64.deb clickhouse-server_18.14.9_all.deb clickhouse-test_18.14.9_all.deb -$ docker run --volume=$HOME/someclickhouse:/package_folder --volume=$HOME/test_output:/test_output yandex/clickhouse-stress-test +$ docker run --volume=$HOME/someclickhouse:/package_folder --volume=$HOME/test_output:/test_output clickhouse/stress-test Selecting previously unselected package clickhouse-common-static. (Reading database ... 14442 files and directories currently installed.) ... diff --git a/docker/test/stress/stress b/docker/test/stress/stress index c71722809d7..73a84ad4c40 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -10,14 +10,6 @@ import logging import time -def get_skip_list_cmd(path): - with open(path, 'r') as f: - for line in f: - if '--use-skip-list' in line: - return '--use-skip-list' - return '' - - def get_options(i): options = [] client_options = [] @@ -56,8 +48,6 @@ def get_options(i): def run_func_test(cmd, output_prefix, num_processes, skip_tests_option, global_time_limit): - skip_list_opt = get_skip_list_cmd(cmd) - global_time_limit_option = '' if global_time_limit: global_time_limit_option = "--global_time_limit={}".format(global_time_limit) @@ -66,7 +56,7 @@ def run_func_test(cmd, output_prefix, num_processes, skip_tests_option, global_t pipes = [] for i in range(0, len(output_paths)): f = open(output_paths[i], 'w') - full_command = "{} {} {} {} {}".format(cmd, skip_list_opt, get_options(i), global_time_limit_option, skip_tests_option) + full_command = "{} {} {} {}".format(cmd, get_options(i), global_time_limit_option, skip_tests_option) logging.info("Run func tests '%s'", full_command) p = Popen(full_command, shell=True, stdout=f, stderr=f) pipes.append(p) diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index c0b3b0102cf..33cdb9db57a 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-style-test . +# docker build -t clickhouse/style-test . FROM ubuntu:20.04 RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list diff --git a/docker/test/test_runner.sh b/docker/test/test_runner.sh index cd6367b2964..0c99c8c2b32 100755 --- a/docker/test/test_runner.sh +++ b/docker/test/test_runner.sh @@ -49,7 +49,7 @@ fi # Build server image (optional) from local packages if [ -z "${CLICKHOUSE_SERVER_IMAGE}" ]; then - CLICKHOUSE_SERVER_IMAGE="yandex/clickhouse-server:local" + CLICKHOUSE_SERVER_IMAGE="clickhouse/server:local" if [ "${CLICKHOUSE_PACKAGES_ARG}" != "${NO_REBUILD_FLAG}" ]; then docker build --network=host \ diff --git a/docker/test/testflows/runner/Dockerfile b/docker/test/testflows/runner/Dockerfile index 81d431635b7..91d0eb844d9 100644 --- a/docker/test/testflows/runner/Dockerfile +++ b/docker/test/testflows/runner/Dockerfile @@ -1,4 +1,4 @@ -# docker build -t yandex/clickhouse-testflows-runner . +# docker build -t clickhouse/testflows-runner . FROM ubuntu:20.04 RUN sed -i 's|http://archive|http://ru.archive|g' /etc/apt/sources.list diff --git a/docker/test/unit/Dockerfile b/docker/test/unit/Dockerfile index e111611eecd..20d67773363 100644 --- a/docker/test/unit/Dockerfile +++ b/docker/test/unit/Dockerfile @@ -1,5 +1,5 @@ -# docker build -t yandex/clickhouse-unit-test . -FROM yandex/clickhouse-stateless-test +# docker build -t clickhouse/unit-test . +FROM clickhouse/stateless-test RUN apt-get install gdb diff --git a/docs/en/development/build-osx.md b/docs/en/development/build-osx.md index 687e0179e07..91e4902d0d2 100644 --- a/docs/en/development/build-osx.md +++ b/docs/en/development/build-osx.md @@ -76,7 +76,7 @@ cd ClickHouse rm -rf build mkdir build cd build -cmake -DCMAKE_C_COMPILER=$(brew --prefix gcc)/bin/gcc-10 -DCMAKE_CXX_COMPILER=$(brew --prefix gcc)/bin/g++-10 -DCMAKE_BUILD_TYPE=RelWithDebInfo .. +cmake -DCMAKE_C_COMPILER=$(brew --prefix gcc)/bin/gcc-11 -DCMAKE_CXX_COMPILER=$(brew --prefix gcc)/bin/g++-11 -DCMAKE_BUILD_TYPE=RelWithDebInfo .. cmake --build . --config RelWithDebInfo cd .. ``` diff --git a/docs/en/development/build.md b/docs/en/development/build.md index be45c1ed5f7..a246d3dea0a 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -23,7 +23,7 @@ $ sudo apt-get install git cmake python ninja-build Or cmake3 instead of cmake on older systems. -### Install clang-11 (recommended) {#install-clang-11} +### Install clang-13 (recommended) {#install-clang-13} On Ubuntu/Debian you can use the automatic installation script (check [official webpage](https://apt.llvm.org/)) @@ -33,11 +33,11 @@ sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" For other Linux distribution - check the availability of the [prebuild packages](https://releases.llvm.org/download.html) or build clang [from sources](https://clang.llvm.org/get_started.html). -#### Use clang-11 for Builds +#### Use clang-13 for Builds ``` bash -$ export CC=clang-11 -$ export CXX=clang++-11 +$ export CC=clang-13 +$ export CXX=clang++-13 ``` Gcc can also be used though it is discouraged. diff --git a/docs/en/engines/table-engines/integrations/mysql.md b/docs/en/engines/table-engines/integrations/mysql.md index a6402e00bc9..7eac159a645 100644 --- a/docs/en/engines/table-engines/integrations/mysql.md +++ b/docs/en/engines/table-engines/integrations/mysql.md @@ -19,6 +19,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] SETTINGS [connection_pool_size=16, ] [connection_max_tries=3, ] + [connection_wait_timeout=5, ] /* 0 -- do not wait */ [connection_auto_close=true ] ; ``` diff --git a/docs/en/engines/table-engines/integrations/s3.md b/docs/en/engines/table-engines/integrations/s3.md index 7249e24aff9..e494e9aec6a 100644 --- a/docs/en/engines/table-engines/integrations/s3.md +++ b/docs/en/engines/table-engines/integrations/s3.md @@ -210,4 +210,4 @@ ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/big_prefix/file- ## See also -- [S3 table function](../../../sql-reference/table-functions/s3.md) +- [s3 table function](../../../sql-reference/table-functions/s3.md) diff --git a/docs/en/engines/table-engines/mergetree-family/replication.md b/docs/en/engines/table-engines/mergetree-family/replication.md index 277339f9b47..8092baba199 100644 --- a/docs/en/engines/table-engines/mergetree-family/replication.md +++ b/docs/en/engines/table-engines/mergetree-family/replication.md @@ -288,5 +288,7 @@ If the data in ZooKeeper was lost or damaged, you can save data by moving it to - [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) - [background_fetches_pool_size](../../../operations/settings/settings.md#background_fetches_pool_size) - [execute_merges_on_single_replica_time_threshold](../../../operations/settings/settings.md#execute-merges-on-single-replica-time-threshold) +- [max_replicated_fetches_network_bandwidth](../../../operations/settings/merge-tree-settings.md#max_replicated_fetches_network_bandwidth) +- [max_replicated_sends_network_bandwidth](../../../operations/settings/merge-tree-settings.md#max_replicated_sends_network_bandwidth) [Original article](https://clickhouse.tech/docs/en/operations/table_engines/replication/) diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index 517be8704e4..66ee99ef345 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -38,6 +38,10 @@ You can also download and install packages manually from [here](https://repo.cli - `clickhouse-client` — Creates a symbolic link for `clickhouse-client` and other client-related tools. and installs client configuration files. - `clickhouse-common-static-dbg` — Installs ClickHouse compiled binary files with debug info. +!!! attention "Attention" + If you need to install specific version of ClickHouse you have to install all packages with the same version: + `sudo apt-get install clickhouse-server=21.8.5.7 clickhouse-client=21.8.5.7 clickhouse-common-static=21.8.5.7` + ### From RPM Packages {#from-rpm-packages} It is recommended to use official pre-compiled `rpm` packages for CentOS, RedHat, and all other rpm-based Linux distributions. diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index bac67ecf140..a533d9357e1 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -74,7 +74,7 @@ The `TabSeparated` format is convenient for processing data using custom program The `TabSeparated` format supports outputting total values (when using WITH TOTALS) and extreme values (when ‘extremes’ is set to 1). In these cases, the total values and extremes are output after the main data. The main result, total values, and extremes are separated from each other by an empty line. Example: ``` sql -SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT TabSeparated`` +SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT TabSeparated ``` ``` text @@ -1270,6 +1270,8 @@ You can insert Parquet data from a file into ClickHouse table by the following c $ cat {filename} | clickhouse-client --query="INSERT INTO {some_table} FORMAT Parquet" ``` +To insert data into [Nested](../sql-reference/data-types/nested-data-structures/nested.md) columns as an array of structs values you must switch on the [input_format_parquet_import_nested](../operations/settings/settings.md#input_format_parquet_import_nested) setting. + You can select data from a ClickHouse table and save them into some file in the Parquet format by the following command: ``` bash @@ -1328,6 +1330,8 @@ You can insert Arrow data from a file into ClickHouse table by the following com $ cat filename.arrow | clickhouse-client --query="INSERT INTO some_table FORMAT Arrow" ``` +To insert data into [Nested](../sql-reference/data-types/nested-data-structures/nested.md) columns as an array of structs values you must switch on the [input_format_arrow_import_nested](../operations/settings/settings.md#input_format_arrow_import_nested) setting. + ### Selecting Data {#selecting-data-arrow} You can select data from a ClickHouse table and save them into some file in the Arrow format by the following command: @@ -1384,6 +1388,8 @@ You can insert ORC data from a file into ClickHouse table by the following comma $ cat filename.orc | clickhouse-client --query="INSERT INTO some_table FORMAT ORC" ``` +To insert data into [Nested](../sql-reference/data-types/nested-data-structures/nested.md) columns as an array of structs values you must switch on the [input_format_orc_import_nested](../operations/settings/settings.md#input_format_orc_import_nested) setting. + ### Selecting Data {#selecting-data-2} You can select data from a ClickHouse table and save them into some file in the ORC format by the following command: diff --git a/docs/en/interfaces/third-party/gui.md b/docs/en/interfaces/third-party/gui.md index f2c6d1ee542..ad39adbd653 100644 --- a/docs/en/interfaces/third-party/gui.md +++ b/docs/en/interfaces/third-party/gui.md @@ -84,7 +84,7 @@ Features: - Table data preview. - Full-text search. -By default, DBeaver does not connect using a session (the CLI for example does). If you require session support (for example to set settings for your session), edit the driver connection properties and set session_id to a random string (it uses the http connection under the hood). Then you can use any setting from the query window +By default, DBeaver does not connect using a session (the CLI for example does). If you require session support (for example to set settings for your session), edit the driver connection properties and set `session_id` to a random string (it uses the http connection under the hood). Then you can use any setting from the query window. ### clickhouse-cli {#clickhouse-cli} diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index be19f476978..8ad450738b7 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -69,29 +69,85 @@ If no conditions met for a data part, ClickHouse uses the `lz4` compression. ``` - + +Load key from environment variable: + +```xml + + + + + +``` + +Where current_key_id sets the current key for encryption, and all specified keys can be used for decryption. + +All this methods can be applied for multiple keys: + +```xml + + + 00112233445566778899aabbccddeeff + + 1 + + +``` + +Where `current_key_id` shows current key for encryption. + +Also user can add nonce that must be 12 bytes long (by default encryption and decryption will use nonce consisting of zero bytes): + +```xml + + + 0123456789101 + + +``` + +Or it can be set in hex: + +```xml + + + abcdefabcdef + + +``` + +Everything above can be applied for `aes_256_gcm_siv` (but key must be 32 bytes length). + ## custom_settings_prefixes {#custom_settings_prefixes} List of prefixes for [custom settings](../../operations/settings/index.md#custom_settings). The prefixes must be separated with commas. diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md index 65d63438aea..a3a258234e1 100644 --- a/docs/en/operations/settings/merge-tree-settings.md +++ b/docs/en/operations/settings/merge-tree-settings.md @@ -181,6 +181,44 @@ Possible values: Default value: 0. +## max_replicated_fetches_network_bandwidth {#max_replicated_fetches_network_bandwidth} + +Limits the maximum speed of data exchange over the network in bytes per second for [replicated](../../engines/table-engines/mergetree-family/replication.md) fetches. This setting is applied to a particular table, unlike the [max_replicated_fetches_network_bandwidth_for_server](settings.md#max_replicated_fetches_network_bandwidth_for_server) setting, which is applied to the server. + +You can limit both server network and network for a particular table, but for this the value of the table-level setting should be less than server-level one. Otherwise the server considers only the `max_replicated_fetches_network_bandwidth_for_server` setting. + +The setting isn't followed perfectly accurately. + +Possible values: + +- Positive integer. +- 0 — Unlimited. + +Default value: `0`. + +**Usage** + +Could be used for throttling speed when replicating data to add or replace new nodes. + +## max_replicated_sends_network_bandwidth {#max_replicated_sends_network_bandwidth} + +Limits the maximum speed of data exchange over the network in bytes per second for [replicated](../../engines/table-engines/mergetree-family/replication.md) sends. This setting is applied to a particular table, unlike the [max_replicated_sends_network_bandwidth_for_server](settings.md#max_replicated_sends_network_bandwidth_for_server) setting, which is applied to the server. + +You can limit both server network and network for a particular table, but for this the value of the table-level setting should be less than server-level one. Otherwise the server considers only the `max_replicated_sends_network_bandwidth_for_server` setting. + +The setting isn't followed perfectly accurately. + +Possible values: + +- Positive integer. +- 0 — Unlimited. + +Default value: `0`. + +**Usage** + +Could be used for throttling speed when replicating data to add or replace new nodes. + ## old_parts_lifetime {#old-parts-lifetime} The time (in seconds) of storing inactive parts to protect against data loss during spontaneous server reboots. diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index c99c39fe05c..f286cf4e99e 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -260,6 +260,39 @@ If an error occurred while reading rows but the error counter is still less than If both `input_format_allow_errors_num` and `input_format_allow_errors_ratio` are exceeded, ClickHouse throws an exception. +## input_format_parquet_import_nested {#input_format_parquet_import_nested} + +Enables or disables the ability to insert the data into [Nested](../../sql-reference/data-types/nested-data-structures/nested.md) columns as an array of structs in [Parquet](../../interfaces/formats.md#data-format-parquet) input format. + +Possible values: + +- 0 — Data can not be inserted into `Nested` columns as an array of structs. +- 1 — Data can be inserted into `Nested` columns as an array of structs. + +Default value: `0`. + +## input_format_arrow_import_nested {#input_format_arrow_import_nested} + +Enables or disables the ability to insert the data into [Nested](../../sql-reference/data-types/nested-data-structures/nested.md) columns as an array of structs in [Arrow](../../interfaces/formats.md#data_types-matching-arrow) input format. + +Possible values: + +- 0 — Data can not be inserted into `Nested` columns as an array of structs. +- 1 — Data can be inserted into `Nested` columns as an array of structs. + +Default value: `0`. + +## input_format_orc_import_nested {#input_format_orc_import_nested} + +Enables or disables the ability to insert the data into [Nested](../../sql-reference/data-types/nested-data-structures/nested.md) columns as an array of structs in [ORC](../../interfaces/formats.md#data-format-orc) input format. + +Possible values: + +- 0 — Data can not be inserted into `Nested` columns as an array of structs. +- 1 — Data can be inserted into `Nested` columns as an array of structs. + +Default value: `0`. + ## input_format_values_interpret_expressions {#settings-input_format_values_interpret_expressions} Enables or disables the full SQL parser if the fast stream parser can’t parse the data. This setting is used only for the [Values](../../interfaces/formats.md#data-format-values) format at the data insertion. For more information about syntax parsing, see the [Syntax](../../sql-reference/syntax.md) section. @@ -1107,6 +1140,40 @@ Possible values: Default value: `5`. +## max_replicated_fetches_network_bandwidth_for_server {#max_replicated_fetches_network_bandwidth_for_server} + +Limits the maximum speed of data exchange over the network in bytes per second for [replicated](../../engines/table-engines/mergetree-family/replication.md) fetches for the server. Only has meaning at server startup. You can also limit the speed for a particular table with [max_replicated_fetches_network_bandwidth](../../operations/settings/merge-tree-settings.md#max_replicated_fetches_network_bandwidth) setting. + +The setting isn't followed perfectly accurately. + +Possible values: + +- Positive integer. +- 0 — Unlimited. + +Default value: `0`. + +**Usage** + +Could be used for throttling speed when replicating the data to add or replace new nodes. + +## max_replicated_sends_network_bandwidth_for_server {#max_replicated_sends_network_bandwidth_for_server} + +Limits the maximum speed of data exchange over the network in bytes per second for [replicated](../../engines/table-engines/mergetree-family/replication.md) sends for the server. Only has meaning at server startup. You can also limit the speed for a particular table with [max_replicated_sends_network_bandwidth](../../operations/settings/merge-tree-settings.md#max_replicated_sends_network_bandwidth) setting. + +The setting isn't followed perfectly accurately. + +Possible values: + +- Positive integer. +- 0 — Unlimited. + +Default value: `0`. + +**Usage** + +Could be used for throttling speed when replicating the data to add or replace new nodes. + ## connect_timeout_with_failover_ms {#connect-timeout-with-failover-ms} The timeout in milliseconds for connecting to a remote server for a Distributed table engine, if the ‘shard’ and ‘replica’ sections are used in the cluster definition. @@ -3466,6 +3533,30 @@ Possible values: Default value: `0`. +## replication_alter_partitions_sync {#replication-alter-partitions-sync} + +Allows to set up waiting for actions to be executed on replicas by [ALTER](../../sql-reference/statements/alter/index.md), [OPTIMIZE](../../sql-reference/statements/optimize.md) or [TRUNCATE](../../sql-reference/statements/truncate.md) queries. + +Possible values: + +- 0 — Do not wait. +- 1 — Wait for own execution. +- 2 — Wait for everyone. + +Default value: `1`. + +## replication_wait_for_inactive_replica_timeout {#replication-wait-for-inactive-replica-timeout} + +Specifies how long (in seconds) to wait for inactive replicas to execute [ALTER](../../sql-reference/statements/alter/index.md), [OPTIMIZE](../../sql-reference/statements/optimize.md) or [TRUNCATE](../../sql-reference/statements/truncate.md) queries. + +Possible values: + +- 0 — Do not wait. +- Negative integer — Wait for unlimited time. +- Positive integer — The number of seconds to wait. + +Default value: `120` seconds. + ## regexp_max_matches_per_row {#regexp-max-matches-per-row} Sets the maximum number of matches for a single regular expression per row. Use it to protect against memory overload when using greedy regular expression in the [extractAllGroupsHorizontal](../../sql-reference/functions/string-search-functions.md#extractallgroups-horizontal) function. diff --git a/docs/en/operations/system-tables/views.md b/docs/en/operations/system-tables/views.md deleted file mode 100644 index 8edebf00a91..00000000000 --- a/docs/en/operations/system-tables/views.md +++ /dev/null @@ -1,44 +0,0 @@ -# system.views {#system-views} - -Contains the dependencies of all views and the type to which the view belongs. The metadata of the view comes from the [system.tables](tables.md). - -Columns: - -- `database` ([String](../../sql-reference/data-types/string.md)) — The name of the database the view is in. - -- `name` ([String](../../sql-reference/data-types/string.md)) — Name of the view. - -- `main_dependency_database` ([String](../../sql-reference/data-types/string.md)) — The name of the database on which the view depends. - -- `main_dependency_table` ([String](../../sql-reference/data-types/string.md)) - The name of the table on which the view depends. - -- `view_type` ([Enum8](../../sql-reference/data-types/enum.md)) — Type of the view. Values: - - `'Default' = 1` — [Default views](../../sql-reference/statements/create/view.md#normal). Should not appear in this log. - - `'Materialized' = 2` — [Materialized views](../../sql-reference/statements/create/view.md#materialized). - - `'Live' = 3` — [Live views](../../sql-reference/statements/create/view.md#live-view). - -**Example** - -```sql -SELECT * FROM system.views LIMIT 2 FORMAT Vertical; -``` - -```text -Row 1: -────── -database: default -name: live_view -main_dependency_database: default -main_dependency_table: view_source_tb -view_type: Live - -Row 2: -────── -database: default -name: materialized_view -main_dependency_database: default -main_dependency_table: view_source_tb -view_type: Materialized -``` - -[Original article](https://clickhouse.tech/docs/en/operations/system-tables/views) diff --git a/docs/en/sql-reference/data-types/nested-data-structures/nested.md b/docs/en/sql-reference/data-types/nested-data-structures/nested.md index ec6c613a956..65849f9cd0f 100644 --- a/docs/en/sql-reference/data-types/nested-data-structures/nested.md +++ b/docs/en/sql-reference/data-types/nested-data-structures/nested.md @@ -3,7 +3,9 @@ toc_priority: 57 toc_title: Nested(Name1 Type1, Name2 Type2, ...) --- -# Nested(name1 Type1, Name2 Type2, …) {#nestedname1-type1-name2-type2} +# Nested {#nested} + +## Nested(name1 Type1, Name2 Type2, …) {#nestedname1-type1-name2-type2} A nested data structure is like a table inside a cell. The parameters of a nested data structure – the column names and types – are specified the same way as in a [CREATE TABLE](../../../sql-reference/statements/create/table.md) query. Each table row can correspond to any number of rows in a nested data structure. diff --git a/docs/en/sql-reference/functions/encoding-functions.md b/docs/en/sql-reference/functions/encoding-functions.md index c22f041e0c3..69dd14da1bf 100644 --- a/docs/en/sql-reference/functions/encoding-functions.md +++ b/docs/en/sql-reference/functions/encoding-functions.md @@ -87,7 +87,23 @@ The function is using uppercase letters `A-F` and not using any prefixes (like ` For integer arguments, it prints hex digits (“nibbles”) from the most significant to least significant (big-endian or “human-readable” order). It starts with the most significant non-zero byte (leading zero bytes are omitted) but always prints both digits of every byte even if the leading digit is zero. -**Example** +Values of type [Date](../../sql-reference/data-types/date.md) and [DateTime](../../sql-reference/data-types/datetime.md) are formatted as corresponding integers (the number of days since Epoch for Date and the value of Unix Timestamp for DateTime). + +For [String](../../sql-reference/data-types/string.md) and [FixedString](../../sql-reference/data-types/fixedstring.md), all bytes are simply encoded as two hexadecimal numbers. Zero bytes are not omitted. + +Values of [Float](../../sql-reference/data-types/float.md) and [Decimal](../../sql-reference/data-types/decimal.md) types are encoded as their representation in memory. As we support little-endian architecture, they are encoded in little-endian. Zero leading/trailing bytes are not omitted. + +**Arguments** + +- `arg` — A value to convert to hexadecimal. Types: [String](../../sql-reference/data-types/string.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md), [Decimal](../../sql-reference/data-types/decimal.md), [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md). + +**Returned value** + +- A string with the hexadecimal representation of the argument. + +Type: [String](../../sql-reference/data-types/string.md). + +**Examples** Query: @@ -101,28 +117,10 @@ Result: 01 ``` -Values of type `Date` and `DateTime` are formatted as corresponding integers (the number of days since Epoch for Date and the value of Unix Timestamp for DateTime). - -For `String` and `FixedString`, all bytes are simply encoded as two hexadecimal numbers. Zero bytes are not omitted. - -Values of floating point and Decimal types are encoded as their representation in memory. As we support little-endian architecture, they are encoded in little-endian. Zero leading/trailing bytes are not omitted. - -**Arguments** - -- `arg` — A value to convert to hexadecimal. Types: [String](../../sql-reference/data-types/string.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md), [Decimal](../../sql-reference/data-types/decimal.md), [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md). - -**Returned value** - -- A string with the hexadecimal representation of the argument. - -Type: `String`. - -**Example** - Query: ``` sql -SELECT hex(toFloat32(number)) as hex_presentation FROM numbers(15, 2); +SELECT hex(toFloat32(number)) AS hex_presentation FROM numbers(15, 2); ``` Result: @@ -137,7 +135,7 @@ Result: Query: ``` sql -SELECT hex(toFloat64(number)) as hex_presentation FROM numbers(15, 2); +SELECT hex(toFloat64(number)) AS hex_presentation FROM numbers(15, 2); ``` Result: @@ -210,52 +208,52 @@ Result: Returns a string containing the argument’s binary representation. -Alias: `BIN`. - **Syntax** ``` sql bin(arg) ``` +Alias: `BIN`. + For integer arguments, it prints bin digits from the most significant to least significant (big-endian or “human-readable” order). It starts with the most significant non-zero byte (leading zero bytes are omitted) but always prints eight digits of every byte if the leading digit is zero. -**Example** +Values of type [Date](../../sql-reference/data-types/date.md) and [DateTime](../../sql-reference/data-types/datetime.md) are formatted as corresponding integers (the number of days since Epoch for `Date` and the value of Unix Timestamp for `DateTime`). -Query: +For [String](../../sql-reference/data-types/string.md) and [FixedString](../../sql-reference/data-types/fixedstring.md), all bytes are simply encoded as eight binary numbers. Zero bytes are not omitted. -``` sql -SELECT bin(1); -``` - -Result: - -``` text -00000001 -``` - -Values of type `Date` and `DateTime` are formatted as corresponding integers (the number of days since Epoch for Date and the value of Unix Timestamp for DateTime). - -For `String` and `FixedString`, all bytes are simply encoded as eight binary numbers. Zero bytes are not omitted. - -Values of floating-point and Decimal types are encoded as their representation in memory. As we support little-endian architecture, they are encoded in little-endian. Zero leading/trailing bytes are not omitted. +Values of [Float](../../sql-reference/data-types/float.md) and [Decimal](../../sql-reference/data-types/decimal.md) types are encoded as their representation in memory. As we support little-endian architecture, they are encoded in little-endian. Zero leading/trailing bytes are not omitted. **Arguments** -- `arg` — A value to convert to binary. Types: [String](../../sql-reference/data-types/string.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md), [Decimal](../../sql-reference/data-types/decimal.md), [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md). +- `arg` — A value to convert to binary. [String](../../sql-reference/data-types/string.md), [FixedString](../../sql-reference/data-types/fixedstring.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md), [Decimal](../../sql-reference/data-types/decimal.md), [Date](../../sql-reference/data-types/date.md), or [DateTime](../../sql-reference/data-types/datetime.md). **Returned value** - A string with the binary representation of the argument. -Type: `String`. +Type: [String](../../sql-reference/data-types/string.md). -**Example** +**Examples** Query: ``` sql -SELECT bin(toFloat32(number)) as bin_presentation FROM numbers(15, 2); +SELECT bin(14); +``` + +Result: + +``` text +┌─bin(14)──┐ +│ 00001110 │ +└──────────┘ +``` + +Query: + +``` sql +SELECT bin(toFloat32(number)) AS bin_presentation FROM numbers(15, 2); ``` Result: @@ -270,7 +268,7 @@ Result: Query: ``` sql -SELECT bin(toFloat64(number)) as bin_presentation FROM numbers(15, 2); +SELECT bin(toFloat64(number)) AS bin_presentation FROM numbers(15, 2); ``` Result: @@ -284,14 +282,7 @@ Result: ## unbin {#unbinstr} -Performs the opposite operation of [bin](#bin). It interprets each pair of binary digits (in the argument) as a number and converts it to the byte represented by the number. The return value is a binary string (BLOB). - -If you want to convert the result to a number, you can use the [reverse](../../sql-reference/functions/string-functions.md#reverse) and [reinterpretAs](../../sql-reference/functions/type-conversion-functions.md#type-conversion-functions) functions. - -!!! note "Note" - If `unbin` is invoked from within the `clickhouse-client`, binary strings display using UTF-8. - -Alias: `UNBIN`. +Interprets each pair of binary digits (in the argument) as a number and converts it to the byte represented by the number. The functions performs the opposite operation to [bin](#bin). **Syntax** @@ -299,11 +290,18 @@ Alias: `UNBIN`. unbin(arg) ``` +Alias: `UNBIN`. + +For a numeric argument `unbin()` does not return the inverse of `bin()`. If you want to convert the result to a number, you can use the [reverse](../../sql-reference/functions/string-functions.md#reverse) and [reinterpretAs](../../sql-reference/functions/type-conversion-functions.md#reinterpretasuint8163264) functions. + +!!! note "Note" + If `unbin` is invoked from within the `clickhouse-client`, binary strings are displayed using UTF-8. + +Supports binary digits `0` and `1`. The number of binary digits does not have to be multiples of eight. If the argument string contains anything other than binary digits, some implementation-defined result is returned (an exception isn’t thrown). + **Arguments** -- `arg` — A string containing any number of binary digits. Type: [String](../../sql-reference/data-types/string.md). - -Supports binary digits `0-1`. The number of binary digits does not have to be multiples of eight. If the argument string contains anything other than binary digits, some implementation-defined result is returned (an exception isn’t thrown). For a numeric argument the inverse of bin(N) is not performed by unbin(). +- `arg` — A string containing any number of binary digits. [String](../../sql-reference/data-types/string.md). **Returned value** @@ -311,7 +309,7 @@ Supports binary digits `0-1`. The number of binary digits does not have to be mu Type: [String](../../sql-reference/data-types/string.md). -**Example** +**Examples** Query: @@ -330,14 +328,14 @@ Result: Query: ``` sql -SELECT reinterpretAsUInt64(reverse(unbin('1010'))) AS num; +SELECT reinterpretAsUInt64(reverse(unbin('1110'))) AS num; ``` Result: ``` text ┌─num─┐ -│ 10 │ +│ 14 │ └─────┘ ``` @@ -396,7 +394,7 @@ Result: Query: ``` sql -select bitPositionsToArray(toInt8(-1)) as bit_positions; +SELECT bitPositionsToArray(toInt8(-1)) AS bit_positions; ``` Result: diff --git a/docs/en/sql-reference/functions/index.md b/docs/en/sql-reference/functions/index.md index 54afd461e1d..47da4e6f3cc 100644 --- a/docs/en/sql-reference/functions/index.md +++ b/docs/en/sql-reference/functions/index.md @@ -59,6 +59,10 @@ A lambda function that accepts multiple arguments can also be passed to a higher For some functions the first argument (the lambda function) can be omitted. In this case, identical mapping is assumed. +## User Defined Functions {#user-defined-functions} + +Custom functions can be created using the [CREATE FUNCTION](../statements/create/function.md) statement. To delete these functions use the [DROP FUNCTION](../statements/drop.md#drop-function) statement. + ## Error Handling {#error-handling} Some functions might throw an exception if the data is invalid. In this case, the query is canceled and an error text is returned to the client. For distributed processing, when an exception occurs on one of the servers, the other servers also attempt to abort the query. diff --git a/docs/en/sql-reference/functions/tuple-map-functions.md b/docs/en/sql-reference/functions/tuple-map-functions.md index ef5f5814017..6e91419b04f 100644 --- a/docs/en/sql-reference/functions/tuple-map-functions.md +++ b/docs/en/sql-reference/functions/tuple-map-functions.md @@ -78,7 +78,7 @@ mapAdd(arg1, arg2 [, ...]) **Arguments** -Arguments are [maps](../../sql-reference/data-types/map.md) or [tuples](../../sql-reference/data-types/tuple.md#tuplet1-t2) of two [arrays](../../sql-reference/data-types/array.md#data-type-array), where items in the first array represent keys, and the second array contains values for the each key. All key arrays should have same type, and all value arrays should contain items which are promote to the one type ([Int64](../../sql-reference/data-types/int-uint.md#int-ranges), [UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges) or [Float64](../../sql-reference/data-types/float.md#float32-float64)). The common promoted type is used as a type for the result array. +Arguments are [maps](../../sql-reference/data-types/map.md) or [tuples](../../sql-reference/data-types/tuple.md#tuplet1-t2) of two [arrays](../../sql-reference/data-types/array.md#data-type-array), where items in the first array represent keys, and the second array contains values for the each key. All key arrays should have same type, and all value arrays should contain items which are promoted to the one type ([Int64](../../sql-reference/data-types/int-uint.md#int-ranges), [UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges) or [Float64](../../sql-reference/data-types/float.md#float32-float64)). The common promoted type is used as a type for the result array. **Returned value** @@ -86,7 +86,7 @@ Arguments are [maps](../../sql-reference/data-types/map.md) or [tuples](../../sq **Example** -Query with a tuple map: +Query with a tuple: ```sql SELECT mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])) as res, toTypeName(res) as type; diff --git a/docs/en/sql-reference/statements/alter/index.md b/docs/en/sql-reference/statements/alter/index.md index 71333e6fcce..382306016e6 100644 --- a/docs/en/sql-reference/statements/alter/index.md +++ b/docs/en/sql-reference/statements/alter/index.md @@ -43,7 +43,11 @@ Entries for finished mutations are not deleted right away (the number of preserv For non-replicated tables, all `ALTER` queries are performed synchronously. For replicated tables, the query just adds instructions for the appropriate actions to `ZooKeeper`, and the actions themselves are performed as soon as possible. However, the query can wait for these actions to be completed on all the replicas. -For `ALTER ... ATTACH|DETACH|DROP` queries, you can use the `replication_alter_partitions_sync` setting to set up waiting. Possible values: `0` – do not wait; `1` – only wait for own execution (default); `2` – wait for all. +For all `ALTER` queries, you can use the [replication_alter_partitions_sync](../../../operations/settings/settings.md#replication-alter-partitions-sync) setting to set up waiting. + +You can specify how long (in seconds) to wait for inactive replicas to execute all `ALTER` queries with the [replication_wait_for_inactive_replica_timeout](../../../operations/settings/settings.md#replication-wait-for-inactive-replica-timeout) setting. + +!!! info "Note" + For all `ALTER` queries, if `replication_alter_partitions_sync = 2` and some replicas are not active for more than the time, specified in the `replication_wait_for_inactive_replica_timeout` setting, then an exception `UNFINISHED` is thrown. For `ALTER TABLE ... UPDATE|DELETE` queries the synchronicity is defined by the [mutations_sync](../../../operations/settings/settings.md#mutations_sync) setting. - diff --git a/docs/en/sql-reference/statements/alter/index/index.md b/docs/en/sql-reference/statements/alter/index/index.md index fd5657c3666..4e2943d37f3 100644 --- a/docs/en/sql-reference/statements/alter/index/index.md +++ b/docs/en/sql-reference/statements/alter/index/index.md @@ -12,7 +12,7 @@ The following operations are available: - `ALTER TABLE [db].name DROP INDEX name` - Removes index description from tables metadata and deletes index files from disk. -- `ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name` - The query rebuilds the secondary index `name` in the partition `partition_name`. Implemented as a [mutation](../../../../sql-reference/statements/alter/index.md#mutations). +- `ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name` - The query rebuilds the secondary index `name` in the partition `partition_name`. Implemented as a [mutation](../../../../sql-reference/statements/alter/index.md#mutations). To rebuild index over the whole data in the table you need to remove `IN PARTITION` from query. The first two commands are lightweight in a sense that they only change metadata or remove files. diff --git a/docs/en/sql-reference/statements/create/function.md b/docs/en/sql-reference/statements/create/function.md new file mode 100644 index 00000000000..ddfcdfef521 --- /dev/null +++ b/docs/en/sql-reference/statements/create/function.md @@ -0,0 +1,59 @@ +--- +toc_priority: 38 +toc_title: FUNCTION +--- + +# CREATE FUNCTION {#create-function} + +Creates a user defined function from a lambda expression. The expression must consist of function parameters, constants, operators, or other function calls. + +**Syntax** + +```sql +CREATE FUNCTION name AS (parameter0, ...) -> expression +``` +A function can have an arbitrary number of parameters. + +There are a few restrictions: + +- The name of a function must be unique among user defined and system functions. +- Recursive functions are not allowed. +- All variables used by a function must be specified in its parameter list. + +If any restriction is violated then an exception is raised. + +**Example** + +Query: + +```sql +CREATE FUNCTION linear_equation AS (x, k, b) -> k*x + b; +SELECT number, linear_equation(number, 2, 1) FROM numbers(3); +``` + +Result: + +``` text +┌─number─┬─plus(multiply(2, number), 1)─┐ +│ 0 │ 1 │ +│ 1 │ 3 │ +│ 2 │ 5 │ +└────────┴──────────────────────────────┘ +``` + +A [conditional function](../../../sql-reference/functions/conditional-functions.md) is called in a user defined function in the following query: + +```sql +CREATE FUNCTION parity_str AS (n) -> if(n % 2, 'odd', 'even'); +SELECT number, parity_str(number) FROM numbers(3); +``` + +Result: + +``` text +┌─number─┬─if(modulo(number, 2), 'odd', 'even')─┐ +│ 0 │ even │ +│ 1 │ odd │ +│ 2 │ even │ +└────────┴──────────────────────────────────────┘ +``` diff --git a/docs/en/sql-reference/statements/create/index.md b/docs/en/sql-reference/statements/create/index.md index 902a4348bac..5721130dd24 100644 --- a/docs/en/sql-reference/statements/create/index.md +++ b/docs/en/sql-reference/statements/create/index.md @@ -12,6 +12,7 @@ Create queries make a new entity of one of the following kinds: - [TABLE](../../../sql-reference/statements/create/table.md) - [VIEW](../../../sql-reference/statements/create/view.md) - [DICTIONARY](../../../sql-reference/statements/create/dictionary.md) +- [FUNCTION](../../../sql-reference/statements/create/function.md) - [USER](../../../sql-reference/statements/create/user.md) - [ROLE](../../../sql-reference/statements/create/role.md) - [ROW POLICY](../../../sql-reference/statements/create/row-policy.md) diff --git a/docs/en/sql-reference/statements/drop.md b/docs/en/sql-reference/statements/drop.md index 90a2a46c7cf..552a7b5f1a9 100644 --- a/docs/en/sql-reference/statements/drop.md +++ b/docs/en/sql-reference/statements/drop.md @@ -97,4 +97,20 @@ Syntax: DROP VIEW [IF EXISTS] [db.]name [ON CLUSTER cluster] ``` -[Оriginal article](https://clickhouse.tech/docs/en/sql-reference/statements/drop/) +## DROP FUNCTION {#drop-function} + +Deletes a user defined function created by [CREATE FUNCTION](./create/function.md). +System functions can not be dropped. + +**Syntax** + +``` sql +DROP FUNCTION [IF EXISTS] function_name +``` + +**Example** + +``` sql +CREATE FUNCTION linear_equation AS (x, k, b) -> k*x + b; +DROP FUNCTION linear_equation; +``` diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md index 25dffc36954..2b3cd68fbb2 100644 --- a/docs/en/sql-reference/statements/grant.md +++ b/docs/en/sql-reference/statements/grant.md @@ -107,11 +107,13 @@ Hierarchy of privileges: - `CREATE TEMPORARY TABLE` - `CREATE VIEW` - `CREATE DICTIONARY` + - `CREATE FUNCTION` - [DROP](#grant-drop) - `DROP DATABASE` - `DROP TABLE` - `DROP VIEW` - `DROP DICTIONARY` + - `DROP FUNCTION` - [TRUNCATE](#grant-truncate) - [OPTIMIZE](#grant-optimize) - [SHOW](#grant-show) diff --git a/docs/en/sql-reference/statements/optimize.md b/docs/en/sql-reference/statements/optimize.md index 864509cec94..4054f373cc1 100644 --- a/docs/en/sql-reference/statements/optimize.md +++ b/docs/en/sql-reference/statements/optimize.md @@ -18,13 +18,17 @@ OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION I The `OPTMIZE` query is supported for [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) family, the [MaterializedView](../../engines/table-engines/special/materializedview.md) and the [Buffer](../../engines/table-engines/special/buffer.md) engines. Other table engines aren’t supported. -When `OPTIMIZE` is used with the [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md) family of table engines, ClickHouse creates a task for merging and waits for execution on all nodes (if the `replication_alter_partitions_sync` setting is enabled). +When `OPTIMIZE` is used with the [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md) family of table engines, ClickHouse creates a task for merging and waits for execution on all replicas (if the [replication_alter_partitions_sync](../../operations/settings/settings.md#replication-alter-partitions-sync) setting is set to `2`) or on current replica (if the [replication_alter_partitions_sync](../../operations/settings/settings.md#replication-alter-partitions-sync) setting is set to `1`). - If `OPTIMIZE` does not perform a merge for any reason, it does not notify the client. To enable notifications, use the [optimize_throw_if_noop](../../operations/settings/settings.md#setting-optimize_throw_if_noop) setting. - If you specify a `PARTITION`, only the specified partition is optimized. [How to set partition expression](../../sql-reference/statements/alter/index.md#alter-how-to-specify-part-expr). - If you specify `FINAL`, optimization is performed even when all the data is already in one part. Also merge is forced even if concurrent merges are performed. - If you specify `DEDUPLICATE`, then completely identical rows (unless by-clause is specified) will be deduplicated (all columns are compared), it makes sense only for the MergeTree engine. +You can specify how long (in seconds) to wait for inactive replicas to execute `OPTIMIZE` queries by the [replication_wait_for_inactive_replica_timeout](../../operations/settings/settings.md#replication-wait-for-inactive-replica-timeout) setting. + +!!! info "Note" + If the `replication_alter_partitions_sync` is set to `2` and some replicas are not active for more than the time, specified by the `replication_wait_for_inactive_replica_timeout` setting, then an exception `UNFINISHED` is thrown. ## BY expression {#by-expression} diff --git a/docs/en/sql-reference/statements/select/where.md b/docs/en/sql-reference/statements/select/where.md index f1532115e55..69505a51db4 100644 --- a/docs/en/sql-reference/statements/select/where.md +++ b/docs/en/sql-reference/statements/select/where.md @@ -6,7 +6,7 @@ toc_title: WHERE `WHERE` clause allows to filter the data that is coming from [FROM](../../../sql-reference/statements/select/from.md) clause of `SELECT`. -If there is a `WHERE` clause, it must contain an expression with the `UInt8` type. This is usually an expression with comparison and logical operators. Rows where this expression evaluates to 0 are expluded from further transformations or result. +If there is a `WHERE` clause, it must contain an expression with the `UInt8` type. This is usually an expression with comparison and logical operators. Rows where this expression evaluates to 0 are excluded from further transformations or result. `WHERE` expression is evaluated on the ability to use indexes and partition pruning, if the underlying table engine supports that. diff --git a/docs/en/sql-reference/statements/truncate.md b/docs/en/sql-reference/statements/truncate.md index f302a8605e2..b5354196fa4 100644 --- a/docs/en/sql-reference/statements/truncate.md +++ b/docs/en/sql-reference/statements/truncate.md @@ -12,3 +12,10 @@ TRUNCATE TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] Removes all data from a table. When the clause `IF EXISTS` is omitted, the query returns an error if the table does not exist. The `TRUNCATE` query is not supported for [View](../../engines/table-engines/special/view.md), [File](../../engines/table-engines/special/file.md), [URL](../../engines/table-engines/special/url.md), [Buffer](../../engines/table-engines/special/buffer.md) and [Null](../../engines/table-engines/special/null.md) table engines. + +You can use the [replication_alter_partitions_sync](../../operations/settings/settings.md#replication-alter-partitions-sync) setting to set up waiting for actions to be executed on replicas. + +You can specify how long (in seconds) to wait for inactive replicas to execute `TRUNCATE` queries with the [replication_wait_for_inactive_replica_timeout](../../operations/settings/settings.md#replication-wait-for-inactive-replica-timeout) setting. + +!!! info "Note" + If the `replication_alter_partitions_sync` is set to `2` and some replicas are not active for more than the time, specified by the `replication_wait_for_inactive_replica_timeout` setting, then an exception `UNFINISHED` is thrown. diff --git a/docs/en/sql-reference/table-functions/s3.md b/docs/en/sql-reference/table-functions/s3.md index d84edb3f46e..ffba8f5c6d3 100644 --- a/docs/en/sql-reference/table-functions/s3.md +++ b/docs/en/sql-reference/table-functions/s3.md @@ -3,7 +3,7 @@ toc_priority: 45 toc_title: s3 --- -# S3 Table Function {#s3-table-function} +# s3 Table Function {#s3-table-function} Provides table-like interface to select/insert files in [Amazon S3](https://aws.amazon.com/s3/). This table function is similar to [hdfs](../../sql-reference/table-functions/hdfs.md), but provides S3-specific features. @@ -125,6 +125,30 @@ INSERT INTO FUNCTION s3('https://storage.yandexcloud.net/my-test-bucket-768/test SELECT name, value FROM existing_table; ``` +## Partitioned Write {#partitioned-write} + +If you specify `PARTITION BY` expression when inserting data into `S3` table, a separate file is created for each partition value. Splitting the data into separate files helps to improve reading operations efficiency. + +**Examples** + +1. Using partition ID in a key creates separate files: + +```sql +INSERT INTO TABLE FUNCTION + s3('http://bucket.amazonaws.com/my_bucket/file_{_partition_id}.csv', 'CSV', 'a String, b UInt32, c UInt32') + PARTITION BY a VALUES ('x', 2, 3), ('x', 4, 5), ('y', 11, 12), ('y', 13, 14), ('z', 21, 22), ('z', 23, 24); +``` +As a result, the data is written into three files: `file_x.csv`, `file_y.csv`, and `file_z.csv`. + +2. Using partition ID in a bucket name creates files in different buckets: + +```sql +INSERT INTO TABLE FUNCTION + s3('http://bucket.amazonaws.com/my_bucket_{_partition_id}/file.csv', 'CSV', 'a UInt32, b UInt32, c UInt32') + PARTITION BY a VALUES (1, 2, 3), (1, 4, 5), (10, 11, 12), (10, 13, 14), (20, 21, 22), (20, 23, 24); +``` +As a result, the data is written into three files in different buckets: `my_bucket_1/file.csv`, `my_bucket_10/file.csv`, and `my_bucket_20/file.csv`. + **See Also** - [S3 engine](../../engines/table-engines/integrations/s3.md) diff --git a/docs/ru/development/build-osx.md b/docs/ru/development/build-osx.md index 8d5d06a544c..bdc80322f34 100644 --- a/docs/ru/development/build-osx.md +++ b/docs/ru/development/build-osx.md @@ -74,7 +74,7 @@ $ /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/ $ rm -rf build $ mkdir build $ cd build - $ cmake -DCMAKE_C_COMPILER=$(brew --prefix gcc)/bin/gcc-10 -DCMAKE_CXX_COMPILER=$(brew --prefix gcc)/bin/g++-10 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_JEMALLOC=OFF .. + $ cmake -DCMAKE_C_COMPILER=$(brew --prefix gcc)/bin/gcc-11 -DCMAKE_CXX_COMPILER=$(brew --prefix gcc)/bin/g++-11 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_JEMALLOC=OFF .. $ cmake --build . --config RelWithDebInfo $ cd .. ``` diff --git a/docs/ru/engines/database-engines/postgresql.md b/docs/ru/engines/database-engines/postgresql.md index 06e2b35b002..f68570d40eb 100644 --- a/docs/ru/engines/database-engines/postgresql.md +++ b/docs/ru/engines/database-engines/postgresql.md @@ -15,7 +15,7 @@ toc_title: PostgreSQL ``` sql CREATE DATABASE test_database -ENGINE = PostgreSQL('host:port', 'database', 'user', 'password'[, `use_table_cache`]); +ENGINE = PostgreSQL('host:port', 'database', 'user', 'password'[, `schema`, `use_table_cache`]); ``` **Параметры движка** @@ -24,6 +24,7 @@ ENGINE = PostgreSQL('host:port', 'database', 'user', 'password'[, `use_table_cac - `database` — имя удаленной БД. - `user` — пользователь PostgreSQL. - `password` — пароль пользователя. + - `schema` — схема PostgreSQL. - `use_table_cache` — определяет кеширование структуры таблиц БД. Необязательный параметр. Значение по умолчанию: `0`. ## Поддерживаемые типы данных {#data_types-support} diff --git a/docs/ru/engines/table-engines/integrations/s3.md b/docs/ru/engines/table-engines/integrations/s3.md index 5895bd43d2f..c90b7293e1c 100644 --- a/docs/ru/engines/table-engines/integrations/s3.md +++ b/docs/ru/engines/table-engines/integrations/s3.md @@ -151,4 +151,4 @@ ENGINE = S3('https://storage.yandexcloud.net/my-test-bucket-768/big_prefix/file- **Смотрите также** -- [Табличная функция S3](../../../sql-reference/table-functions/s3.md) +- [Табличная функция s3](../../../sql-reference/table-functions/s3.md) diff --git a/docs/ru/engines/table-engines/mergetree-family/replication.md b/docs/ru/engines/table-engines/mergetree-family/replication.md index db749ba455e..a7c1a7d2d1b 100644 --- a/docs/ru/engines/table-engines/mergetree-family/replication.md +++ b/docs/ru/engines/table-engines/mergetree-family/replication.md @@ -253,4 +253,5 @@ $ sudo -u clickhouse touch /var/lib/clickhouse/flags/force_restore_data - [background_schedule_pool_size](../../../operations/settings/settings.md#background_schedule_pool_size) - [background_fetches_pool_size](../../../operations/settings/settings.md#background_fetches_pool_size) - [execute_merges_on_single_replica_time_threshold](../../../operations/settings/settings.md#execute-merges-on-single-replica-time-threshold) - +- [max_replicated_fetches_network_bandwidth](../../../operations/settings/merge-tree-settings.md#max_replicated_fetches_network_bandwidth) +- [max_replicated_sends_network_bandwidth](../../../operations/settings/merge-tree-settings.md#max_replicated_sends_network_bandwidth) diff --git a/docs/ru/getting-started/install.md b/docs/ru/getting-started/install.md index 1cbeb70ef96..7bc21179e21 100644 --- a/docs/ru/getting-started/install.md +++ b/docs/ru/getting-started/install.md @@ -31,6 +31,19 @@ grep -q sse4_2 /proc/cpuinfo && echo "SSE 4.2 supported" || echo "SSE 4.2 not su Если вы хотите использовать наиболее свежую версию, замените `stable` на `testing` (рекомендуется для тестовых окружений). +Также вы можете вручную скачать и установить пакеты из [репозитория](https://repo.clickhouse.tech/deb/stable/main/). + +#### Пакеты {#packages} + +- `clickhouse-common-static` — Устанавливает исполняемые файлы ClickHouse. +- `clickhouse-server` — Создает символические ссылки для `clickhouse-server` и устанавливает конфигурационные файлы. +- `clickhouse-client` — Создает символические ссылки для `clickhouse-client` и других клиентских инструментов и устанавливает конфигурационные файлы `clickhouse-client`. +- `clickhouse-common-static-dbg` — Устанавливает исполняемые файлы ClickHouse собранные с отладочной информацией. + +!!! attention "Внимание" + Если вам нужно установить ClickHouse определенной версии, вы должны установить все пакеты одной версии: + `sudo apt-get install clickhouse-server=21.8.5.7 clickhouse-client=21.8.5.7 clickhouse-common-static=21.8.5.7` + ### Из RPM пакетов {#from-rpm-packages} Команда ClickHouse в Яндексе рекомендует использовать официальные предкомпилированные `rpm` пакеты для CentOS, RedHat и всех остальных дистрибутивов Linux, основанных на rpm. diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 1dd10e1f76d..970c6c36e9f 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1180,7 +1180,7 @@ ClickHouse поддерживает настраиваемую точность Типы данных столбцов в ClickHouse могут отличаться от типов данных соответствующих полей файла в формате Parquet. При вставке данных ClickHouse интерпретирует типы данных в соответствии с таблицей выше, а затем [приводит](../sql-reference/functions/type-conversion-functions/#type_conversion_function-cast) данные к тому типу, который установлен для столбца таблицы. -### Вставка и выборка данных {#vstavka-i-vyborka-dannykh} +### Вставка и выборка данных {#inserting-and-selecting-data} Чтобы вставить в ClickHouse данные из файла в формате Parquet, выполните команду следующего вида: @@ -1188,6 +1188,8 @@ ClickHouse поддерживает настраиваемую точность $ cat {filename} | clickhouse-client --query="INSERT INTO {some_table} FORMAT Parquet" ``` +Чтобы вставить данные в колонки типа [Nested](../sql-reference/data-types/nested-data-structures/nested.md) в виде массива структур, нужно включить настройку [input_format_parquet_import_nested](../operations/settings/settings.md#input_format_parquet_import_nested). + Чтобы получить данные из таблицы ClickHouse и сохранить их в файл формата Parquet, используйте команду следующего вида: ``` bash @@ -1246,6 +1248,8 @@ ClickHouse поддерживает настраиваемую точность $ cat filename.arrow | clickhouse-client --query="INSERT INTO some_table FORMAT Arrow" ``` +Чтобы вставить данные в колонки типа [Nested](../sql-reference/data-types/nested-data-structures/nested.md) в виде массива структур, нужно включить настройку [input_format_arrow_import_nested](../operations/settings/settings.md#input_format_arrow_import_nested). + ### Вывод данных {#selecting-data-arrow} Чтобы получить данные из таблицы ClickHouse и сохранить их в файл формата Arrow, используйте команду следующего вида: @@ -1294,7 +1298,7 @@ ClickHouse поддерживает настраиваемую точность Типы данных столбцов в таблицах ClickHouse могут отличаться от типов данных для соответствующих полей ORC. При вставке данных ClickHouse интерпретирует типы данных ORC согласно таблице соответствия, а затем [приводит](../sql-reference/functions/type-conversion-functions/#type_conversion_function-cast) данные к типу, установленному для столбца таблицы ClickHouse. -### Вставка данных {#vstavka-dannykh-1} +### Вставка данных {#inserting-data-2} Чтобы вставить в ClickHouse данные из файла в формате ORC, используйте команду следующего вида: @@ -1302,7 +1306,9 @@ ClickHouse поддерживает настраиваемую точность $ cat filename.orc | clickhouse-client --query="INSERT INTO some_table FORMAT ORC" ``` -### Вывод данных {#vyvod-dannykh-1} +Чтобы вставить данные в колонки типа [Nested](../sql-reference/data-types/nested-data-structures/nested.md) в виде массива структур, нужно включить настройку [input_format_orc_import_nested](../operations/settings/settings.md#input_format_orc_import_nested). + +### Вывод данных {#selecting-data-2} Чтобы получить данные из таблицы ClickHouse и сохранить их в файл формата ORC, используйте команду следующего вида: diff --git a/docs/ru/interfaces/third-party/gui.md b/docs/ru/interfaces/third-party/gui.md index 9cb28a2c9a2..fb14e9f955c 100644 --- a/docs/ru/interfaces/third-party/gui.md +++ b/docs/ru/interfaces/third-party/gui.md @@ -75,11 +75,13 @@ toc_title: "Визуальные интерфейсы от сторонних р Основные возможности: -- Построение запросов с подсветкой синтаксиса; -- Просмотр таблиц; -- Автодополнение команд; +- Построение запросов с подсветкой синтаксиса. +- Просмотр таблиц. +- Автодополнение команд. - Полнотекстовый поиск. +По умолчанию DBeaver не использует сессии при подключении (в отличие от CLI, например). Если вам нужна поддержка сессий (например, для установки настроек на сессию), измените настройки подключения драйвера и укажите для настройки `session_id` любое произвольное значение (драйвер использует подключение по http). После этого вы можете использовать любую настройку (setting) в окне запроса. + ### clickhouse-cli {#clickhouse-cli} [clickhouse-cli](https://github.com/hatarist/clickhouse-cli) - это альтернативный клиент командной строки для ClickHouse, написанный на Python 3. diff --git a/docs/ru/operations/settings/merge-tree-settings.md b/docs/ru/operations/settings/merge-tree-settings.md index 88c511d4d80..08ea9979426 100644 --- a/docs/ru/operations/settings/merge-tree-settings.md +++ b/docs/ru/operations/settings/merge-tree-settings.md @@ -201,6 +201,44 @@ Eсли суммарное число активных кусков во все Значение по умолчанию: `0`. +## max_replicated_fetches_network_bandwidth {#max_replicated_fetches_network_bandwidth} + +Ограничивает максимальную скорость скачивания данных в сети (в байтах в секунду) для синхронизаций между [репликами](../../engines/table-engines/mergetree-family/replication.md). Настройка применяется к конкретной таблице, в отличие от [max_replicated_fetches_network_bandwidth_for_server](settings.md#max_replicated_fetches_network_bandwidth_for_server), которая применяется к серверу. + +Можно ограничить скорость обмена данными как для всего сервера, так и для конкретной таблицы, но для этого значение табличной настройки должно быть меньше серверной. Иначе сервер будет учитывать только настройку `max_replicated_fetches_network_bandwidth_for_server`. + +Настройка соблюдается неточно. + +Возможные значения: + +- Любое целое положительное число. +- 0 — Скорость не ограничена. + +Значение по умолчанию: `0`. + +**Использование** + +Может быть использована для ограничения скорости передачи данных при репликации данных для добавления или замены новых узлов. + +## max_replicated_sends_network_bandwidth {#max_replicated_sends_network_bandwidth} + +Ограничивает максимальную скорость отправки данных по сети (в байтах в секунду) для синхронизации между [репликами](../../engines/table-engines/mergetree-family/replication.md). Настройка применяется к конкретной таблице, в отличие от [max_replicated_sends_network_bandwidth_for_server](settings.md#max_replicated_sends_network_bandwidth_for_server), которая применяется к серверу. + +Можно ограничить скорость обмена данными как для всего сервера, так и для конкретной таблицы, но для этого значение табличной настройки должно быть меньше серверной. Иначе сервер будет учитывать только настройку `max_replicated_sends_network_bandwidth_for_server`. + +Настройка следуется неточно. + +Возможные значения: + +- Любое целое положительное число. +- 0 — Скорость не ограничена. + +Значение по умолчанию: `0`. + +**Использование** + +Может быть использована для ограничения скорости сети при репликации данных для добавления или замены новых узлов. + ## max_bytes_to_merge_at_max_space_in_pool {#max-bytes-to-merge-at-max-space-in-pool} Максимальный суммарный размер кусков (в байтах) в одном слиянии, если есть свободные ресурсы в фоновом пуле. diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 56b04e81a94..742d86bdd9e 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -237,6 +237,39 @@ ClickHouse применяет настройку в тех случаях, ко В случае превышения `input_format_allow_errors_ratio` ClickHouse генерирует исключение. +## input_format_parquet_import_nested {#input_format_parquet_import_nested} + +Включает или отключает возможность вставки данных в колонки типа [Nested](../../sql-reference/data-types/nested-data-structures/nested.md) в виде массива структур в формате ввода [Parquet](../../interfaces/formats.md#data-format-parquet). + +Возможные значения: + +- 0 — данные не могут быть вставлены в колонки типа `Nested` в виде массива структур. +- 0 — данные могут быть вставлены в колонки типа `Nested` в виде массива структур. + +Значение по умолчанию: `0`. + +## input_format_arrow_import_nested {#input_format_arrow_import_nested} + +Включает или отключает возможность вставки данных в колонки типа [Nested](../../sql-reference/data-types/nested-data-structures/nested.md) в виде массива структур в формате ввода [Arrow](../../interfaces/formats.md#data_types-matching-arrow). + +Возможные значения: + +- 0 — данные не могут быть вставлены в колонки типа `Nested` в виде массива структур. +- 0 — данные могут быть вставлены в колонки типа `Nested` в виде массива структур. + +Значение по умолчанию: `0`. + +## input_format_orc_import_nested {#input_format_orc_import_nested} + +Включает или отключает возможность вставки данных в колонки типа [Nested](../../sql-reference/data-types/nested-data-structures/nested.md) в виде массива структур в формате ввода [ORC](../../interfaces/formats.md#data-format-orc). + +Возможные значения: + +- 0 — данные не могут быть вставлены в колонки типа `Nested` в виде массива структур. +- 0 — данные могут быть вставлены в колонки типа `Nested` в виде массива структур. + +Значение по умолчанию: `0`. + ## input_format_values_interpret_expressions {#settings-input_format_values_interpret_expressions} Включает или отключает парсер SQL, если потоковый парсер не может проанализировать данные. Этот параметр используется только для формата [Values](../../interfaces/formats.md#data-format-values) при вставке данных. Дополнительные сведения о парсерах читайте в разделе [Синтаксис](../../sql-reference/syntax.md). @@ -1065,6 +1098,40 @@ SELECT type, query FROM system.query_log WHERE log_comment = 'log_comment test' Значение по умолчанию: `5`. +## max_replicated_fetches_network_bandwidth_for_server {#max_replicated_fetches_network_bandwidth_for_server} + +Ограничивает максимальную скорость обмена данными в сети (в байтах в секунду) для синхронизации между [репликами](../../engines/table-engines/mergetree-family/replication.md). Применяется только при запуске сервера. Можно также ограничить скорость для конкретной таблицы с помощью настройки [max_replicated_fetches_network_bandwidth](../../operations/settings/merge-tree-settings.md#max_replicated_fetches_network_bandwidth). + +Значение настройки соблюдается неточно. + +Возможные значения: + +- Любое целое положительное число. +- 0 — Скорость не ограничена. + +Значение по умолчанию: `0`. + +**Использование** + +Может быть использована для ограничения скорости сети при репликации данных для добавления или замены новых узлов. + +## max_replicated_sends_network_bandwidth_for_server {#max_replicated_sends_network_bandwidth_for_server} + +Ограничивает максимальную скорость обмена данными в сети (в байтах в секунду) для [репликационных](../../engines/table-engines/mergetree-family/replication.md) отправок. Применяется только при запуске сервера. Можно также ограничить скорость для конкретной таблицы с помощью настройки [max_replicated_sends_network_bandwidth](../../operations/settings/merge-tree-settings.md#max_replicated_sends_network_bandwidth). + +Значение настройки соблюдается неточно. + +Возможные значения: + +- Любое целое положительное число. +- 0 — Скорость не ограничена. + +Значение по умолчанию: `0`. + +**Использование** + +Может быть использована для ограничения скорости сети при репликации данных для добавления или замены новых узлов. + ## connect_timeout_with_failover_ms {#connect-timeout-with-failover-ms} Таймаут в миллисекундах на соединение с удалённым сервером, для движка таблиц Distributed, если используются секции shard и replica в описании кластера. @@ -3275,6 +3342,30 @@ SETTINGS index_granularity = 8192 │ Значение по умолчанию: `0`. +## replication_alter_partitions_sync {#replication-alter-partitions-sync} + +Позволяет настроить ожидание выполнения действий на репликах запросами [ALTER](../../sql-reference/statements/alter/index.md), [OPTIMIZE](../../sql-reference/statements/optimize.md) или [TRUNCATE](../../sql-reference/statements/truncate.md). + +Возможные значения: + +- 0 — не ждать. +- 1 — ждать выполнения действий на своей реплике. +- 2 — ждать выполнения действий на всех репликах. + +Значение по умолчанию: `1`. + +## replication_wait_for_inactive_replica_timeout {#replication-wait-for-inactive-replica-timeout} + +Указывает время ожидания (в секундах) выполнения запросов [ALTER](../../sql-reference/statements/alter/index.md), [OPTIMIZE](../../sql-reference/statements/optimize.md) или [TRUNCATE](../../sql-reference/statements/truncate.md) для неактивных реплик. + +Возможные значения: + +- 0 — не ждать. +- Отрицательное целое число — ждать неограниченное время. +- Положительное целое число — установить соответствующее количество секунд ожидания. + +Значение по умолчанию: `120` секунд. + ## regexp_max_matches_per_row {#regexp-max-matches-per-row} Задает максимальное количество совпадений для регулярного выражения. Настройка применяется для защиты памяти от перегрузки при использовании "жадных" квантификаторов в регулярном выражении для функции [extractAllGroupsHorizontal](../../sql-reference/functions/string-search-functions.md#extractallgroups-horizontal). @@ -3283,4 +3374,4 @@ SETTINGS index_granularity = 8192 │ - Положительное целое число. -Значение по умолчанию: `1000`. \ No newline at end of file +Значение по умолчанию: `1000`. diff --git a/docs/ru/operations/system-tables/replicas.md b/docs/ru/operations/system-tables/replicas.md index 7879ee707a4..ff58355145d 100644 --- a/docs/ru/operations/system-tables/replicas.md +++ b/docs/ru/operations/system-tables/replicas.md @@ -78,10 +78,11 @@ active_replicas: 2 - `log_max_index` (`UInt64`) - максимальный номер записи в общем логе действий. - `log_pointer` (`UInt64`) - максимальный номер записи из общего лога действий, которую реплика скопировала в свою очередь для выполнения, плюс единица. Если log_pointer сильно меньше log_max_index, значит что-то не так. -- `last_queue_update` (`DateTime`) - When the queue was updated last time. -- `absolute_delay` (`UInt64`) - How big lag in seconds the current replica has. +- `last_queue_update` (`DateTime`) - время последнего обновления запроса. +- `absolute_delay` (`UInt64`) - задержка (в секундах) для текущей реплики. - `total_replicas` (`UInt8`) - общее число известных реплик этой таблицы. - `active_replicas` (`UInt8`) - число реплик этой таблицы, имеющих сессию в ZK; то есть, число работающих реплик. +- `replica_is_active` ([Map(String, UInt8)](../../sql-reference/data-types/map.md)) — соответствие между именем реплики и признаком активности реплики. Если запрашивать все столбцы, то таблица может работать слегка медленно, так как на каждую строчку делается несколько чтений из ZK. Если не запрашивать последние 4 столбца (log_max_index, log_pointer, total_replicas, active_replicas), то таблица работает быстро. diff --git a/docs/ru/sql-reference/data-types/nested-data-structures/nested.md b/docs/ru/sql-reference/data-types/nested-data-structures/nested.md index 718fe77ae95..db957e57502 100644 --- a/docs/ru/sql-reference/data-types/nested-data-structures/nested.md +++ b/docs/ru/sql-reference/data-types/nested-data-structures/nested.md @@ -1,4 +1,6 @@ -# Nested(Name1 Type1, Name2 Type2, …) {#nestedname1-type1-name2-type2} +# Nested {#nested} + +## Nested(Name1 Type1, Name2 Type2, …) {#nestedname1-type1-name2-type2} Вложенная структура данных - это как будто вложенная таблица. Параметры вложенной структуры данных - имена и типы столбцов, указываются так же, как у запроса CREATE. Каждой строке таблицы может соответствовать произвольное количество строк вложенной структуры данных. @@ -95,4 +97,3 @@ LIMIT 10 При запросе DESCRIBE, столбцы вложенной структуры данных перечисляются так же по отдельности. Работоспособность запроса ALTER для элементов вложенных структур данных, является сильно ограниченной. - diff --git a/docs/ru/sql-reference/functions/encoding-functions.md b/docs/ru/sql-reference/functions/encoding-functions.md index 161c1304b7c..694dfef7d75 100644 --- a/docs/ru/sql-reference/functions/encoding-functions.md +++ b/docs/ru/sql-reference/functions/encoding-functions.md @@ -17,13 +17,13 @@ char(number_1, [number_2, ..., number_n]); **Аргументы** -- `number_1, number_2, ..., number_n` — числовые аргументы, которые интерпретируются как целые числа. Типы: [Int](../../sql-reference/functions/encoding-functions.md), [Float](../../sql-reference/functions/encoding-functions.md). +- `number_1, number_2, ..., number_n` — числовые аргументы, которые интерпретируются как целые числа. Типы: [Int](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md). **Возвращаемое значение** - Строка из соответствующих байт. -Тип: `String`. +Тип: [String](../../sql-reference/data-types/string.md). **Пример** @@ -73,61 +73,57 @@ SELECT char(0xE4, 0xBD, 0xA0, 0xE5, 0xA5, 0xBD) AS hello; ## hex {#hex} -Returns a string containing the argument’s hexadecimal representation. +Возвращает строку, содержащую шестнадцатеричное представление аргумента. Синоним: `HEX`. -**Syntax** +**Синтаксис** ``` sql hex(arg) ``` -The function is using uppercase letters `A-F` and not using any prefixes (like `0x`) or suffixes (like `h`). +Функция использует прописные буквы `A-F` и не использует никаких префиксов (например, `0x`) или суффиксов (например, `h`). -For integer arguments, it prints hex digits («nibbles») from the most significant to least significant (big endian or «human readable» order). It starts with the most significant non-zero byte (leading zero bytes are omitted) but always prints both digits of every byte even if leading digit is zero. +Для целочисленных аргументов возвращает шестнадцатеричные цифры от наиболее до наименее значимых (`big endian`, человекочитаемый порядок).Он начинается с самого значимого ненулевого байта (начальные нулевые байты опущены), но всегда выводит обе цифры каждого байта, даже если начальная цифра равна нулю. -Example: +Значения типа [Date](../../sql-reference/data-types/date.md) и [DateTime](../../sql-reference/data-types/datetime.md) формируются как соответствующие целые числа (количество дней с момента Unix-эпохи для `Date` и значение Unix Timestamp для `DateTime`). -**Example** +Для [String](../../sql-reference/data-types/string.md) и [FixedString](../../sql-reference/data-types/fixedstring.md), все байты просто кодируются как два шестнадцатеричных числа. Нулевые байты не опущены. -Query: +Значения [Float](../../sql-reference/data-types/float.md) и [Decimal](../../sql-reference/data-types/decimal.md) кодируются как их представление в памяти. Поскольку ClickHouse поддерживает архитектуру `little-endian`, они кодируются от младшего к старшему байту. Нулевые начальные/конечные байты не опущены. + +**Аргументы** + +- `arg` — значение для преобразования в шестнадцатеричное. [String](../../sql-reference/data-types/string.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md), [Decimal](../../sql-reference/data-types/decimal.md), [Date](../../sql-reference/data-types/date.md) или [DateTime](../../sql-reference/data-types/datetime.md). + +**Возвращаемое значение** + +- Строка — шестнадцатеричное представление аргумента. + +Тип: [String](../../sql-reference/data-types/string.md). + +**Примеры** + +Запрос: ``` sql SELECT hex(1); ``` -Result: +Результат: ``` text 01 ``` -Values of type `Date` and `DateTime` are formatted as corresponding integers (the number of days since Epoch for Date and the value of Unix Timestamp for DateTime). - -For `String` and `FixedString`, all bytes are simply encoded as two hexadecimal numbers. Zero bytes are not omitted. - -Values of floating point and Decimal types are encoded as their representation in memory. As we support little endian architecture, they are encoded in little endian. Zero leading/trailing bytes are not omitted. - -**Parameters** - -- `arg` — A value to convert to hexadecimal. Types: [String](../../sql-reference/functions/encoding-functions.md), [UInt](../../sql-reference/functions/encoding-functions.md), [Float](../../sql-reference/functions/encoding-functions.md), [Decimal](../../sql-reference/functions/encoding-functions.md), [Date](../../sql-reference/functions/encoding-functions.md) or [DateTime](../../sql-reference/functions/encoding-functions.md). - -**Returned value** - -- A string with the hexadecimal representation of the argument. - -Type: `String`. - -**Example** - -Query: +Запрос: ``` sql -SELECT hex(toFloat32(number)) as hex_presentation FROM numbers(15, 2); +SELECT hex(toFloat32(number)) AS hex_presentation FROM numbers(15, 2); ``` -Result: +Результат: ``` text ┌─hex_presentation─┐ @@ -136,13 +132,13 @@ Result: └──────────────────┘ ``` -Query: +Запрос: ``` sql -SELECT hex(toFloat64(number)) as hex_presentation FROM numbers(15, 2); +SELECT hex(toFloat64(number)) AS hex_presentation FROM numbers(15, 2); ``` -Result: +Результат: ``` text ┌─hex_presentation─┐ @@ -208,6 +204,141 @@ SELECT reinterpretAsUInt64(reverse(unhex('FFF'))) AS num; └──────┘ ``` +## bin {#bin} + +Возвращает строку, содержащую бинарное представление аргумента. + +**Синтаксис** + +``` sql +bin(arg) +``` + +Синоним: `BIN`. + +Для целочисленных аргументов возвращаются двоичные числа от наиболее значимого до наименее значимого (`big-endian`, человекочитаемый порядок). Порядок начинается с самого значимого ненулевого байта (начальные нулевые байты опущены), но всегда возвращает восемь цифр каждого байта, если начальная цифра равна нулю. + +Значения типа [Date](../../sql-reference/data-types/date.md) и [DateTime](../../sql-reference/data-types/datetime.md) формируются как соответствующие целые числа (количество дней с момента Unix-эпохи для `Date` и значение Unix Timestamp для `DateTime`). + +Для [String](../../sql-reference/data-types/string.md) и [FixedString](../../sql-reference/data-types/fixedstring.md) все байты кодируются как восемь двоичных чисел. Нулевые байты не опущены. + +Значения [Float](../../sql-reference/data-types/float.md) и [Decimal](../../sql-reference/data-types/decimal.md) кодируются как их представление в памяти. Поскольку ClickHouse поддерживает архитектуру `little-endian`, они кодируются от младшего к старшему байту. Нулевые начальные/конечные байты не опущены. + +**Аргументы** + +- `arg` — значение для преобразования в двоичный код. [String](../../sql-reference/data-types/string.md), [FixedString](../../sql-reference/data-types/fixedstring.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md), [Decimal](../../sql-reference/data-types/decimal.md), [Date](../../sql-reference/data-types/date.md) или [DateTime](../../sql-reference/data-types/datetime.md). + +**Возвращаемое значение** + +- Бинарная строка (BLOB) — двоичное представление аргумента. + +Тип: [String](../../sql-reference/data-types/string.md). + +**Примеры** + +Запрос: + +``` sql +SELECT bin(14); +``` + +Результат: + +``` text +┌─bin(14)──┐ +│ 00001110 │ +└──────────┘ +``` + +Запрос: + +``` sql +SELECT bin(toFloat32(number)) AS bin_presentation FROM numbers(15, 2); +``` + +Результат: + +``` text +┌─bin_presentation─────────────────┐ +│ 00000000000000000111000001000001 │ +│ 00000000000000001000000001000001 │ +└──────────────────────────────────┘ +``` + +Запрос: + +``` sql +SELECT bin(toFloat64(number)) AS bin_presentation FROM numbers(15, 2); +``` + +Результат: + +``` text +┌─bin_presentation─────────────────────────────────────────────────┐ +│ 0000000000000000000000000000000000000000000000000010111001000000 │ +│ 0000000000000000000000000000000000000000000000000011000001000000 │ +└──────────────────────────────────────────────────────────────────┘ +``` + +## unbin {#unbinstr} + +Интерпретирует каждую пару двоичных цифр аргумента как число и преобразует его в байт, представленный числом. Функция выполняет операцию, противоположную [bin](#bin). + +**Синтаксис** + +``` sql +unbin(arg) +``` + +Синоним: `UNBIN`. + +Для числового аргумента `unbin()` не возвращает значение, обратное результату `bin()`. Чтобы преобразовать результат в число, используйте функции [reverse](../../sql-reference/functions/string-functions.md#reverse) и [reinterpretAs](../../sql-reference/functions/type-conversion-functions.md#reinterpretasuint8163264). + +!!! note "Примечание" + Если `unbin` вызывается из клиента `clickhouse-client`, бинарная строка возвращается в кодировке UTF-8. + +Поддерживает двоичные цифры `0` и `1`. Количество двоичных цифр не обязательно должно быть кратно восьми. Если строка аргумента содержит что-либо, кроме двоичных цифр, возвращается некоторый результат, определенный реализацией (ошибки не возникает). + +**Аргументы** + +- `arg` — строка, содержащая любое количество двоичных цифр. [String](../../sql-reference/data-types/string.md). + +**Возвращаемое значение** + +- Бинарная строка (BLOB). + +Тип: [String](../../sql-reference/data-types/string.md). + +**Примеры** + +Запрос: + +``` sql +SELECT UNBIN('001100000011000100110010'), UNBIN('0100110101111001010100110101000101001100'); +``` + +Результат: + +``` text +┌─unbin('001100000011000100110010')─┬─unbin('0100110101111001010100110101000101001100')─┐ +│ 012 │ MySQL │ +└───────────────────────────────────┴───────────────────────────────────────────────────┘ +``` + +Запрос: + +``` sql +SELECT reinterpretAsUInt64(reverse(unbin('1110'))) AS num; +``` + +Результат: + +``` text +┌─num─┐ +│ 14 │ +└─────┘ +``` + ## UUIDStringToNum(str) {#uuidstringtonumstr} Принимает строку, содержащую 36 символов в формате `123e4567-e89b-12d3-a456-426655440000`, и возвращает в виде набора байт в FixedString(16). @@ -263,7 +394,7 @@ SELECT bitPositionsToArray(toInt8(1)) AS bit_positions; Запрос: ``` sql -select bitPositionsToArray(toInt8(-1)) as bit_positions; +SELECT bitPositionsToArray(toInt8(-1)) AS bit_positions; ``` Результат: diff --git a/docs/ru/sql-reference/functions/index.md b/docs/ru/sql-reference/functions/index.md index 15da9d36ef5..92bd1c1c2f8 100644 --- a/docs/ru/sql-reference/functions/index.md +++ b/docs/ru/sql-reference/functions/index.md @@ -58,6 +58,10 @@ str -> str != Referer Для некоторых функций первый аргумент (лямбда-функция) может отсутствовать. В этом случае подразумевается тождественное отображение. +## Пользовательские функции {#user-defined-functions} + +Функции можно создавать с помощью выражения [CREATE FUNCTION](../statements/create/function.md). Для удаления таких функций используется выражение [DROP FUNCTION](../statements/drop.md#drop-function). + ## Обработка ошибок {#obrabotka-oshibok} Некоторые функции могут кидать исключения в случае ошибочных данных. В этом случае, выполнение запроса прерывается, и текст ошибки выводится клиенту. При распределённой обработке запроса, при возникновении исключения на одном из серверов, на другие серверы пытается отправиться просьба тоже прервать выполнение запроса. diff --git a/docs/ru/sql-reference/functions/tuple-map-functions.md b/docs/ru/sql-reference/functions/tuple-map-functions.md index 4775152fb54..e4cc1fefab4 100644 --- a/docs/ru/sql-reference/functions/tuple-map-functions.md +++ b/docs/ru/sql-reference/functions/tuple-map-functions.md @@ -73,22 +73,22 @@ SELECT a['key2'] FROM table_map; **Синтаксис** ``` sql -mapAdd(Tuple(Array, Array), Tuple(Array, Array) [, ...]) +mapAdd(arg1, arg2 [, ...]) ``` **Аргументы** -Аргументами являются [кортежи](../../sql-reference/data-types/tuple.md#tuplet1-t2) из двух [массивов](../../sql-reference/data-types/array.md#data-type-array), где элементы в первом массиве представляют ключи, а второй массив содержит значения для каждого ключа. +Аргументами являются контейнеры [Map](../../sql-reference/data-types/map.md) или [кортежи](../../sql-reference/data-types/tuple.md#tuplet1-t2) из двух [массивов](../../sql-reference/data-types/array.md#data-type-array), где элементы в первом массиве представляют ключи, а второй массив содержит значения для каждого ключа. Все массивы ключей должны иметь один и тот же тип, а все массивы значений должны содержать элементы, которые можно приводить к одному типу ([Int64](../../sql-reference/data-types/int-uint.md#int-ranges), [UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges) или [Float64](../../sql-reference/data-types/float.md#float32-float64)). Общий приведенный тип используется в качестве типа для результирующего массива. **Возвращаемое значение** -- Возвращает один [кортеж](../../sql-reference/data-types/tuple.md#tuplet1-t2), в котором первый массив содержит отсортированные ключи, а второй — значения. +- В зависимости от типа аргументов возвращает один [Map](../../sql-reference/data-types/map.md) или [кортеж](../../sql-reference/data-types/tuple.md#tuplet1-t2), в котором первый массив содержит отсортированные ключи, а второй — значения. **Пример** -Запрос: +Запрос с кортежем: ``` sql SELECT mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])) as res, toTypeName(res) as type; @@ -102,6 +102,20 @@ SELECT mapAdd(([toUInt8(1), 2], [1, 1]), ([toUInt8(1), 2], [1, 1])) as res, toTy └───────────────┴────────────────────────────────────┘ ``` +Запрос с контейнером `Map`: + +```sql +SELECT mapAdd(map(1,1), map(1,1)); +``` + +Result: + +```text +┌─mapAdd(map(1, 1), map(1, 1))─┐ +│ {1:2} │ +└──────────────────────────────┘ +``` + ## mapSubtract {#function-mapsubtract} Собирает все ключи и вычитает соответствующие значения. diff --git a/docs/ru/sql-reference/statements/alter/index.md b/docs/ru/sql-reference/statements/alter/index.md index 043ac3839d9..2b7caa5ad5b 100644 --- a/docs/ru/sql-reference/statements/alter/index.md +++ b/docs/ru/sql-reference/statements/alter/index.md @@ -64,8 +64,11 @@ ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name Для нереплицируемых таблиц, все запросы `ALTER` выполняются синхронно. Для реплицируемых таблиц, запрос всего лишь добавляет инструкцию по соответствующим действиям в `ZooKeeper`, а сами действия осуществляются при первой возможности. Но при этом, запрос может ждать завершения выполнения этих действий на всех репликах. -Для запросов `ALTER ... ATTACH|DETACH|DROP` можно настроить ожидание, с помощью настройки `replication_alter_partitions_sync`. -Возможные значения: `0` - не ждать, `1` - ждать выполнения только у себя (по умолчанию), `2` - ждать всех. +Для всех запросов `ALTER` можно настроить ожидание с помощью настройки [replication_alter_partitions_sync](../../../operations/settings/settings.md#replication-alter-partitions-sync). + +Вы можете указать время ожидания (в секундах) выполнения всех запросов `ALTER` для неактивных реплик с помощью настройки [replication_wait_for_inactive_replica_timeout](../../../operations/settings/settings.md#replication-wait-for-inactive-replica-timeout). + +!!! info "Примечание" + Для всех запросов `ALTER` при `replication_alter_partitions_sync = 2` и неактивности некоторых реплик больше времени, заданного настройкой `replication_wait_for_inactive_replica_timeout`, генерируется исключение `UNFINISHED`. Для запросов `ALTER TABLE ... UPDATE|DELETE` синхронность выполнения определяется настройкой [mutations_sync](../../../operations/settings/settings.md#mutations_sync). - diff --git a/docs/ru/sql-reference/statements/alter/index/index.md b/docs/ru/sql-reference/statements/alter/index/index.md index 1f6bbea5c4b..c3d1e75a77d 100644 --- a/docs/ru/sql-reference/statements/alter/index/index.md +++ b/docs/ru/sql-reference/statements/alter/index/index.md @@ -19,7 +19,7 @@ ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name Команда `ADD INDEX` добавляет описание индексов в метаданные, а `DROP INDEX` удаляет индекс из метаданных и стирает файлы индекса с диска, поэтому они легковесные и работают мгновенно. Если индекс появился в метаданных, то он начнет считаться в последующих слияниях и записях в таблицу, а не сразу после выполнения операции `ALTER`. -`MATERIALIZE INDEX` - перестраивает индекс в указанной партиции. Реализовано как мутация. +`MATERIALIZE INDEX` - перестраивает индекс в указанной партиции. Реализовано как мутация. В случае если нужно перестроить индекс над всеми данными то писать `IN PARTITION` не нужно. Запрос на изменение индексов реплицируется, сохраняя новые метаданные в ZooKeeper и применяя изменения на всех репликах. diff --git a/docs/ru/sql-reference/statements/create/function.md b/docs/ru/sql-reference/statements/create/function.md new file mode 100644 index 00000000000..90838b25744 --- /dev/null +++ b/docs/ru/sql-reference/statements/create/function.md @@ -0,0 +1,59 @@ +--- +toc_priority: 38 +toc_title: FUNCTION +--- + +# CREATE FUNCTION {#create-function} + +Создает пользовательскую функцию из лямбда-выражения. Выражение должно состоять из параметров функции, констант, операторов и вызовов других функций. + +**Синтаксис** + +```sql +CREATE FUNCTION name AS (parameter0, ...) -> expression +``` +У функции может быть произвольное число параметров. + +Существует несколько ограничений на создаваемые функции: + +- Имя функции должно быть уникальным среди всех пользовательских и системных функций. +- Рекурсивные функции запрещены. +- Все переменные, используемые функцией, должны быть перечислены в списке ее параметров. + +Если какое-нибудь ограничение нарушается, то при попытке создать функцию возникает исключение. + +**Пример** + +Запрос: + +```sql +CREATE FUNCTION linear_equation AS (x, k, b) -> k*x + b; +SELECT number, linear_equation(number, 2, 1) FROM numbers(3); +``` + +Результат: + +``` text +┌─number─┬─plus(multiply(2, number), 1)─┐ +│ 0 │ 1 │ +│ 1 │ 3 │ +│ 2 │ 5 │ +└────────┴──────────────────────────────┘ +``` + +В следующем запросе пользовательская функция вызывает [условную функцию](../../../sql-reference/functions/conditional-functions.md): + +```sql +CREATE FUNCTION parity_str AS (n) -> if(n % 2, 'odd', 'even'); +SELECT number, parity_str(number) FROM numbers(3); +``` + +Результат: + +``` text +┌─number─┬─if(modulo(number, 2), 'odd', 'even')─┐ +│ 0 │ even │ +│ 1 │ odd │ +│ 2 │ even │ +└────────┴──────────────────────────────────────┘ +``` diff --git a/docs/ru/sql-reference/statements/create/index.md b/docs/ru/sql-reference/statements/create/index.md index dfa5c28fff7..61d4d053fec 100644 --- a/docs/ru/sql-reference/statements/create/index.md +++ b/docs/ru/sql-reference/statements/create/index.md @@ -12,6 +12,7 @@ toc_title: "Обзор" - [TABLE](../../../sql-reference/statements/create/table.md) - [VIEW](../../../sql-reference/statements/create/view.md) - [DICTIONARY](../../../sql-reference/statements/create/dictionary.md) +- [FUNCTION](../../../sql-reference/statements/create/function.md) - [USER](../../../sql-reference/statements/create/user.md) - [ROLE](../../../sql-reference/statements/create/role.md) - [ROW POLICY](../../../sql-reference/statements/create/row-policy.md) diff --git a/docs/ru/sql-reference/statements/drop.md b/docs/ru/sql-reference/statements/drop.md index 118f8eb923a..437c2d02a94 100644 --- a/docs/ru/sql-reference/statements/drop.md +++ b/docs/ru/sql-reference/statements/drop.md @@ -97,3 +97,20 @@ DROP [SETTINGS] PROFILE [IF EXISTS] name [,...] [ON CLUSTER cluster_name] DROP VIEW [IF EXISTS] [db.]name [ON CLUSTER cluster] ``` +## DROP FUNCTION {#drop-function} + +Удаляет пользовательскую функцию, созданную с помощью [CREATE FUNCTION](./create/function.md). +Удалить системные функции нельзя. + +**Синтаксис** + +``` sql +DROP FUNCTION [IF EXISTS] function_name +``` + +**Пример** + +``` sql +CREATE FUNCTION linear_equation AS (x, k, b) -> k*x + b; +DROP FUNCTION linear_equation; +``` diff --git a/docs/ru/sql-reference/statements/grant.md b/docs/ru/sql-reference/statements/grant.md index 8d6605e1571..45ba9bb0343 100644 --- a/docs/ru/sql-reference/statements/grant.md +++ b/docs/ru/sql-reference/statements/grant.md @@ -109,11 +109,13 @@ GRANT SELECT(x,y) ON db.table TO john WITH GRANT OPTION - `CREATE TEMPORARY TABLE` - `CREATE VIEW` - `CREATE DICTIONARY` + - `CREATE FUNCTION` - [DROP](#grant-drop) - `DROP DATABASE` - `DROP TABLE` - `DROP VIEW` - `DROP DICTIONARY` + - `DROP FUNCTION` - [TRUNCATE](#grant-truncate) - [OPTIMIZE](#grant-optimize) - [SHOW](#grant-show) diff --git a/docs/ru/sql-reference/statements/optimize.md b/docs/ru/sql-reference/statements/optimize.md index 1f0c5a0ebe9..e6a71c4f611 100644 --- a/docs/ru/sql-reference/statements/optimize.md +++ b/docs/ru/sql-reference/statements/optimize.md @@ -18,7 +18,7 @@ OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION I Может применяться к таблицам семейства [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md), [MaterializedView](../../engines/table-engines/special/materializedview.md) и [Buffer](../../engines/table-engines/special/buffer.md). Другие движки таблиц не поддерживаются. -Если запрос `OPTIMIZE` применяется к таблицам семейства [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md), ClickHouse создаёт задачу на слияние и ожидает её исполнения на всех узлах (если активирована настройка `replication_alter_partitions_sync`). +Если запрос `OPTIMIZE` применяется к таблицам семейства [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md), ClickHouse создаёт задачу на слияние и ожидает её исполнения на всех репликах (если значение настройки [replication_alter_partitions_sync](../../operations/settings/settings.md#replication-alter-partitions-sync) равно `2`) или на текущей реплике (если значение настройки [replication_alter_partitions_sync](../../operations/settings/settings.md#replication-alter-partitions-sync) равно `1`). - По умолчанию, если запросу `OPTIMIZE` не удалось выполнить слияние, то ClickHouse не оповещает клиента. Чтобы включить оповещения, используйте настройку [optimize_throw_if_noop](../../operations/settings/settings.md#setting-optimize_throw_if_noop). @@ -26,6 +26,11 @@ ClickHouse не оповещает клиента. Чтобы включить - Если указать `FINAL`, то оптимизация выполняется даже в том случае, если все данные уже лежат в одном куске данных. Кроме того, слияние является принудительным, даже если выполняются параллельные слияния. - Если указать `DEDUPLICATE`, то произойдет схлопывание полностью одинаковых строк (сравниваются значения во всех столбцах), имеет смысл только для движка MergeTree. +Вы можете указать время ожидания (в секундах) выполнения запросов `OPTIMIZE` для неактивных реплик с помощью настройки [replication_wait_for_inactive_replica_timeout](../../operations/settings/settings.md#replication-wait-for-inactive-replica-timeout). + +!!! info "Примечание" + Если значение настройки `replication_alter_partitions_sync` равно `2` и некоторые реплики не активны больше времени, заданного настройкой `replication_wait_for_inactive_replica_timeout`, то генерируется исключение `UNFINISHED`. + ## Выражение BY {#by-expression} Чтобы выполнить дедупликацию по произвольному набору столбцов, вы можете явно указать список столбцов или использовать любую комбинацию подстановки [`*`](../../sql-reference/statements/select/index.md#asterisk), выражений [`COLUMNS`](../../sql-reference/statements/select/index.md#columns-expression) и [`EXCEPT`](../../sql-reference/statements/select/index.md#except-modifier). diff --git a/docs/ru/sql-reference/statements/truncate.md b/docs/ru/sql-reference/statements/truncate.md index 63f7fa86ea5..028959690cd 100644 --- a/docs/ru/sql-reference/statements/truncate.md +++ b/docs/ru/sql-reference/statements/truncate.md @@ -13,4 +13,9 @@ TRUNCATE TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] Запрос `TRUNCATE` не поддерживается для следующих движков: [View](../../engines/table-engines/special/view.md), [File](../../engines/table-engines/special/file.md), [URL](../../engines/table-engines/special/url.md), [Buffer](../../engines/table-engines/special/buffer.md) и [Null](../../engines/table-engines/special/null.md). +Вы можете настроить ожидание выполнения действий на репликах с помощью настройки [replication_alter_partitions_sync](../../operations/settings/settings.md#replication-alter-partitions-sync). +Вы можете указать время ожидания (в секундах) выполнения запросов `TRUNCATE` для неактивных реплик с помощью настройки [replication_wait_for_inactive_replica_timeout](../../operations/settings/settings.md#replication-wait-for-inactive-replica-timeout). + +!!! info "Примечание" + Если значение настройки `replication_alter_partitions_sync` равно `2` и некоторые реплики не активны больше времени, заданного настройкой `replication_wait_for_inactive_replica_timeout`, то генерируется исключение `UNFINISHED`. diff --git a/docs/ru/sql-reference/table-functions/s3.md b/docs/ru/sql-reference/table-functions/s3.md index 597f145c096..c8dbcf81559 100644 --- a/docs/ru/sql-reference/table-functions/s3.md +++ b/docs/ru/sql-reference/table-functions/s3.md @@ -133,6 +133,30 @@ INSERT INTO FUNCTION s3('https://storage.yandexcloud.net/my-test-bucket-768/test SELECT name, value FROM existing_table; ``` +## Партиционирование при записи данных {#partitioned-write} + +Если при добавлении данных в таблицу S3 указать выражение `PARTITION BY`, то для каждого значения ключа партиционирования создается отдельный файл. Это повышает эффективность операций чтения. + +**Примеры** + +1. При использовании ID партиции в имени ключа создаются отдельные файлы: + +```sql +INSERT INTO TABLE FUNCTION + s3('http://bucket.amazonaws.com/my_bucket/file_{_partition_id}.csv', 'CSV', 'a UInt32, b UInt32, c UInt32') + PARTITION BY a VALUES ('x', 2, 3), ('x', 4, 5), ('y', 11, 12), ('y', 13, 14), ('z', 21, 22), ('z', 23, 24); +``` +В результате данные будут записаны в три файла: `file_x.csv`, `file_y.csv` и `file_z.csv`. + +2. При использовании ID партиции в названии бакета создаются файлы в разных бакетах: + +```sql +INSERT INTO TABLE FUNCTION + s3('http://bucket.amazonaws.com/my_bucket_{_partition_id}/file.csv', 'CSV', 'a UInt32, b UInt32, c UInt32') + PARTITION BY a VALUES (1, 2, 3), (1, 4, 5), (10, 11, 12), (10, 13, 14), (20, 21, 22), (20, 23, 24); +``` +В результате будут созданы три файла в разных бакетах: `my_bucket_1/file.csv`, `my_bucket_10/file.csv` и `my_bucket_20/file.csv`. + **Смотрите также** - [Движок таблиц S3](../../engines/table-engines/integrations/s3.md) diff --git a/programs/client/CMakeLists.txt b/programs/client/CMakeLists.txt index 1de5ea88aee..9e774deadf0 100644 --- a/programs/client/CMakeLists.txt +++ b/programs/client/CMakeLists.txt @@ -4,6 +4,7 @@ set (CLICKHOUSE_CLIENT_SOURCES QueryFuzzer.cpp Suggest.cpp TestHint.cpp + TestTags.cpp ) set (CLICKHOUSE_CLIENT_LINK diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 87ae03161a9..568928c28a3 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -6,6 +6,7 @@ #include "QueryFuzzer.h" #include "Suggest.h" #include "TestHint.h" +#include "TestTags.h" #if USE_REPLXX # include @@ -1031,19 +1032,30 @@ private: if (server_exception) { bool print_stack_trace = config().getBool("stacktrace", false); - std::cerr << "Received exception from server (version " << server_version << "):" << std::endl - << getExceptionMessage(*server_exception, print_stack_trace, true) << std::endl; + fmt::print(stderr, "Received exception from server (version {}):\n{}\n", + server_version, + getExceptionMessage(*server_exception, print_stack_trace, true)); if (is_interactive) - std::cerr << std::endl; + { + fmt::print(stderr, "\n"); + } + else + { + fmt::print(stderr, "(query: {})\n", full_query); + } } if (client_exception) { - fmt::print(stderr, "Error on processing query '{}':\n{}\n", full_query, client_exception->message()); + fmt::print(stderr, "Error on processing query: {}\n", client_exception->message()); if (is_interactive) { fmt::print(stderr, "\n"); } + else + { + fmt::print(stderr, "(query: {})\n", full_query); + } } // A debug check -- at least some exception must be set, if the error @@ -1067,12 +1079,17 @@ private: bool echo_query = echo_queries; + /// Test tags are started with "--" so they are interpreted as comments anyway. + /// But if the echo is enabled we have to remove the test tags from `all_queries_text` + /// because we don't want test tags to be echoed. + size_t test_tags_length = test_mode ? getTestTagsLength(all_queries_text) : 0; + /// Several queries separated by ';'. /// INSERT data is ended by the end of line, not ';'. /// An exception is VALUES format where we also support semicolon in /// addition to end of line. - const char * this_query_begin = all_queries_text.data(); + const char * this_query_begin = all_queries_text.data() + test_tags_length; const char * all_queries_end = all_queries_text.data() + all_queries_text.size(); while (this_query_begin < all_queries_end) @@ -1244,13 +1261,17 @@ private: if (!server_exception) { error_matches_hint = false; - fmt::print(stderr, "Expected server error code '{}' but got no server error.\n", test_hint.serverError()); + fmt::print(stderr, "Expected server error code '{}' but got no server error (query: {}).\n", + test_hint.serverError(), + full_query); } else if (server_exception->code() != test_hint.serverError()) { error_matches_hint = false; - std::cerr << "Expected server error code: " << test_hint.serverError() << " but got: " << server_exception->code() - << "." << std::endl; + fmt::print(stderr, "Expected server error code: {} but got: {} (query: {}).\n", + test_hint.serverError(), + server_exception->code(), + full_query); } } @@ -1259,13 +1280,17 @@ private: if (!client_exception) { error_matches_hint = false; - fmt::print(stderr, "Expected client error code '{}' but got no client error.\n", test_hint.clientError()); + fmt::print(stderr, "Expected client error code '{}' but got no client error (query: {}).\n", + test_hint.clientError(), + full_query); } else if (client_exception->code() != test_hint.clientError()) { error_matches_hint = false; - fmt::print( - stderr, "Expected client error code '{}' but got '{}'.\n", test_hint.clientError(), client_exception->code()); + fmt::print(stderr, "Expected client error code '{}' but got '{}' (query: {}).\n", + test_hint.clientError(), + client_exception->code(), + full_query); } } @@ -1281,13 +1306,17 @@ private: { if (test_hint.clientError()) { - fmt::print(stderr, "The query succeeded but the client error '{}' was expected.\n", test_hint.clientError()); + fmt::print(stderr, "The query succeeded but the client error '{}' was expected (query: {}).\n", + test_hint.clientError(), + full_query); error_matches_hint = false; } if (test_hint.serverError()) { - fmt::print(stderr, "The query succeeded but the server error '{}' was expected.\n", test_hint.serverError()); + fmt::print(stderr, "The query succeeded but the server error '{}' was expected (query: {}).\n", + test_hint.serverError(), + full_query); error_matches_hint = false; } } @@ -2010,8 +2039,21 @@ private: PullingAsyncPipelineExecutor executor(pipeline); Block block; - while (executor.pull(block)) + while (true) { + try + { + if (!executor.pull(block)) + { + break; + } + } + catch (Exception & e) + { + e.addMessage(fmt::format("(in query: {})", full_query)); + throw; + } + /// Check if server send Log packet receiveLogs(); diff --git a/programs/client/TestTags.cpp b/programs/client/TestTags.cpp new file mode 100644 index 00000000000..f3cb49cdabd --- /dev/null +++ b/programs/client/TestTags.cpp @@ -0,0 +1,51 @@ +#include "TestTags.h" + +#include + +namespace DB +{ + +size_t getTestTagsLength(const String & multiline_query) +{ + const String & text = multiline_query; + size_t pos = 0; + bool first_line = true; + + while (true) + { + size_t line_start = pos; + + /// Skip spaces. + while ((pos != text.length()) && (text[pos] == ' ' || text[pos] == '\t')) + ++pos; + + /// Skip comment "--". + static constexpr const char comment[] = "--"; + if (text.compare(pos, strlen(comment), comment) != 0) + return line_start; + pos += strlen(comment); + + /// Skip the prefix "Tags:" if it's the first line. + if (first_line) + { + while ((pos != text.length()) && (text[pos] == ' ' || text[pos] == '\t')) + ++pos; + + static constexpr const char tags_prefix[] = "Tags:"; + if (text.compare(pos, strlen(tags_prefix), tags_prefix) != 0) + return 0; + pos += strlen(tags_prefix); + first_line = false; + } + + /// Skip end-of-line. + size_t eol_pos = text.find_first_of("\r\n", pos); + if (eol_pos == String::npos) + return text.length(); + bool two_chars_eol = (eol_pos + 1 < text.length()) && ((text[eol_pos + 1] == '\r') || (text[eol_pos + 1] == '\n')) && (text[eol_pos + 1] != text[eol_pos]); + size_t eol_length = two_chars_eol ? 2 : 1; + pos = eol_pos + eol_length; + } +} + +} diff --git a/programs/client/TestTags.h b/programs/client/TestTags.h new file mode 100644 index 00000000000..e2e36698cb5 --- /dev/null +++ b/programs/client/TestTags.h @@ -0,0 +1,18 @@ +#pragma once + +#include + +namespace DB +{ + +/// Returns the length of a text looking like +/// -- Tags: x, y, z +/// -- Tag x: explanation of tag x +/// -- Tag y: explanation of tag y +/// -- Tag z: explanation of tag z +/// +/// at the beginning of a multiline query. +/// If there are no test tags in the multiline query the function returns 0. +size_t getTestTagsLength(const String & multiline_query); + +} diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 2b1b6185321..d2661874beb 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -179,20 +180,18 @@ void LocalServer::tryInitPath() } -static void attachSystemTables(ContextPtr context) +static DatabasePtr createMemoryDatabaseIfNotExists(ContextPtr context, const String & database_name) { - DatabasePtr system_database = DatabaseCatalog::instance().tryGetDatabase(DatabaseCatalog::SYSTEM_DATABASE); + DatabasePtr system_database = DatabaseCatalog::instance().tryGetDatabase(database_name); if (!system_database) { /// TODO: add attachTableDelayed into DatabaseMemory to speedup loading - system_database = std::make_shared(DatabaseCatalog::SYSTEM_DATABASE, context); - DatabaseCatalog::instance().attachDatabase(DatabaseCatalog::SYSTEM_DATABASE, system_database); + system_database = std::make_shared(database_name, context); + DatabaseCatalog::instance().attachDatabase(database_name, system_database); } - - attachSystemTablesLocal(*system_database); + return system_database; } - int LocalServer::main(const std::vector & /*args*/) try { @@ -246,6 +245,8 @@ try /// Sets external authenticators config (LDAP, Kerberos). global_context->setExternalAuthenticatorsConfig(config()); + global_context->initializeBackgroundExecutors(); + setupUsers(); /// Limit on total number of concurrently executing queries. @@ -301,14 +302,18 @@ try fs::create_directories(fs::path(path) / "data/"); fs::create_directories(fs::path(path) / "metadata/"); loadMetadataSystem(global_context); - attachSystemTables(global_context); + attachSystemTablesLocal(*createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE)); + attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA)); + attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE)); loadMetadata(global_context); DatabaseCatalog::instance().loadDatabases(); LOG_DEBUG(log, "Loaded metadata."); } else if (!config().has("no-system-tables")) { - attachSystemTables(global_context); + attachSystemTablesLocal(*createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::SYSTEM_DATABASE)); + attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA)); + attachInformationSchema(global_context, *createMemoryDatabaseIfNotExists(global_context, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE)); } processQueries(); @@ -393,6 +398,7 @@ void LocalServer::processQueries() auto context = session.makeQueryContext(); context->makeSessionContext(); /// initial_create_query requires a session context to be set. context->setCurrentQueryId(""); + applyCmdSettings(context); /// Use the same query_id (and thread group) for all queries diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index c09b4f774fa..2d09495c338 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include @@ -45,19 +45,18 @@ #include #include #include +#include +#include #include #include #include #include -#include -#include -#include -#include -#include #include +#include #include #include #include +#include #include #include #include @@ -80,6 +79,7 @@ #include #include #include +#include #if !defined(ARCADIA_BUILD) # include "config_core.h" @@ -252,7 +252,6 @@ namespace ErrorCodes extern const int SUPPORT_IS_DISABLED; extern const int ARGUMENT_OUT_OF_BOUND; extern const int EXCESSIVE_ELEMENT_IN_CONFIG; - extern const int INCORRECT_DATA; extern const int INVALID_CONFIG_PARAMETER; extern const int SYSTEM_ERROR; extern const int FAILED_TO_GETPWUID; @@ -457,40 +456,6 @@ void checkForUsersNotInMainConfig( } } -static void loadEncryptionKey(const std::string & key_command [[maybe_unused]], Poco::Logger * log) -{ -#if USE_BASE64 && USE_SSL && USE_INTERNAL_SSL_LIBRARY - - auto process = ShellCommand::execute(key_command); - - std::string b64_key; - readStringUntilEOF(b64_key, process->out); - process->wait(); - - // turbob64 doesn't like whitespace characters in input. Strip - // them before decoding. - std::erase_if(b64_key, [](char c) - { - return c == ' ' || c == '\t' || c == '\r' || c == '\n'; - }); - - std::vector buf(b64_key.size()); - const size_t key_size = tb64dec(reinterpret_cast(b64_key.data()), b64_key.size(), - reinterpret_cast(buf.data())); - if (!key_size) - throw Exception("Failed to decode encryption key", ErrorCodes::INCORRECT_DATA); - else if (key_size < 16) - LOG_WARNING(log, "The encryption key should be at least 16 octets long."); - - const std::string_view key = std::string_view(buf.data(), key_size); - CompressionCodecEncrypted::setMasterKey(key); - -#else - LOG_WARNING(log, "Server was built without Base64 or SSL support. Encryption is disabled."); -#endif -} - - [[noreturn]] void forceShutdown() { #if defined(THREAD_SANITIZER) && defined(OS_LINUX) @@ -549,6 +514,8 @@ if (ThreadFuzzer::instance().isEffective()) // ignore `max_thread_pool_size` in configs we fetch from ZK, but oh well. GlobalThreadPool::initialize(config().getUInt("max_thread_pool_size", 10000)); + global_context->initializeBackgroundExecutors(); + ConnectionCollector::init(global_context, config().getUInt("max_threads_for_connection_collector", 10)); bool has_zookeeper = config().has("zookeeper"); @@ -903,6 +870,8 @@ if (ThreadFuzzer::instance().isEffective()) global_context->updateStorageConfiguration(*config); global_context->updateInterserverCredentials(*config); + + CompressionCodecEncrypted::Configuration::instance().tryLoad(*config, "encryption_codecs"); }, /* already_loaded = */ false); /// Reload it right now (initial loading) @@ -961,7 +930,7 @@ if (ThreadFuzzer::instance().isEffective()) global_context->setMMappedFileCache(mmap_cache_size); #if USE_EMBEDDED_COMPILER - constexpr size_t compiled_expression_cache_size_default = 1024 * 1024 * 1024; + constexpr size_t compiled_expression_cache_size_default = 1024 * 1024 * 128; size_t compiled_expression_cache_size = config().getUInt64("compiled_expression_cache_size", compiled_expression_cache_size_default); CompiledExpressionCacheFactory::instance().init(compiled_expression_cache_size); #endif @@ -975,9 +944,9 @@ if (ThreadFuzzer::instance().isEffective()) global_context->getMergeTreeSettings().sanityCheck(settings); global_context->getReplicatedMergeTreeSettings().sanityCheck(settings); - /// Set up encryption. - if (config().has("encryption.key_command")) - loadEncryptionKey(config().getString("encryption.key_command"), log); + + /// try set up encryption. There are some errors in config, error will be printed and server wouldn't start. + CompressionCodecEncrypted::Configuration::instance().load(config(), "encryption_codecs"); Poco::Timespan keep_alive_timeout(config().getUInt("keep_alive_timeout", 10), 0); @@ -1131,6 +1100,8 @@ if (ThreadFuzzer::instance().isEffective()) global_context->setSystemZooKeeperLogAfterInitializationIfNeeded(); /// After the system database is created, attach virtual system tables (in addition to query_log and part_log) attachSystemTablesServer(*database_catalog.getSystemDatabase(), has_zookeeper); + attachInformationSchema(global_context, *database_catalog.getDatabase(DatabaseCatalog::INFORMATION_SCHEMA)); + attachInformationSchema(global_context, *database_catalog.getDatabase(DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE)); /// Firstly remove partially dropped databases, to avoid race with MaterializedMySQLSyncThread, /// that may execute DROP before loadMarkedAsDroppedTables() in background, /// and so loadMarkedAsDroppedTables() will find it and try to add, and UUID will overlap. @@ -1510,7 +1481,7 @@ if (ThreadFuzzer::instance().isEffective()) server.start(); LOG_INFO(log, "Ready for connections."); - SCOPE_EXIT({ + SCOPE_EXIT_SAFE({ LOG_DEBUG(log, "Received termination signal."); LOG_DEBUG(log, "Waiting for current connections to close."); diff --git a/programs/server/config.xml b/programs/server/config.xml index 18bb23c6227..6c98ac740fe 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -331,7 +331,7 @@ 1000 - 1073741824 + 134217728 /var/lib/clickhouse/ @@ -965,6 +965,14 @@ 1000 + + + system + session_log
+ + toYYYYMM(event_date) + 7500 + - - - - + + + + + + + + + + + + + + + + + + + + + + diff --git a/programs/server/config.yaml.example b/programs/server/config.yaml.example index 5b2da1d3128..ae4eac49a64 100644 --- a/programs/server/config.yaml.example +++ b/programs/server/config.yaml.example @@ -280,7 +280,7 @@ mark_cache_size: 5368709120 mmap_cache_size: 1000 # Cache size for compiled expressions. -compiled_expression_cache_size: 1073741824 +compiled_expression_cache_size: 134217728 # Path to data directory, with trailing slash. path: /var/lib/clickhouse/ diff --git a/programs/server/users.d/session_log_test.xml b/programs/server/users.d/session_log_test.xml new file mode 120000 index 00000000000..85377f51630 --- /dev/null +++ b/programs/server/users.d/session_log_test.xml @@ -0,0 +1 @@ +../../../tests/config/users.d/session_log_test.xml \ No newline at end of file diff --git a/src/Access/AccessControlManager.h b/src/Access/AccessControlManager.h index 15079ea61be..79f7073ef69 100644 --- a/src/Access/AccessControlManager.h +++ b/src/Access/AccessControlManager.h @@ -143,10 +143,11 @@ public: std::vector getAllQuotasUsage() const; - std::shared_ptr getEnabledSettings(const UUID & user_id, - const SettingsProfileElements & settings_from_user, - const boost::container::flat_set & enabled_roles, - const SettingsProfileElements & settings_from_enabled_roles) const; + std::shared_ptr getEnabledSettings( + const UUID & user_id, + const SettingsProfileElements & settings_from_user, + const boost::container::flat_set & enabled_roles, + const SettingsProfileElements & settings_from_enabled_roles) const; std::shared_ptr getSettingsProfileInfo(const UUID & profile_id); diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 39b57a40e7a..5a615ad8fd6 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -119,8 +119,10 @@ namespace AccessRights res = access; res.modifyFlags(modifier); - /// Anyone has access to the "system" database. + /// Anyone has access to the "system" and "information_schema" database. res.grant(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE); + res.grant(AccessType::SELECT, DatabaseCatalog::INFORMATION_SCHEMA); + res.grant(AccessType::SELECT, DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE); return res; } diff --git a/src/Access/SettingsProfilesInfo.h b/src/Access/SettingsProfilesInfo.h index d1fba0e9f5f..f554ba8d89e 100644 --- a/src/Access/SettingsProfilesInfo.h +++ b/src/Access/SettingsProfilesInfo.h @@ -36,6 +36,16 @@ struct SettingsProfilesInfo friend bool operator ==(const SettingsProfilesInfo & lhs, const SettingsProfilesInfo & rhs); friend bool operator !=(const SettingsProfilesInfo & lhs, const SettingsProfilesInfo & rhs) { return !(lhs == rhs); } + Strings getProfileNames() const + { + Strings result; + result.reserve(profiles.size()); + for (const auto & profile_id : profiles) + result.push_back(names_of_profiles.at(profile_id)); + + return result; + } + private: const AccessControlManager & manager; }; diff --git a/src/AggregateFunctions/AggregateFunctionArray.cpp b/src/AggregateFunctions/AggregateFunctionArray.cpp index 982180ab50c..3591bea5f9e 100644 --- a/src/AggregateFunctions/AggregateFunctionArray.cpp +++ b/src/AggregateFunctions/AggregateFunctionArray.cpp @@ -21,6 +21,8 @@ class AggregateFunctionCombinatorArray final : public IAggregateFunctionCombinat public: String getName() const override { return "Array"; } + bool supportsNesting() const override { return true; } + DataTypes transformArguments(const DataTypes & arguments) const override { if (arguments.empty()) diff --git a/src/AggregateFunctions/AggregateFunctionAvg.h b/src/AggregateFunctions/AggregateFunctionAvg.h index ad5c67d88d4..fd8f9faa3f8 100644 --- a/src/AggregateFunctions/AggregateFunctionAvg.h +++ b/src/AggregateFunctions/AggregateFunctionAvg.h @@ -21,11 +21,9 @@ namespace DB { struct Settings; -template -using DecimalOrVectorCol = std::conditional_t, ColumnDecimal, ColumnVector>; template constexpr bool DecimalOrExtendedInt = - IsDecimalNumber + is_decimal || std::is_same_v || std::is_same_v || std::is_same_v @@ -44,7 +42,7 @@ struct AvgFraction /// Invoked only is either Numerator or Denominator are Decimal. Float64 NO_SANITIZE_UNDEFINED divideIfAnyDecimal(UInt32 num_scale, UInt32 denom_scale [[maybe_unused]]) const { - if constexpr (IsDecimalNumber && IsDecimalNumber) + if constexpr (is_decimal && is_decimal) { // According to the docs, num(S1) / denom(S2) would have scale S1 @@ -60,7 +58,7 @@ struct AvgFraction /// Numerator is always casted to Float64 to divide correctly if the denominator is not Float64. Float64 num_converted; - if constexpr (IsDecimalNumber) + if constexpr (is_decimal) num_converted = DecimalUtils::convertTo(numerator, num_scale); else num_converted = static_cast(numerator); /// all other types, including extended integral. @@ -68,7 +66,7 @@ struct AvgFraction std::conditional_t, Float64, Denominator> denom_converted; - if constexpr (IsDecimalNumber) + if constexpr (is_decimal) denom_converted = DecimalUtils::convertTo(denominator, denom_scale); else if constexpr (DecimalOrExtendedInt) /// no way to divide Float64 and extended integral type without an explicit cast. @@ -139,7 +137,7 @@ public: void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override { - if constexpr (IsDecimalNumber || IsDecimalNumber) + if constexpr (is_decimal || is_decimal) assert_cast &>(to).getData().push_back( this->data(place).divideIfAnyDecimal(num_scale, denom_scale)); else @@ -222,7 +220,7 @@ private: }; template -using AvgFieldType = std::conditional_t, +using AvgFieldType = std::conditional_t, std::conditional_t, Decimal256, Decimal128>, NearestFieldType>; @@ -239,7 +237,7 @@ public: void NO_SANITIZE_UNDEFINED add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena *) const final { - this->data(place).numerator += static_cast &>(*columns[0]).getData()[row_num]; + this->data(place).numerator += static_cast &>(*columns[0]).getData()[row_num]; ++this->data(place).denominator; } diff --git a/src/AggregateFunctions/AggregateFunctionAvgWeighted.h b/src/AggregateFunctions/AggregateFunctionAvgWeighted.h index 68d48803718..6189ae92ce4 100644 --- a/src/AggregateFunctions/AggregateFunctionAvgWeighted.h +++ b/src/AggregateFunctions/AggregateFunctionAvgWeighted.h @@ -8,7 +8,7 @@ namespace DB struct Settings; template -using AvgWeightedFieldType = std::conditional_t, +using AvgWeightedFieldType = std::conditional_t, std::conditional_t, Decimal256, Decimal128>, std::conditional_t, Float64, // no way to do UInt128 * UInt128, better cast to Float64 @@ -34,10 +34,10 @@ public: void NO_SANITIZE_UNDEFINED add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena *) const override { - const auto& weights = static_cast &>(*columns[1]); + const auto& weights = static_cast &>(*columns[1]); this->data(place).numerator += static_cast( - static_cast &>(*columns[0]).getData()[row_num]) * + static_cast &>(*columns[0]).getData()[row_num]) * static_cast(weights.getData()[row_num]); this->data(place).denominator += static_cast(weights.getData()[row_num]); diff --git a/src/AggregateFunctions/AggregateFunctionFactory.cpp b/src/AggregateFunctions/AggregateFunctionFactory.cpp index c9dcdb54424..c9a44dba6f2 100644 --- a/src/AggregateFunctions/AggregateFunctionFactory.cpp +++ b/src/AggregateFunctions/AggregateFunctionFactory.cpp @@ -29,6 +29,7 @@ namespace ErrorCodes { extern const int UNKNOWN_AGGREGATE_FUNCTION; extern const int LOGICAL_ERROR; + extern const int ILLEGAL_AGGREGATION; } const String & getAggregateFunctionCanonicalNameIfAny(const String & name) @@ -159,13 +160,32 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl( if (AggregateFunctionCombinatorPtr combinator = AggregateFunctionCombinatorFactory::instance().tryFindSuffix(name)) { + const std::string & combinator_name = combinator->getName(); + if (combinator->isForInternalUsageOnly()) - throw Exception("Aggregate function combinator '" + combinator->getName() + "' is only for internal usage", ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION); + throw Exception(ErrorCodes::UNKNOWN_AGGREGATE_FUNCTION, + "Aggregate function combinator '{}' is only for internal usage", + combinator_name); if (query_context && query_context->getSettingsRef().log_queries) - query_context->addQueryFactoriesInfo(Context::QueryLogFactories::AggregateFunctionCombinator, combinator->getName()); + query_context->addQueryFactoriesInfo(Context::QueryLogFactories::AggregateFunctionCombinator, combinator_name); + + String nested_name = name.substr(0, name.size() - combinator_name.size()); + /// Nested identical combinators (i.e. uniqCombinedIfIf) is not + /// supported (since they even don't work -- silently). + /// + /// But non-identical does supported and works, for example + /// uniqCombinedIfMergeIf, it is useful in case when the underlying + /// storage stores AggregateFunction(uniqCombinedIf) and in SELECT you + /// need to filter aggregation result based on another column for + /// example. + if (!combinator->supportsNesting() && nested_name.ends_with(combinator_name)) + { + throw Exception(ErrorCodes::ILLEGAL_AGGREGATION, + "Nested identical combinator '{}' is not supported", + combinator_name); + } - String nested_name = name.substr(0, name.size() - combinator->getName().size()); DataTypes nested_types = combinator->transformArguments(argument_types); Array nested_parameters = combinator->transformParameters(parameters); diff --git a/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.cpp b/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.cpp index e4eff6be901..3e0f1849545 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.cpp +++ b/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.cpp @@ -25,14 +25,14 @@ namespace template struct MovingSum { - using Data = MovingSumData, Decimal128, NearestFieldType>>; + using Data = MovingSumData, Decimal128, NearestFieldType>>; using Function = MovingImpl; }; template struct MovingAvg { - using Data = MovingAvgData, Decimal128, Float64>>; + using Data = MovingAvgData, Decimal128, Float64>>; using Function = MovingImpl; }; diff --git a/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.h b/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.h index daaa7a69baf..a3b7dc841b6 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.h +++ b/src/AggregateFunctions/AggregateFunctionGroupArrayMoving.h @@ -87,18 +87,10 @@ class MovingImpl final public: using ResultT = typename Data::Accumulator; - using ColumnSource = std::conditional_t, - ColumnDecimal, - ColumnVector>; + using ColumnSource = ColumnVectorOrDecimal; /// Probably for overflow function in the future. - using ColumnResult = std::conditional_t, - ColumnDecimal, - ColumnVector>; - - using DataTypeResult = std::conditional_t, - DataTypeDecimal, - DataTypeNumber>; + using ColumnResult = ColumnVectorOrDecimal; explicit MovingImpl(const DataTypePtr & data_type_, UInt64 window_size_ = std::numeric_limits::max()) : IAggregateFunctionDataHelper>({data_type_}, {}) @@ -106,14 +98,7 @@ public: String getName() const override { return Data::name; } - DataTypePtr getReturnType() const override - { - if constexpr (IsDecimalNumber) - return std::make_shared(std::make_shared( - DataTypeResult::maxPrecision(), getDecimalScale(*this->argument_types.at(0)))); - else - return std::make_shared(std::make_shared()); - } + DataTypePtr getReturnType() const override { return std::make_shared(getReturnTypeElement()); } void NO_SANITIZE_UNDEFINED add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override { @@ -196,6 +181,18 @@ public: { return true; } + +private: + auto getReturnTypeElement() const + { + if constexpr (!is_decimal) + return std::make_shared>(); + else + { + using Res = DataTypeDecimal; + return std::make_shared(Res::maxPrecision(), getDecimalScale(*this->argument_types.at(0))); + } + } }; #undef AGGREGATE_FUNCTION_MOVING_MAX_ARRAY_SIZE diff --git a/src/AggregateFunctions/AggregateFunctionIf.cpp b/src/AggregateFunctions/AggregateFunctionIf.cpp index 5082952f386..4ac6a2dce21 100644 --- a/src/AggregateFunctions/AggregateFunctionIf.cpp +++ b/src/AggregateFunctions/AggregateFunctionIf.cpp @@ -10,7 +10,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int ILLEGAL_AGGREGATION; } class AggregateFunctionCombinatorIf final : public IAggregateFunctionCombinator @@ -37,10 +36,6 @@ public: const DataTypes & arguments, const Array & params) const override { - if (nested_function->getName().find(getName()) != String::npos) - { - throw Exception(ErrorCodes::ILLEGAL_AGGREGATION, "nested function for {0}-combinator must not have {0}-combinator", getName()); - } return std::make_shared(nested_function, arguments, params); } }; diff --git a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h index e5471b8a727..8fa4b235c2c 100644 --- a/src/AggregateFunctions/AggregateFunctionMinMaxAny.h +++ b/src/AggregateFunctions/AggregateFunctionMinMaxAny.h @@ -44,7 +44,7 @@ struct SingleValueDataFixed { private: using Self = SingleValueDataFixed; - using ColVecType = std::conditional_t, ColumnDecimal, ColumnVector>; + using ColVecType = ColumnVectorOrDecimal; bool has_value = false; /// We need to remember if at least one value has been passed. This is necessary for AggregateFunctionIf. T value; diff --git a/src/AggregateFunctions/AggregateFunctionOrFill.cpp b/src/AggregateFunctions/AggregateFunctionOrFill.cpp index 3ba20e65e32..7aee0289879 100644 --- a/src/AggregateFunctions/AggregateFunctionOrFill.cpp +++ b/src/AggregateFunctions/AggregateFunctionOrFill.cpp @@ -23,6 +23,9 @@ private: public: explicit AggregateFunctionCombinatorOrFill(Kind kind_) : kind(kind_) {} + /// Due to aggregate_functions_null_for_empty + bool supportsNesting() const override { return true; } + String getName() const override { return kind == Kind::OrNull ? "OrNull" : "OrDefault"; diff --git a/src/AggregateFunctions/AggregateFunctionQuantile.h b/src/AggregateFunctions/AggregateFunctionQuantile.h index 65d1df9fe77..5ca52f6d73a 100644 --- a/src/AggregateFunctions/AggregateFunctionQuantile.h +++ b/src/AggregateFunctions/AggregateFunctionQuantile.h @@ -67,10 +67,10 @@ class AggregateFunctionQuantile final : public IAggregateFunctionDataHelper> { private: - using ColVecType = std::conditional_t, ColumnDecimal, ColumnVector>; + using ColVecType = ColumnVectorOrDecimal; static constexpr bool returns_float = !(std::is_same_v); - static_assert(!IsDecimalNumber || !returns_float); + static_assert(!is_decimal || !returns_float); QuantileLevels levels; diff --git a/src/AggregateFunctions/AggregateFunctionSparkbar.cpp b/src/AggregateFunctions/AggregateFunctionSparkbar.cpp new file mode 100644 index 00000000000..7f1196173a7 --- /dev/null +++ b/src/AggregateFunctions/AggregateFunctionSparkbar.cpp @@ -0,0 +1,72 @@ +#include +#include +#include +#include + + +namespace DB +{ + +struct Settings; + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + +namespace +{ + +template